Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/build_and_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ jobs:
- version: v1.34.2
ipFamily: ipv4
profile: xds-name-scheme-v2
gatewayApiVersion: v1.3.0
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: ./tools/github-actions/setup-deps
Expand All @@ -174,6 +175,7 @@ jobs:
IMAGE_PULL_POLICY: IfNotPresent
IP_FAMILY: ${{ matrix.target.ipFamily }}
KUBE_DEPLOY_PROFILE: ${{ matrix.target.profile }}
E2E_GATEWAY_API_VERSION: ${{ matrix.target.gatewayApiVersion }}
run: make conformance

e2e-test:
Expand Down Expand Up @@ -205,6 +207,7 @@ jobs:
- version: v1.34.2
ipFamily: ipv4
profile: xds-name-scheme-v2
gatewayApiVersion: v1.3.0

steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
Expand All @@ -229,6 +232,7 @@ jobs:
IMAGE_PULL_POLICY: IfNotPresent
IP_FAMILY: ${{ matrix.target.ipFamily }}
KUBE_DEPLOY_PROFILE: ${{ matrix.target.profile }}
E2E_GATEWAY_API_VERSION: ${{ matrix.target.gatewayApiVersion }}
E2E_TIMEOUT: 1h
NUM_WORKERS: 2
# QPS more than 3000 may cause e2e flaky test.
Expand Down
127 changes: 102 additions & 25 deletions internal/provider/kubernetes/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ type gatewayAPIReconciler struct {
extBackendGVKs []schema.GroupVersionKind
gatewayNamespaceMode bool

backendTLSPolicyGVK schema.GroupVersionKind
backendCRDExists bool
bTLSPolicyCRDExists bool
btpCRDExists bool
Expand Down Expand Up @@ -170,6 +171,10 @@ func newGatewayAPIController(ctx context.Context, mgr manager.Manager, cfg *conf
extServerPolicies: extServerPoliciesGVKs,
extBackendGVKs: extBackendGVKs,
gatewayNamespaceMode: cfg.EnvoyGateway.GatewayNamespaceMode(),
backendTLSPolicyGVK: schema.GroupVersion{
Group: gwapiv1.GroupVersion.Group,
Version: gwapiv1.GroupVersion.Version,
}.WithKind(resource.KindBackendTLSPolicy),
}

if byNamespaceSelectorEnabled(cfg.EnvoyGateway) {
Expand Down Expand Up @@ -295,6 +300,13 @@ func isTransientError(err error) bool {
errors.Is(err, context.DeadlineExceeded)
}

func (r *gatewayAPIReconciler) useBackendTLSPolicyV1Alpha3() bool {
return r.backendTLSPolicyGVK.GroupVersion() == (schema.GroupVersion{
Group: gwapiv1a3.GroupVersion.Group,
Version: gwapiv1a3.GroupVersion.Version,
})
}

// Reconcile handles reconciling all resources in a single call. Any resource event should enqueue the
// same reconcile.Request containing the gateway controller name. This allows multiple resource updates to
// be handled by a single call to Reconcile. The reconcile.Request DOES NOT map to a specific resource.
Expand Down Expand Up @@ -1880,16 +1892,12 @@ func (r *gatewayAPIReconciler) processSecurityPolicies(
func (r *gatewayAPIReconciler) processBackendTLSPolicies(
ctx context.Context, resourceTree *resource.Resources, resourceMap *resourceMappings,
) error {
backendTLSPolicies := gwapiv1.BackendTLSPolicyList{}
if err := r.client.List(ctx, &backendTLSPolicies); err != nil {
backendTLSPolicies, err := r.listBackendTLSPolicies(ctx)
if err != nil {
return fmt.Errorf("error listing BackendTLSPolicies: %w", err)
}

for i := range backendTLSPolicies.Items {
backendTLSPolicy := &backendTLSPolicies.Items[i]
// Discard Status to reduce memory consumption in watchable
// It will be recomputed by the gateway-api layer
backendTLSPolicy.Status = gwapiv1.PolicyStatus{}
for _, backendTLSPolicy := range backendTLSPolicies {
if !resourceMap.allAssociatedBackendTLSPolicies.Has(utils.NamespacedName(backendTLSPolicy).String()) {
resourceMap.allAssociatedBackendTLSPolicies.Insert(utils.NamespacedName(backendTLSPolicy).String())
resourceTree.BackendTLSPolicies = append(resourceTree.BackendTLSPolicies, backendTLSPolicy)
Expand All @@ -1900,6 +1908,80 @@ func (r *gatewayAPIReconciler) processBackendTLSPolicies(
return r.processBackendTLSPolicyRefs(ctx, resourceTree, resourceMap)
}

func (r *gatewayAPIReconciler) watchBackendTLSPolicy(mgr manager.Manager, c controller.Controller) error {
if r.useBackendTLSPolicyV1Alpha3() {
btlsPredicates := []predicate.TypedPredicate[*gwapiv1a3.BackendTLSPolicy]{
predicate.TypedGenerationChangedPredicate[*gwapiv1a3.BackendTLSPolicy]{},
}
if r.namespaceLabel != nil {
btlsPredicates = append(btlsPredicates, predicate.NewTypedPredicateFuncs(func(btp *gwapiv1a3.BackendTLSPolicy) bool {
return r.hasMatchingNamespaceLabels(btp)
}))
}

return c.Watch(
source.Kind(mgr.GetCache(), &gwapiv1a3.BackendTLSPolicy{},
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, btp *gwapiv1a3.BackendTLSPolicy) []reconcile.Request {
return r.enqueueClass(ctx, btp)
}),
btlsPredicates...))
}

btlsPredicates := []predicate.TypedPredicate[*gwapiv1.BackendTLSPolicy]{
predicate.TypedGenerationChangedPredicate[*gwapiv1.BackendTLSPolicy]{},
}
if r.namespaceLabel != nil {
btlsPredicates = append(btlsPredicates, predicate.NewTypedPredicateFuncs(func(btp *gwapiv1.BackendTLSPolicy) bool {
return r.hasMatchingNamespaceLabels(btp)
}))
}

return c.Watch(
source.Kind(mgr.GetCache(), &gwapiv1.BackendTLSPolicy{},
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, btp *gwapiv1.BackendTLSPolicy) []reconcile.Request {
return r.enqueueClass(ctx, btp)
}),
btlsPredicates...))
}

// listBackendTLSPolicies lists BackendTLSPolicy resources, handling both v1 and v1alpha3 versions to ensure compatibility.
func (r *gatewayAPIReconciler) listBackendTLSPolicies(ctx context.Context, opts ...client.ListOption) ([]*gwapiv1.BackendTLSPolicy, error) {
if !r.bTLSPolicyCRDExists {
return nil, nil
}

if !r.useBackendTLSPolicyV1Alpha3() {
backendTLSPolicies := gwapiv1.BackendTLSPolicyList{}
if err := r.client.List(ctx, &backendTLSPolicies, opts...); err != nil {
return nil, err
}
results := make([]*gwapiv1.BackendTLSPolicy, 0, len(backendTLSPolicies.Items))
for i := range backendTLSPolicies.Items {
backendTLSPolicy := &backendTLSPolicies.Items[i]
backendTLSPolicy.Status = gwapiv1.PolicyStatus{}
results = append(results, backendTLSPolicy)
}
return results, nil
}

alphaList := gwapiv1a3.BackendTLSPolicyList{}
if err := r.client.List(ctx, &alphaList, opts...); err != nil {
return nil, err
}

results := make([]*gwapiv1.BackendTLSPolicy, 0, len(alphaList.Items))
for i := range alphaList.Items {
converted := &gwapiv1.BackendTLSPolicy{}
if err := r.client.Scheme().Convert(&alphaList.Items[i], converted, nil); err != nil {
return nil, fmt.Errorf("converting BackendTLSPolicy v1alpha3 to v1: %w", err)
}
converted.Status = gwapiv1.PolicyStatus{}
results = append(results, converted)
}

return results, nil
}

// removeFinalizer removes the GatewayClass finalizer from the provided gc, if it exists.
func (r *gatewayAPIReconciler) removeFinalizer(ctx context.Context, gc *gwapiv1.GatewayClass) error {
if slice.ContainsString(gc.Finalizers, gatewayClassFinalizer) {
Expand Down Expand Up @@ -2433,28 +2515,23 @@ func (r *gatewayAPIReconciler) watchResources(ctx context.Context, mgr manager.M

r.bTLSPolicyCRDExists = r.crdExists(mgr, resource.KindBackendTLSPolicy, gwapiv1.GroupVersion.String())
if !r.bTLSPolicyCRDExists {
r.log.Info("BackendTLSPolicy CRD not found, skipping BackendTLSPolicy watch")
} else {
// Watch BackendTLSPolicy
btlsPredicates := []predicate.TypedPredicate[*gwapiv1.BackendTLSPolicy]{
predicate.TypedGenerationChangedPredicate[*gwapiv1.BackendTLSPolicy]{},
}
if r.namespaceLabel != nil {
btlsPredicates = append(btlsPredicates, predicate.NewTypedPredicateFuncs(func(btp *gwapiv1.BackendTLSPolicy) bool {
return r.hasMatchingNamespaceLabels(btp)
}))
if r.crdExists(mgr, resource.KindBackendTLSPolicy, gwapiv1a3.GroupVersion.String()) {
r.bTLSPolicyCRDExists = true
r.backendTLSPolicyGVK = schema.GroupVersion{
Group: gwapiv1a3.GroupVersion.Group,
Version: gwapiv1a3.GroupVersion.Version,
}.WithKind(resource.KindBackendTLSPolicy)
r.log.Info("BackendTLSPolicy v1 CRD not found, falling back to v1alpha3")
} else {
r.log.Info("BackendTLSPolicy CRD not found, skipping BackendTLSPolicy watch")
}

if err := c.Watch(
source.Kind(mgr.GetCache(), &gwapiv1.BackendTLSPolicy{},
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, btp *gwapiv1.BackendTLSPolicy) []reconcile.Request {
return r.enqueueClass(ctx, btp)
}),
btlsPredicates...)); err != nil {
}
if r.bTLSPolicyCRDExists {
if err := r.watchBackendTLSPolicy(mgr, c); err != nil {
return err
}

if err := addBtlsIndexers(ctx, mgr); err != nil {
if err := addBtlsIndexers(ctx, mgr, r.useBackendTLSPolicyV1Alpha3()); err != nil {
return err
}
}
Expand Down
65 changes: 64 additions & 1 deletion internal/provider/kubernetes/indexers.go
Original file line number Diff line number Diff line change
Expand Up @@ -1008,7 +1008,23 @@ func secretRouteFilterIndexFunc(rawObj client.Object) []string {
// addBtlsIndexers adds indexing on BackendTLSPolicy, for ConfigMap and Secret objects that are
// referenced in BackendTLSPolicy objects. This helps in querying for BackendTLSPolicies that are
// affected by a particular ConfigMap CRUD.
func addBtlsIndexers(ctx context.Context, mgr manager.Manager) error {
func addBtlsIndexers(ctx context.Context, mgr manager.Manager, useV1Alpha3 bool) error {
if useV1Alpha3 {
if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1a3.BackendTLSPolicy{}, configMapBtlsIndex, configMapBtlsIndexFuncV1Alpha3); err != nil {
return err
}

if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1a3.BackendTLSPolicy{}, secretBtlsIndex, secretBtlsIndexFuncV1Alpha3); err != nil {
return err
}

if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1a3.BackendTLSPolicy{}, clusterTrustBundleBtlsIndex, clusterTrustBundleBtlsIndexFuncV1Alpha3); err != nil {
return err
}

return nil
}

if err := mgr.GetFieldIndexer().IndexField(ctx, &gwapiv1.BackendTLSPolicy{}, configMapBtlsIndex, configMapBtlsIndexFunc); err != nil {
return err
}
Expand Down Expand Up @@ -1071,6 +1087,53 @@ func clusterTrustBundleBtlsIndexFunc(rawObj client.Object) []string {
return refs
}

func configMapBtlsIndexFuncV1Alpha3(rawObj client.Object) []string {
btls := rawObj.(*gwapiv1a3.BackendTLSPolicy)
var configMapReferences []string
if btls.Spec.Validation.CACertificateRefs != nil {
for _, caCertRef := range btls.Spec.Validation.CACertificateRefs {
if string(caCertRef.Kind) == resource.KindConfigMap {
configMapReferences = append(configMapReferences,
types.NamespacedName{
Namespace: btls.Namespace,
Name: string(caCertRef.Name),
}.String(),
)
}
}
}
return configMapReferences
}

func secretBtlsIndexFuncV1Alpha3(rawObj client.Object) []string {
btls := rawObj.(*gwapiv1a3.BackendTLSPolicy)
var secretReferences []string
if btls.Spec.Validation.CACertificateRefs != nil {
for _, caCertRef := range btls.Spec.Validation.CACertificateRefs {
if string(caCertRef.Kind) == resource.KindSecret {
secretReferences = append(secretReferences,
types.NamespacedName{
Namespace: btls.Namespace,
Name: string(caCertRef.Name),
}.String(),
)
}
}
}
return secretReferences
}

func clusterTrustBundleBtlsIndexFuncV1Alpha3(rawObj client.Object) []string {
btls := rawObj.(*gwapiv1a3.BackendTLSPolicy)
var refs []string
for _, caCertRef := range btls.Spec.Validation.CACertificateRefs {
if string(caCertRef.Kind) == resource.KindClusterTrustBundle {
refs = append(refs, string(caCertRef.Name))
}
}
return refs
}

// addEnvoyExtensionPolicyIndexers adds indexing on EnvoyExtensionPolicy.
// - For Service objects that are referenced in EnvoyExtensionPolicy objects via
// `.spec.extProc.[*].service.backendObjectReference`. This helps in querying for
Expand Down
24 changes: 8 additions & 16 deletions internal/provider/kubernetes/predicates.go
Original file line number Diff line number Diff line change
Expand Up @@ -253,19 +253,15 @@ func (r *gatewayAPIReconciler) isBackendReferencingClusterTrustBundle(ctb *certi
}

func (r *gatewayAPIReconciler) isBackendTLSPolicyReferencingClusterTrustBundle(ctb *certificatesv1b1.ClusterTrustBundle) bool {
btlsList := &gwapiv1.BackendTLSPolicyList{}
if err := r.client.List(context.Background(), btlsList, &client.ListOptions{
btlsList, err := r.listBackendTLSPolicies(context.Background(), &client.ListOptions{
FieldSelector: fields.OneTermEqualSelector(clusterTrustBundleBtlsIndex, ctb.Name),
}); err != nil {
})
if err != nil {
r.log.Error(err, "unable to find associated BackendTLSPolicy")
return false
}

if len(btlsList.Items) > 0 {
return true
}

return false
return len(btlsList) > 0
}

func (r *gatewayAPIReconciler) isHTTPRouteFilterReferencingSecret(nsName *types.NamespacedName) bool {
Expand All @@ -285,19 +281,15 @@ func (r *gatewayAPIReconciler) isHTTPRouteFilterReferencingSecret(nsName *types.
}

func (r *gatewayAPIReconciler) isBackendTLSPolicyReferencingSecret(nsName *types.NamespacedName) bool {
btlsList := &gwapiv1.BackendTLSPolicyList{}
if err := r.client.List(context.Background(), btlsList, &client.ListOptions{
btlsList, err := r.listBackendTLSPolicies(context.Background(), &client.ListOptions{
FieldSelector: fields.OneTermEqualSelector(secretBtlsIndex, nsName.String()),
}); err != nil {
})
if err != nil {
r.log.Error(err, "unable to find associated BackendTLSPolicy")
return false
}

if len(btlsList.Items) > 0 {
return true
}

return false
return len(btlsList) > 0
}

func (r *gatewayAPIReconciler) isEnvoyProxyReferencingSecret(nsName *types.NamespacedName) bool {
Expand Down
9 changes: 9 additions & 0 deletions test/e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,15 @@ func TestE2E(t *testing.T) {
)
}

// Skip these BTLSPolicy tests since their manifests are written in v1.
// V1alpha3 are covered in the BackendTLSTest test.
if tests.IsBTLSPolicyV1alpha3() {
skipTests = append(skipTests,
tests.BackendTLSSettingsTest.ShortName,
tests.BackendClusterTrustBundleTest.ShortName,
)
}

enabledFeatures := sets.New(features.SupportGateway)
if tests.EnabledClusterTrustBundle() {
tlog.Logf(t, "ClusterTrustBundle feature is enabled")
Expand Down
Loading
Loading