Skip to content

[release-4.21] OCPBUGS-74418: Enable test-e2e-encryption #2130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Feb 24, 2026
Merged

[release-4.21] OCPBUGS-74418: Enable test-e2e-encryption #2130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ verify-podnetworkconnectivitychecks:
$(MAKE) -C pkg/operator/connectivitycheckcontroller verify

test-e2e-encryption: GO_TEST_PACKAGES :=./test/e2e-encryption/...
test-e2e-encryption: GO_TEST_FLAGS += -v
test-e2e-encryption: test-unit
.PHONY: test-e2e-encryption

test-e2e-monitoring: GO_TEST_PACKAGES :=./test/e2e-monitoring/...
Expand Down
147 changes: 112 additions & 35 deletions test/e2e-encryption/encryption_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@ import (

"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
Expand All @@ -25,7 +26,6 @@ import (
"k8s.io/client-go/kubernetes"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/retry"
"k8s.io/utils/clock"
"sigs.k8s.io/yaml"

Expand Down Expand Up @@ -67,11 +67,11 @@ func TestEncryptionIntegration(tt *testing.T) {
kubeClient, err := kubernetes.NewForConfig(kubeConfig)
require.NoError(t, err)
kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, "openshift-config-managed")
apiextensionsClient, err := v1beta1.NewForConfig(kubeConfig)
apiextensionsClient, err := v1.NewForConfig(kubeConfig)
require.NoError(t, err)

// create ExtensionTest operator CRD
var operatorCRD apiextensionsv1beta1.CustomResourceDefinition
var operatorCRD apiextensionsv1.CustomResourceDefinition
require.NoError(t, yaml.Unmarshal([]byte(encryptionTestOperatorCRD), &operatorCRD))
crd, err := apiextensionsClient.CustomResourceDefinitions().Create(ctx, &operatorCRD, metav1.CreateOptions{})
if errors.IsAlreadyExists(err) {
Expand All @@ -81,6 +81,23 @@ func TestEncryptionIntegration(tt *testing.T) {
}
defer apiextensionsClient.CustomResourceDefinitions().Delete(ctx, crd.Name, metav1.DeleteOptions{})

t.Logf("Waiting for CRD to be ready")
err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
oCRD, crdErr := apiextensionsClient.CustomResourceDefinitions().Get(ctx, operatorCRD.Name, metav1.GetOptions{})
if crdErr != nil {
return false, crdErr
}

for _, condition := range oCRD.Status.Conditions {
if condition.Type == apiextensionsv1.Established {
operatorCRD = *oCRD
return condition.Status == apiextensionsv1.ConditionTrue, nil
}
}
return false, nil
})
require.NoError(t, err)

// create operator client and create instance with ManagementState="Managed"
operatorGVR := schema.GroupVersionResource{Group: operatorCRD.Spec.Group, Version: "v1", Resource: operatorCRD.Spec.Names.Plural}
operatorGVK := schema.GroupVersionKind{Group: operatorCRD.Spec.Group, Version: "v1", Kind: operatorCRD.Spec.Names.Kind}
Expand All @@ -99,7 +116,7 @@ func TestEncryptionIntegration(tt *testing.T) {
require.NoError(t, err)
dynamicClient, err := dynamic.NewForConfig(kubeConfig)
require.NoError(t, err)
err = wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
err = wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
_, err := dynamicClient.Resource(operatorGVR).Create(ctx, &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "operator.openshift.io/v1",
Expand All @@ -121,7 +138,7 @@ func TestEncryptionIntegration(tt *testing.T) {
require.NoError(t, err)

// create APIServer clients
fakeConfigClient := configv1clientfake.NewSimpleClientset(&configv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}})
fakeConfigClient := configv1clientfake.NewClientset(&configv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}})
fakeConfigInformer := configv1informers.NewSharedInformerFactory(fakeConfigClient, 10*time.Minute)
fakeApiServerClient := fakeConfigClient.ConfigV1().APIServers()

Expand Down Expand Up @@ -159,6 +176,26 @@ func TestEncryptionIntegration(tt *testing.T) {
operatorInformer.Start(stopCh)
go controllers.Run(ctx, 1)

t.Logf("Waiting for informers to sync")
for gvr, synced := range operatorInformer.WaitForCacheSync(stopCh) {
if !synced {
t.Fatalf("informer for %v not synced", gvr)
}
}
for gvr, synced := range fakeConfigInformer.WaitForCacheSync(stopCh) {
if !synced {
t.Fatalf("informer for %v not synced", gvr)
}
}
for ns, nsSynced := range kubeInformers.WaitForCacheSync(stopCh) {
for gvr, synced := range nsSynced {
if !synced {
t.Fatalf("informer for %v in namespace %v not synced", gvr, ns)
}
}
}
t.Logf("informers are sync'ed")

waitForConfigEventuallyCond := func(cond func(s string) bool) {
t.Helper()
stopCh := time.After(wait.ForeverTestTimeout)
Expand Down Expand Up @@ -208,22 +245,16 @@ func TestEncryptionIntegration(tt *testing.T) {
}
return operatorv1.ConditionUnknown
}
requireConditionStatus := func(condType string, expected operatorv1.ConditionStatus) {
t.Helper()
if status := conditionStatus(condType); status != expected {
t.Errorf("expected condition %s of status %s, found: %q", condType, expected, status)
}
}
waitForConditionStatus := func(condType string, expected operatorv1.ConditionStatus) {
t.Helper()
err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
return conditionStatus(condType) == expected, nil
})
require.NoError(t, err)
}
waitForMigration := func(key string) {
t.Helper()
err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
s, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-%s", component, key), metav1.GetOptions{})
require.NoError(t, err)

Expand All @@ -245,7 +276,7 @@ func TestEncryptionIntegration(tt *testing.T) {
keySecretsLabel := fmt.Sprintf("%s=%s", secrets.EncryptionKeySecretsLabel, component)
waitForKeys := func(n int) {
t.Helper()
err := wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
err := wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
l, err := kubeClient.CoreV1().Secrets("openshift-config-managed").List(ctx, metav1.ListOptions{LabelSelector: keySecretsLabel})
if err != nil {
return false, err
Expand All @@ -264,7 +295,7 @@ func TestEncryptionIntegration(tt *testing.T) {
"kubeapiservers.operator.openshift.io=aescbc:1,identity;kubeschedulers.operator.openshift.io=aescbc:1,identity",
)
waitForMigration("1")
requireConditionStatus("Encrypted", operatorv1.ConditionTrue)
waitForConditionStatus("Encrypted", operatorv1.ConditionTrue)

t.Logf("Switch to identity")
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"identity"}}}`), metav1.PatchOptions{})
Expand All @@ -274,14 +305,14 @@ func TestEncryptionIntegration(tt *testing.T) {
"kubeapiservers.operator.openshift.io=aescbc:1,identity,aesgcm:2;kubeschedulers.operator.openshift.io=aescbc:1,identity,aesgcm:2",
"kubeapiservers.operator.openshift.io=identity,aescbc:1,aesgcm:2;kubeschedulers.operator.openshift.io=identity,aescbc:1,aesgcm:2",
)
requireConditionStatus("Encrypted", operatorv1.ConditionFalse)
waitForConditionStatus("Encrypted", operatorv1.ConditionFalse)

t.Logf("Switch to empty mode")
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":""}}}`), metav1.PatchOptions{})
require.NoError(t, err)
time.Sleep(5 * time.Second) // give controller time to create keys (it shouldn't)
waitForKeys(2)
requireConditionStatus("Encrypted", operatorv1.ConditionFalse)
waitForConditionStatus("Encrypted", operatorv1.ConditionFalse)

t.Logf("Switch to aescbc again")
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"aescbc"}}}`), metav1.PatchOptions{})
Expand All @@ -297,15 +328,11 @@ func TestEncryptionIntegration(tt *testing.T) {
t.Logf("Setting external reason")
setExternalReason := func(reason string) {
t.Helper()
err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
spec, _, rv, err := operatorClient.GetOperatorState()
if err != nil {
return err
}
spec.UnsupportedConfigOverrides.Raw = []byte(fmt.Sprintf(`{"encryption":{"reason":%q}}`, reason))
_, _, err = operatorClient.UpdateOperatorSpec(context.TODO(), rv, spec)
return err
})
applyConfig := applyoperatorv1.OperatorSpec().
WithUnsupportedConfigOverrides(runtime.RawExtension{
Raw: []byte(fmt.Sprintf(`{"encryption":{"reason":%q}}`, reason)),
})
err = operatorClient.ApplyOperatorSpec(ctx, "encryption-test", applyConfig)
require.NoError(t, err)
}
setExternalReason("a")
Expand Down Expand Up @@ -341,7 +368,7 @@ func TestEncryptionIntegration(tt *testing.T) {
require.NoError(t, err)
err = kubeClient.CoreV1().Secrets("openshift-config-managed").Delete(ctx, fmt.Sprintf("encryption-key-%s-6", component), metav1.DeleteOptions{})
require.NoError(t, err)
err = wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
err = wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
_, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-7", component), metav1.GetOptions{})
if errors.IsNotFound(err) {
return false, nil
Expand Down Expand Up @@ -378,16 +405,16 @@ func TestEncryptionIntegration(tt *testing.T) {
t.Logf("Delete the openshift-config-managed config")
deployer.DeleteOperandConfig()
waitForConfigs(
// 7 is migrated and hence only one needed, but we rotate through identity
"kubeapiservers.operator.openshift.io=identity,aescbc:7;kubeschedulers.operator.openshift.io=identity,aescbc:7",
// 7 is migrated, plus one backed key (5). 6 is deleted, and therefore is not preserved (would be if the operand config was not deleted)
// 7 is migrated, backed key (5) is immediately included when rotating through identity. 6 is deleted, not preserved (would be if operand config was not deleted)
"kubeapiservers.operator.openshift.io=identity,aescbc:7,aescbc:5;kubeschedulers.operator.openshift.io=identity,aescbc:7,aescbc:5",
// promote aescbc:7 back to write position
"kubeapiservers.operator.openshift.io=aescbc:7,aescbc:5,identity;kubeschedulers.operator.openshift.io=aescbc:7,aescbc:5,identity",
)
waitForConditionStatus("Encrypted", operatorv1.ConditionTrue)
}

const encryptionTestOperatorCRD = `
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: encryptiontests.operator.openshift.io
Expand All @@ -399,12 +426,62 @@ spec:
plural: encryptiontests
singular: encryptiontest
scope: Cluster
subresources:
status: {}
versions:
- name: v1
served: true
storage: true
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
required:
- spec
properties:
spec:
type: object
required:
- managementState
properties:
managementState:
type: string
enum:
- Managed
- Unmanaged
- Removed
observedConfig:
type: object
nullable: true
x-kubernetes-preserve-unknown-fields: true
unsupportedConfigOverrides:
type: object
nullable: true
x-kubernetes-preserve-unknown-fields: true
status:
type: object
properties:
conditions:
type: array
items:
type: object
required:
- type
- status
properties:
type:
type: string
status:
type: string
lastTransitionTime:
type: string
format: date-time
reason:
type: string
message:
type: string
observedGeneration:
type: integer
format: int64
`

func toString(c *apiserverv1.EncryptionConfiguration) string {
Expand Down