Skip to content

NO-JIRA: Enable test-e2e-encryption #2087

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Jan 29, 2026
Merged

NO-JIRA: Enable test-e2e-encryption #2087

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ verify-podnetworkconnectivitychecks:
$(MAKE) -C pkg/operator/connectivitycheckcontroller verify

test-e2e-encryption: GO_TEST_PACKAGES :=./test/e2e-encryption/...
test-e2e-encryption: GO_TEST_FLAGS += -v
test-e2e-encryption: test-unit
.PHONY: test-e2e-encryption

test-e2e-monitoring: GO_TEST_PACKAGES :=./test/e2e-monitoring/...
Expand Down
147 changes: 112 additions & 35 deletions test/e2e-encryption/encryption_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@ import (

"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
Expand All @@ -25,7 +26,6 @@ import (
"k8s.io/client-go/kubernetes"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/retry"
"k8s.io/utils/clock"
"sigs.k8s.io/yaml"

Expand Down Expand Up @@ -67,11 +67,11 @@ func TestEncryptionIntegration(tt *testing.T) {
kubeClient, err := kubernetes.NewForConfig(kubeConfig)
require.NoError(t, err)
kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, "openshift-config-managed")
apiextensionsClient, err := v1beta1.NewForConfig(kubeConfig)
apiextensionsClient, err := v1.NewForConfig(kubeConfig)
require.NoError(t, err)

// create ExtensionTest operator CRD
var operatorCRD apiextensionsv1beta1.CustomResourceDefinition
var operatorCRD apiextensionsv1.CustomResourceDefinition
require.NoError(t, yaml.Unmarshal([]byte(encryptionTestOperatorCRD), &operatorCRD))
crd, err := apiextensionsClient.CustomResourceDefinitions().Create(ctx, &operatorCRD, metav1.CreateOptions{})
if errors.IsAlreadyExists(err) {
Expand All @@ -81,6 +81,23 @@ func TestEncryptionIntegration(tt *testing.T) {
}
defer apiextensionsClient.CustomResourceDefinitions().Delete(ctx, crd.Name, metav1.DeleteOptions{})

t.Logf("Waiting for CRD to be ready")
err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
oCRD, crdErr := apiextensionsClient.CustomResourceDefinitions().Get(ctx, operatorCRD.Name, metav1.GetOptions{})
if crdErr != nil {
return false, crdErr
}

for _, condition := range oCRD.Status.Conditions {
Copy link
Copy Markdown
Contributor

@p0lyn0mial p0lyn0mial Jan 29, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could we use apiextensions.IsCRDConditionTrue(oCRD, apiextensions.Established) ?

Copy link
Copy Markdown
Member Author

@ardaguclu ardaguclu Jan 29, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It appears that IsCRDConditionTrue accepts internal CustomResourceDefinition but in our case, we have external CustomResourceDefinition.

if condition.Type == apiextensionsv1.Established {
operatorCRD = *oCRD
return condition.Status == apiextensionsv1.ConditionTrue, nil
}
}
return false, nil
})
require.NoError(t, err)

// create operator client and create instance with ManagementState="Managed"
operatorGVR := schema.GroupVersionResource{Group: operatorCRD.Spec.Group, Version: "v1", Resource: operatorCRD.Spec.Names.Plural}
operatorGVK := schema.GroupVersionKind{Group: operatorCRD.Spec.Group, Version: "v1", Kind: operatorCRD.Spec.Names.Kind}
Expand All @@ -99,7 +116,7 @@ func TestEncryptionIntegration(tt *testing.T) {
require.NoError(t, err)
dynamicClient, err := dynamic.NewForConfig(kubeConfig)
require.NoError(t, err)
err = wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
err = wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
Copy link
Copy Markdown
Contributor

@p0lyn0mial p0lyn0mial Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here we use wait.ForeverTestTimeout :)

_, err := dynamicClient.Resource(operatorGVR).Create(ctx, &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "operator.openshift.io/v1",
Expand All @@ -121,7 +138,7 @@ func TestEncryptionIntegration(tt *testing.T) {
require.NoError(t, err)

// create APIServer clients
fakeConfigClient := configv1clientfake.NewSimpleClientset(&configv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}})
fakeConfigClient := configv1clientfake.NewClientset(&configv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "cluster"}})
fakeConfigInformer := configv1informers.NewSharedInformerFactory(fakeConfigClient, 10*time.Minute)
fakeApiServerClient := fakeConfigClient.ConfigV1().APIServers()

Expand Down Expand Up @@ -159,6 +176,26 @@ func TestEncryptionIntegration(tt *testing.T) {
operatorInformer.Start(stopCh)
go controllers.Run(ctx, 1)

t.Logf("Waiting for informers to sync")
for gvr, synced := range operatorInformer.WaitForCacheSync(stopCh) {
if !synced {
t.Fatalf("informer for %v not synced", gvr)
}
}
for gvr, synced := range fakeConfigInformer.WaitForCacheSync(stopCh) {
if !synced {
t.Fatalf("informer for %v not synced", gvr)
}
}
for ns, nsSynced := range kubeInformers.WaitForCacheSync(stopCh) {
for gvr, synced := range nsSynced {
if !synced {
t.Fatalf("informer for %v in namespace %v not synced", gvr, ns)
}
}
}
t.Logf("informers are sync'ed")

waitForConfigEventuallyCond := func(cond func(s string) bool) {
t.Helper()
stopCh := time.After(wait.ForeverTestTimeout)
Expand Down Expand Up @@ -208,22 +245,16 @@ func TestEncryptionIntegration(tt *testing.T) {
}
return operatorv1.ConditionUnknown
}
requireConditionStatus := func(condType string, expected operatorv1.ConditionStatus) {
t.Helper()
if status := conditionStatus(condType); status != expected {
t.Errorf("expected condition %s of status %s, found: %q", condType, expected, status)
}
}
waitForConditionStatus := func(condType string, expected operatorv1.ConditionStatus) {
t.Helper()
err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
return conditionStatus(condType) == expected, nil
})
require.NoError(t, err)
}
waitForMigration := func(key string) {
t.Helper()
err := wait.PollImmediate(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
s, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-%s", component, key), metav1.GetOptions{})
require.NoError(t, err)

Expand All @@ -245,7 +276,7 @@ func TestEncryptionIntegration(tt *testing.T) {
keySecretsLabel := fmt.Sprintf("%s=%s", secrets.EncryptionKeySecretsLabel, component)
waitForKeys := func(n int) {
t.Helper()
err := wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
err := wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
l, err := kubeClient.CoreV1().Secrets("openshift-config-managed").List(ctx, metav1.ListOptions{LabelSelector: keySecretsLabel})
if err != nil {
return false, err
Expand All @@ -264,7 +295,7 @@ func TestEncryptionIntegration(tt *testing.T) {
"kubeapiservers.operator.openshift.io=aescbc:1,identity;kubeschedulers.operator.openshift.io=aescbc:1,identity",
)
waitForMigration("1")
requireConditionStatus("Encrypted", operatorv1.ConditionTrue)
waitForConditionStatus("Encrypted", operatorv1.ConditionTrue)

t.Logf("Switch to identity")
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"identity"}}}`), metav1.PatchOptions{})
Expand All @@ -274,14 +305,14 @@ func TestEncryptionIntegration(tt *testing.T) {
"kubeapiservers.operator.openshift.io=aescbc:1,identity,aesgcm:2;kubeschedulers.operator.openshift.io=aescbc:1,identity,aesgcm:2",
"kubeapiservers.operator.openshift.io=identity,aescbc:1,aesgcm:2;kubeschedulers.operator.openshift.io=identity,aescbc:1,aesgcm:2",
)
requireConditionStatus("Encrypted", operatorv1.ConditionFalse)
waitForConditionStatus("Encrypted", operatorv1.ConditionFalse)

t.Logf("Switch to empty mode")
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":""}}}`), metav1.PatchOptions{})
require.NoError(t, err)
time.Sleep(5 * time.Second) // give controller time to create keys (it shouldn't)
waitForKeys(2)
requireConditionStatus("Encrypted", operatorv1.ConditionFalse)
waitForConditionStatus("Encrypted", operatorv1.ConditionFalse)

t.Logf("Switch to aescbc again")
_, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"aescbc"}}}`), metav1.PatchOptions{})
Expand All @@ -297,15 +328,11 @@ func TestEncryptionIntegration(tt *testing.T) {
t.Logf("Setting external reason")
setExternalReason := func(reason string) {
t.Helper()
err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
spec, _, rv, err := operatorClient.GetOperatorState()
if err != nil {
return err
}
spec.UnsupportedConfigOverrides.Raw = []byte(fmt.Sprintf(`{"encryption":{"reason":%q}}`, reason))
_, _, err = operatorClient.UpdateOperatorSpec(context.TODO(), rv, spec)
return err
})
applyConfig := applyoperatorv1.OperatorSpec().
WithUnsupportedConfigOverrides(runtime.RawExtension{
Raw: []byte(fmt.Sprintf(`{"encryption":{"reason":%q}}`, reason)),
})
err = operatorClient.ApplyOperatorSpec(ctx, "encryption-test", applyConfig)
require.NoError(t, err)
}
setExternalReason("a")
Expand Down Expand Up @@ -341,7 +368,7 @@ func TestEncryptionIntegration(tt *testing.T) {
require.NoError(t, err)
err = kubeClient.CoreV1().Secrets("openshift-config-managed").Delete(ctx, fmt.Sprintf("encryption-key-%s-6", component), metav1.DeleteOptions{})
require.NoError(t, err)
err = wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
err = wait.PollUntilContextTimeout(ctx, time.Second, wait.ForeverTestTimeout, true, func(ctx context.Context) (bool, error) {
_, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-7", component), metav1.GetOptions{})
if errors.IsNotFound(err) {
return false, nil
Expand Down Expand Up @@ -378,16 +405,16 @@ func TestEncryptionIntegration(tt *testing.T) {
t.Logf("Delete the openshift-config-managed config")
deployer.DeleteOperandConfig()
waitForConfigs(
// 7 is migrated and hence only one needed, but we rotate through identity
"kubeapiservers.operator.openshift.io=identity,aescbc:7;kubeschedulers.operator.openshift.io=identity,aescbc:7",
// 7 is migrated, plus one backed key (5). 6 is deleted, and therefore is not preserved (would be if the operand config was not deleted)
// 7 is migrated, backed key (5) is immediately included when rotating through identity. 6 is deleted, not preserved (would be if operand config was not deleted)
"kubeapiservers.operator.openshift.io=identity,aescbc:7,aescbc:5;kubeschedulers.operator.openshift.io=identity,aescbc:7,aescbc:5",
Copy link
Copy Markdown
Member Author

@ardaguclu ardaguclu Jan 26, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one is missed in #615

Copy link
Copy Markdown
Contributor

@p0lyn0mial p0lyn0mial Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, i think you are right.

// promote aescbc:7 back to write position
"kubeapiservers.operator.openshift.io=aescbc:7,aescbc:5,identity;kubeschedulers.operator.openshift.io=aescbc:7,aescbc:5,identity",
)
waitForConditionStatus("Encrypted", operatorv1.ConditionTrue)
}

const encryptionTestOperatorCRD = `
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: encryptiontests.operator.openshift.io
Expand All @@ -399,12 +426,62 @@ spec:
plural: encryptiontests
singular: encryptiontest
scope: Cluster
subresources:
status: {}
versions:
- name: v1
served: true
storage: true
subresources:
status: {}
schema:
openAPIV3Schema:
type: object
required:
- spec
properties:
spec:
type: object
required:
- managementState
properties:
managementState:
type: string
enum:
- Managed
- Unmanaged
- Removed
observedConfig:
type: object
nullable: true
x-kubernetes-preserve-unknown-fields: true
unsupportedConfigOverrides:
type: object
nullable: true
x-kubernetes-preserve-unknown-fields: true
status:
type: object
properties:
conditions:
type: array
items:
type: object
required:
- type
- status
properties:
type:
type: string
status:
type: string
lastTransitionTime:
type: string
format: date-time
reason:
type: string
message:
type: string
observedGeneration:
type: integer
format: int64
`

func toString(c *apiserverv1.EncryptionConfiguration) string {
Expand Down