From c61b8b3e99371d0748597ac45191358e4e0f8749 Mon Sep 17 00:00:00 2001 From: gangwgr Date: Fri, 12 Dec 2025 17:46:37 +0530 Subject: [PATCH 1/2] Add cert rotation tests with Ginkgo integration --- .../dependencymagnet.go | 8 + test/e2e/bound_sa_token_test.go | 1 + test/e2e/certrotation.go | 157 ++++++++++++++++++ test/e2e/certrotation_test.go | 151 ++--------------- 4 files changed, 179 insertions(+), 138 deletions(-) create mode 100644 cmd/cluster-kube-apiserver-operator-tests-ext/dependencymagnet.go create mode 100644 test/e2e/certrotation.go diff --git a/cmd/cluster-kube-apiserver-operator-tests-ext/dependencymagnet.go b/cmd/cluster-kube-apiserver-operator-tests-ext/dependencymagnet.go new file mode 100644 index 0000000000..7c54e175b2 --- /dev/null +++ b/cmd/cluster-kube-apiserver-operator-tests-ext/dependencymagnet.go @@ -0,0 +1,8 @@ +// This file imports test packages to ensure they are included in the build. +// These imports are necessary to register Ginkgo tests with the OpenShift Tests Extension framework. +package main + +import ( + // Import test packages to register Ginkgo tests + _ "github.com/openshift/cluster-kube-apiserver-operator/test/e2e" +) diff --git a/test/e2e/bound_sa_token_test.go b/test/e2e/bound_sa_token_test.go index 74d004c980..6eefc878bc 100644 --- a/test/e2e/bound_sa_token_test.go +++ b/test/e2e/bound_sa_token_test.go @@ -32,6 +32,7 @@ const ( // with respect to the resources it manages. // // Note: this test will roll out a new version - multiple times +// TODO: CNTRLPLANE-2223 - Migrate this test to OTE ginkgo framework func TestBoundTokenSignerController(t *testing.T) { kubeConfig, err := testlibrary.NewClientConfigForTest() require.NoError(t, err) diff --git a/test/e2e/certrotation.go b/test/e2e/certrotation.go new file mode 100644 index 0000000000..5f9e033a2c --- /dev/null +++ b/test/e2e/certrotation.go @@ -0,0 +1,157 @@ +package e2e + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/clock" + + configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" + configclient "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" + "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator" + "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient" + testlibrary "github.com/openshift/cluster-kube-apiserver-operator/test/library" + configv1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" + "github.com/openshift/library-go/pkg/operator/genericoperatorclient" + "github.com/openshift/library-go/pkg/operator/v1helpers" + + g "github.com/onsi/ginkgo/v2" +) + +var _ = g.Describe("[sig-api-machinery] kube-apiserver operator", func() { + g.It("[Operator][Serial] TestCertRotationTimeUpgradeable", func() { + testCertRotationTimeUpgradeable(g.GinkgoTB()) + }) + g.It("[Operator][Serial] TestCertRotationStompOnBadType", func() { + testCertRotationStompOnBadType(g.GinkgoTB()) + }) +}) + +func testCertRotationTimeUpgradeable(t testing.TB) { + kubeConfig, err := testlibrary.NewClientConfigForTest() + require.NoError(t, err) + operatorClient, _, err := genericoperatorclient.NewStaticPodOperatorClient( + clock.RealClock{}, + kubeConfig, + operatorv1.GroupVersion.WithResource("kubeapiservers"), + operatorv1.GroupVersion.WithKind("KubeAPIServer"), + operator.ExtractStaticPodOperatorSpec, + operator.ExtractStaticPodOperatorStatus) + require.NoError(t, err) + configClient, err := configclient.NewForConfig(kubeConfig) + require.NoError(t, err) + + ctx := context.Background() + _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) + require.NoError(t, err) + require.True(t, v1helpers.IsOperatorConditionTrue(operatorStatus.Conditions, "CertRotationTimeUpgradeable")) + + kubeClient := kubernetes.NewForConfigOrDie(kubeConfig) + t.Logf("Creating unsupported-cert-rotation-config...") + _, err = kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Create(context.TODO(), &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "unsupported-cert-rotation-config"}, + Data: map[string]string{"base": "2y"}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + defer func() { + kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Delete(context.TODO(), "unsupported-cert-rotation-config", metav1.DeleteOptions{}) + }() + + err = wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) { + _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) + if err != nil { + return false, err + } + clusteroperator, err := configClient.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) + if err != nil { + return false, err + } + + certRotationCondition := v1helpers.FindOperatorCondition(operatorStatus.Conditions, "CertRotationTimeUpgradeable") + upgradeableCondition := configv1helpers.FindStatusCondition(clusteroperator.Status.Conditions, "Upgradeable") + if certRotationCondition == nil || upgradeableCondition == nil { + return false, fmt.Errorf("Couldn't find CertRotationTimeUpgradeable or Upgradeable condition") + } + if certRotationCondition.Status == operatorv1.ConditionFalse && + upgradeableCondition.Status == configv1.ConditionFalse && strings.Contains(upgradeableCondition.Reason, "CertRotationTime") { + return true, nil + } + t.Logf("\nCertRotationTimeUpgradeable: %#v\nUpgradeable: %#v", certRotationCondition, upgradeableCondition) + return false, nil + }) + require.NoError(t, err) + + t.Logf("Removing unsupported-cert-rotation-config...") + err = kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Delete(context.TODO(), "unsupported-cert-rotation-config", metav1.DeleteOptions{}) + require.NoError(t, err) + + err = wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) { + _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) + if err != nil { + return false, err + } + clusteroperator, err := configClient.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) + if err != nil { + return false, err + } + certRotationCondition := v1helpers.FindOperatorCondition(operatorStatus.Conditions, "CertRotationTimeUpgradeable") + upgradeableCondition := configv1helpers.FindStatusCondition(clusteroperator.Status.Conditions, "Upgradeable") + if certRotationCondition == nil || upgradeableCondition == nil { + return false, fmt.Errorf("Couldn't find CertRotationTimeUpgradeable or Upgradeable condition") + } + if certRotationCondition.Status == operatorv1.ConditionTrue && + (upgradeableCondition.Status == configv1.ConditionTrue || !strings.Contains(upgradeableCondition.Reason, "CertRotationTime")) { + return true, nil + } + t.Logf("\nCertRotationTimeUpgradeable: %#v\nUpgradeable: %#v", certRotationCondition, upgradeableCondition) + return false, nil + }) + require.NoError(t, err) +} + +func testCertRotationStompOnBadType(t testing.TB) { + kubeConfig, err := testlibrary.NewClientConfigForTest() + require.NoError(t, err) + kubeClient := kubernetes.NewForConfigOrDie(kubeConfig) + + // this is inherently racy against a controller + err = wait.PollImmediate(10*time.Millisecond, 5*time.Second, func() (done bool, err error) { + if err := kubeClient.CoreV1().Secrets(operatorclient.OperatorNamespace).Delete(context.TODO(), "aggregator-client-signer", metav1.DeleteOptions{}); err != nil { + return false, nil + } + if _, err := kubeClient.CoreV1().Secrets(operatorclient.OperatorNamespace).Create(context.TODO(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.OperatorNamespace, Name: "aggregator-client-signer"}, + Type: "SecretTypeTLS", + }, metav1.CreateOptions{}); err != nil { + return false, nil + } + return true, nil + }) + require.NoError(t, err) + + err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (done bool, err error) { + curr, err := kubeClient.CoreV1().Secrets(operatorclient.OperatorNamespace).Get(context.TODO(), "aggregator-client-signer", metav1.GetOptions{}) + if errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + if curr.Type == corev1.SecretTypeTLS { + return true, nil + } + return false, nil + }) + require.NoError(t, err) +} diff --git a/test/e2e/certrotation_test.go b/test/e2e/certrotation_test.go index 1c023bc369..6aea7a674c 100644 --- a/test/e2e/certrotation_test.go +++ b/test/e2e/certrotation_test.go @@ -1,146 +1,21 @@ package e2e -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/utils/clock" - - configv1 "github.com/openshift/api/config/v1" - operatorv1 "github.com/openshift/api/operator/v1" - configclient "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" - "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator" - "github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient" - test "github.com/openshift/cluster-kube-apiserver-operator/test/library" - configv1helpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" - "github.com/openshift/library-go/pkg/operator/genericoperatorclient" - "github.com/openshift/library-go/pkg/operator/v1helpers" -) +import "testing" +// This test calls the shared function which +// can be called from both standard Go tests and Ginkgo +// +// This situation is temporary until we test the new e2e-gcp-operator-serial-ote job. +// Eventually all tests will be run only as part of the OTE framework. func TestCertRotationTimeUpgradeable(t *testing.T) { - kubeConfig, err := test.NewClientConfigForTest() - require.NoError(t, err) - operatorClient, _, err := genericoperatorclient.NewStaticPodOperatorClient( - clock.RealClock{}, - kubeConfig, - operatorv1.GroupVersion.WithResource("kubeapiservers"), - operatorv1.GroupVersion.WithKind("KubeAPIServer"), - operator.ExtractStaticPodOperatorSpec, - operator.ExtractStaticPodOperatorStatus) - require.NoError(t, err) - configClient, err := configclient.NewForConfig(kubeConfig) - require.NoError(t, err) - - ctx := context.Background() - _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) - require.NoError(t, err) - require.True(t, v1helpers.IsOperatorConditionTrue(operatorStatus.Conditions, "CertRotationTimeUpgradeable")) - - kubeClient := kubernetes.NewForConfigOrDie(kubeConfig) - t.Logf("Creating unsupported-cert-rotation-config...") - _, err = kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Create(context.TODO(), &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "unsupported-cert-rotation-config"}, - Data: map[string]string{"base": "2y"}, - }, metav1.CreateOptions{}) - require.NoError(t, err) - defer func() { - kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Delete(context.TODO(), "unsupported-cert-rotation-config", metav1.DeleteOptions{}) - }() - - err = wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) { - _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) - if err != nil { - return false, err - } - clusteroperator, err := configClient.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) - if err != nil { - return false, err - } - - certRotationCondition := v1helpers.FindOperatorCondition(operatorStatus.Conditions, "CertRotationTimeUpgradeable") - upgradeableCondition := configv1helpers.FindStatusCondition(clusteroperator.Status.Conditions, "Upgradeable") - if certRotationCondition == nil || upgradeableCondition == nil { - return false, fmt.Errorf("Couldn't find CertRotationTimeUpgradeable or Upgradeable condition") - } - if certRotationCondition.Status == operatorv1.ConditionFalse && - upgradeableCondition.Status == configv1.ConditionFalse && strings.Contains(upgradeableCondition.Reason, "CertRotationTime") { - return true, nil - } - t.Logf("\nCertRotationTimeUpgradeable: %#v\nUpgradeable: %#v", certRotationCondition, upgradeableCondition) - return false, nil - }) - require.NoError(t, err) - - t.Logf("Removing unsupported-cert-rotation-config...") - err = kubeClient.CoreV1().ConfigMaps(operatorclient.GlobalUserSpecifiedConfigNamespace).Delete(context.TODO(), "unsupported-cert-rotation-config", metav1.DeleteOptions{}) - require.NoError(t, err) - - err = wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) { - _, operatorStatus, _, err := operatorClient.GetStaticPodOperatorStateWithQuorum(ctx) - if err != nil { - return false, err - } - clusteroperator, err := configClient.ClusterOperators().Get(context.TODO(), "kube-apiserver", metav1.GetOptions{}) - if err != nil { - return false, err - } - certRotationCondition := v1helpers.FindOperatorCondition(operatorStatus.Conditions, "CertRotationTimeUpgradeable") - upgradeableCondition := configv1helpers.FindStatusCondition(clusteroperator.Status.Conditions, "Upgradeable") - if certRotationCondition == nil || upgradeableCondition == nil { - return false, fmt.Errorf("Couldn't find CertRotationTimeUpgradeable or Upgradeable condition") - } - if certRotationCondition.Status == operatorv1.ConditionTrue && - (upgradeableCondition.Status == configv1.ConditionTrue || !strings.Contains(upgradeableCondition.Reason, "CertRotationTime")) { - return true, nil - } - t.Logf("\nCertRotationTimeUpgradeable: %#v\nUpgradeable: %#v", certRotationCondition, upgradeableCondition) - return false, nil - }) - require.NoError(t, err) + testCertRotationTimeUpgradeable(t) } +// This test calls the shared function which +// can be called from both standard Go tests and Ginkgo +// +// This situation is temporary until we test the new e2e-gcp-operator-serial-ote job. +// Eventually all tests will be run only as part of the OTE framework. func TestCertRotationStompOnBadType(t *testing.T) { - kubeConfig, err := test.NewClientConfigForTest() - require.NoError(t, err) - kubeClient := kubernetes.NewForConfigOrDie(kubeConfig) - - // this is inherently racy against a controller - err = wait.PollImmediate(10*time.Millisecond, 5*time.Second, func() (done bool, err error) { - if err := kubeClient.CoreV1().Secrets(operatorclient.OperatorNamespace).Delete(context.TODO(), "aggregator-client-signer", metav1.DeleteOptions{}); err != nil { - return false, nil - } - if _, err := kubeClient.CoreV1().Secrets(operatorclient.OperatorNamespace).Create(context.TODO(), &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{Namespace: operatorclient.OperatorNamespace, Name: "aggregator-client-signer"}, - Type: "SecretTypeTLS", - }, metav1.CreateOptions{}); err != nil { - return false, nil - } - return true, nil - }) - require.NoError(t, err) - - err = wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (done bool, err error) { - curr, err := kubeClient.CoreV1().Secrets(operatorclient.OperatorNamespace).Get(context.TODO(), "aggregator-client-signer", metav1.GetOptions{}) - if errors.IsNotFound(err) { - return false, nil - } - if err != nil { - return false, err - } - if curr.Type == corev1.SecretTypeTLS { - return true, nil - } - return false, nil - }) - require.NoError(t, err) + testCertRotationStompOnBadType(t) } From 01fae2558e44977e917b01a8dad93fdb11bf8966 Mon Sep 17 00:00:00 2001 From: gangwgr Date: Fri, 12 Dec 2025 17:47:36 +0530 Subject: [PATCH 2/2] Update go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ef0bf3002d..e912873ae8 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/imdario/mergo v0.3.8 github.com/miekg/dns v1.1.61 + github.com/onsi/ginkgo/v2 v2.21.0 github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292 github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee @@ -78,7 +79,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.21.0 // indirect github.com/onsi/gomega v1.35.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect