From 66455e11b335a7dd27e6cdbb71340ffc13388b96 Mon Sep 17 00:00:00 2001 From: nhamza Date: Mon, 5 Jan 2026 13:22:51 +0200 Subject: [PATCH 1/3] update test logic for degraded tests Signed-off-by: nhamza --- test/extended/operators/certs.go | 43 +++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/test/extended/operators/certs.go b/test/extended/operators/certs.go index 4629657f3d65..ce462911bb28 100644 --- a/test/extended/operators/certs.go +++ b/test/extended/operators/certs.go @@ -47,6 +47,7 @@ import ( "k8s.io/apimachinery/pkg/watch" watchtools "k8s.io/client-go/tools/watch" e2e "k8s.io/kubernetes/test/e2e/framework" + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" ) const certInspectResultFile = "/tmp/shared/pkiList.json" @@ -128,7 +129,16 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g // Skip metal jobs if test image pullspec cannot be determined if jobType.Platform != "metal" || err == nil { o.Expect(err).NotTo(o.HaveOccurred()) - onDiskPKIContent, err = fetchOnDiskCertificates(ctx, kubeClient, oc.AdminConfig(), masters, openshiftTestImagePullSpec) + readyMasters, skipped := filterReadyNodes(masters) + if len(skipped) > 0 { + e2e.Logf("Skipping on-disk cert collection for NotReady control-plane nodes: %v", skipped) + } + if len(readyMasters) == 0 { + e2eskipper.Skipf("No Ready control-plane nodes for on-disk cert collection") + } + + onDiskPKIContent, err = fetchOnDiskCertificates(ctx, kubeClient, oc.AdminConfig(), readyMasters, openshiftTestImagePullSpec) + o.Expect(err).NotTo(o.HaveOccurred()) } @@ -160,14 +170,13 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g o.Expect(err).NotTo(o.HaveOccurred()) pkiDir := filepath.Join(exutil.ArtifactDirPath(), "rawTLSInfo") - err = os.MkdirAll(pkiDir, 0755) + err = os.MkdirAll(pkiDir, 0o755) o.Expect(err).NotTo(o.HaveOccurred()) - err = os.WriteFile(filepath.Join(pkiDir, tlsArtifactFilename), jsonBytes, 0644) + err = os.WriteFile(filepath.Join(pkiDir, tlsArtifactFilename), jsonBytes, 0o644) o.Expect(err).NotTo(o.HaveOccurred()) }) g.It("all tls artifacts must be registered", func() { - violationsPKIContent, err := certs.GetPKIInfoFromEmbeddedOwnership(ownership.PKIViolations) o.Expect(err).NotTo(o.HaveOccurred()) @@ -181,7 +190,6 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g _, err := certgraphutils.LocateCertKeyPairBySecretLocation(currLocation, expectedPKIContent.CertKeyPairs) if err != nil { - newTLSRegistry.CertKeyPairs = append(newTLSRegistry.CertKeyPairs, certgraphapi.PKIRegistryCertKeyPair{InClusterLocation: &actualPKIContent.InClusterResourceData.CertKeyPairs[i]}) } @@ -269,11 +277,11 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g if len(newTLSRegistry.CertKeyPairs) > 0 || len(newTLSRegistry.CertificateAuthorityBundles) > 0 { registryString, err := json.MarshalIndent(newTLSRegistry, "", " ") if err != nil { - //g.Fail("Failed to marshal registry %#v: %v", newTLSRegistry, err) + // g.Fail("Failed to marshal registry %#v: %v", newTLSRegistry, err) testresult.Flakef("Failed to marshal registry %#v: %v", newTLSRegistry, err) } // TODO: uncomment when test no longer fails and enhancement is merged - //g.Fail(fmt.Sprintf("Unregistered TLS certificates:\n%s", registryString)) + // g.Fail(fmt.Sprintf("Unregistered TLS certificates:\n%s", registryString)) testresult.Flakef("Unregistered TLS certificates found:\n%s\nSee tls/ownership/README.md in origin repo", registryString) } }) @@ -285,7 +293,7 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g if len(messages) > 0 { // TODO: uncomment when test no longer fails and enhancement is merged - //g.Fail(strings.Join(messages, "\n")) + // g.Fail(strings.Join(messages, "\n")) testresult.Flakef("%s", strings.Join(messages, "\n")) } }) @@ -323,7 +331,6 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g testresult.Flakef("Errors found: %s", utilerrors.NewAggregate(errs).Error()) } }) - }) func fetchOnDiskCertificates(ctx context.Context, kubeClient kubernetes.Interface, podRESTConfig *rest.Config, nodeList []*corev1.Node, testPullSpec string) (*certgraphapi.PKIList, error) { @@ -480,3 +487,21 @@ func isCertKeyPairFromIgnoredNamespace(cert certgraphapi.CertKeyPair, ignoredNam } return false } + +func filterReadyNodes(nodes []*corev1.Node) (ready []*corev1.Node, notReady []string) { + for _, n := range nodes { + isReady := false + for _, c := range n.Status.Conditions { + if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue { + isReady = true + break + } + } + if isReady { + ready = append(ready, n) + } else { + notReady = append(notReady, n.Name) + } + } + return ready, notReady +} From 3717fab34863e86810226e192fd5f69554acbb6d Mon Sep 17 00:00:00 2001 From: nhamza Date: Thu, 8 Jan 2026 16:06:03 +0200 Subject: [PATCH 2/3] update test to keep logic as-is Signed-off-by: nhamza --- test/extended/operators/certs.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/extended/operators/certs.go b/test/extended/operators/certs.go index ce462911bb28..8417e781d7cd 100644 --- a/test/extended/operators/certs.go +++ b/test/extended/operators/certs.go @@ -32,6 +32,7 @@ import ( "github.com/openshift/library-go/pkg/certs/cert-inspection/certgraphutils" "github.com/openshift/library-go/pkg/operator/resource/resourceread" + configv1 "github.com/openshift/api/config/v1" "github.com/openshift/origin/pkg/certs" "github.com/openshift/origin/pkg/monitortestlibrary/platformidentification" testresult "github.com/openshift/origin/pkg/test/ginkgo/result" @@ -47,7 +48,6 @@ import ( "k8s.io/apimachinery/pkg/watch" watchtools "k8s.io/client-go/tools/watch" e2e "k8s.io/kubernetes/test/e2e/framework" - e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" ) const certInspectResultFile = "/tmp/shared/pkiList.json" @@ -129,17 +129,17 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g // Skip metal jobs if test image pullspec cannot be determined if jobType.Platform != "metal" || err == nil { o.Expect(err).NotTo(o.HaveOccurred()) - readyMasters, skipped := filterReadyNodes(masters) - if len(skipped) > 0 { - e2e.Logf("Skipping on-disk cert collection for NotReady control-plane nodes: %v", skipped) + topo, topoErr := exutil.GetControlPlaneTopology(oc) + o.Expect(topoErr).NotTo(o.HaveOccurred()) + + if *topo == configv1.DualReplicaTopologyMode { + readyMasters, _ := filterReadyNodes(masters) + onDiskPKIContent, err = fetchOnDiskCertificates(ctx, kubeClient, oc.AdminConfig(), readyMasters, openshiftTestImagePullSpec) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + onDiskPKIContent, err = fetchOnDiskCertificates(ctx, kubeClient, oc.AdminConfig(), masters, openshiftTestImagePullSpec) + o.Expect(err).NotTo(o.HaveOccurred()) } - if len(readyMasters) == 0 { - e2eskipper.Skipf("No Ready control-plane nodes for on-disk cert collection") - } - - onDiskPKIContent, err = fetchOnDiskCertificates(ctx, kubeClient, oc.AdminConfig(), readyMasters, openshiftTestImagePullSpec) - - o.Expect(err).NotTo(o.HaveOccurred()) } actualPKIContent = certgraphanalysis.MergePKILists(ctx, inClusterPKIContent, onDiskPKIContent) From 657fd3ad10a7f40e2d42c396817767e29b231908 Mon Sep 17 00:00:00 2001 From: nhamza Date: Thu, 8 Jan 2026 18:18:24 +0200 Subject: [PATCH 3/3] add env var gate for degraded mode Signed-off-by: nhamza --- cmd/openshift-tests/openshift-tests.go | 9 ++++++++- test/extended/operators/certs.go | 7 +++---- test/extended/util/framework.go | 13 +++++++------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/cmd/openshift-tests/openshift-tests.go b/cmd/openshift-tests/openshift-tests.go index 761d7bfe93aa..0846627e3a11 100644 --- a/cmd/openshift-tests/openshift-tests.go +++ b/cmd/openshift-tests/openshift-tests.go @@ -63,8 +63,15 @@ func main() { os.Unsetenv("ENABLE_STORAGE_GCE_PD_DRIVER") } + // Detect intentionally degraded clusters (e.g. TNF degraded) based on + // CI-provided environment signals and propagate that context to extended tests + if os.Getenv("DEGRADED_NODE") == "true" { + exutil.ClusterDegraded = true + logrus.Infof("openshift-tests targeting intentionally degraded cluster") + } + pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) - //pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + // pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) extensionRegistry, originExtension, err := extensions.InitializeOpenShiftTestsExtensionFramework() if err != nil { diff --git a/test/extended/operators/certs.go b/test/extended/operators/certs.go index 8417e781d7cd..f55053a3af7c 100644 --- a/test/extended/operators/certs.go +++ b/test/extended/operators/certs.go @@ -32,7 +32,6 @@ import ( "github.com/openshift/library-go/pkg/certs/cert-inspection/certgraphutils" "github.com/openshift/library-go/pkg/operator/resource/resourceread" - configv1 "github.com/openshift/api/config/v1" "github.com/openshift/origin/pkg/certs" "github.com/openshift/origin/pkg/monitortestlibrary/platformidentification" testresult "github.com/openshift/origin/pkg/test/ginkgo/result" @@ -129,10 +128,10 @@ var _ = g.Describe(fmt.Sprintf("[sig-arch][Late][Jira:%q]", "kube-apiserver"), g // Skip metal jobs if test image pullspec cannot be determined if jobType.Platform != "metal" || err == nil { o.Expect(err).NotTo(o.HaveOccurred()) - topo, topoErr := exutil.GetControlPlaneTopology(oc) - o.Expect(topoErr).NotTo(o.HaveOccurred()) - if *topo == configv1.DualReplicaTopologyMode { + // Only relax on-disk cert collection when the cluster is intentionally degraded + // and the topology is Two-Node Fencing (DualReplica). + if exutil.ClusterDegraded && exutil.IsTwoNodeFencing(ctx, configClient) { readyMasters, _ := filterReadyNodes(masters) onDiskPKIContent, err = fetchOnDiskCertificates(ctx, kubeClient, oc.AdminConfig(), readyMasters, openshiftTestImagePullSpec) o.Expect(err).NotTo(o.HaveOccurred()) diff --git a/test/extended/util/framework.go b/test/extended/util/framework.go index 82307ba5d8d4..d2815ae01bea 100644 --- a/test/extended/util/framework.go +++ b/test/extended/util/framework.go @@ -327,7 +327,6 @@ func WaitForOpenShiftNamespaceImageStreams(oc *CLI) error { // Check to see if SamplesOperator managementState is Removed out, err := oc.AsAdmin().Run("get").Args("configs.samples.operator.openshift.io", "cluster", "-o", "yaml").Output() - if err != nil { e2e.Logf("\n error on getting samples operator CR: %+v\n%#v\n", err, out) } @@ -781,7 +780,7 @@ func VarSubOnFile(srcFile string, destFile string, vars map[string]string) error k = "${" + k + "}" srcString = strings.Replace(srcString, k, v, -1) // -1 means unlimited replacements } - err = ioutil.WriteFile(destFile, []byte(srcString), 0644) + err = ioutil.WriteFile(destFile, []byte(srcString), 0o644) } return err } @@ -1654,11 +1653,11 @@ func restoreFixtureAsset(dir, name string) error { if err != nil { return err } - err = os.MkdirAll(assetFilePath(dir, filepath.Dir(name)), os.FileMode(0755)) + err = os.MkdirAll(assetFilePath(dir, filepath.Dir(name)), os.FileMode(0o755)) if err != nil { return err } - err = ioutil.WriteFile(assetFilePath(dir, name), data, 0640) + err = ioutil.WriteFile(assetFilePath(dir, name), data, 0o640) if err != nil { return err } @@ -1990,10 +1989,10 @@ type GitRepo struct { // AddAndCommit commits a file with its content to local repo func (r GitRepo) AddAndCommit(file, content string) error { dir := filepath.Dir(file) - if err := os.MkdirAll(filepath.Join(r.RepoPath, dir), 0777); err != nil { + if err := os.MkdirAll(filepath.Join(r.RepoPath, dir), 0o777); err != nil { return err } - if err := ioutil.WriteFile(filepath.Join(r.RepoPath, file), []byte(content), 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(r.RepoPath, file), []byte(content), 0o666); err != nil { return err } if err := r.repo.Add(r.RepoPath, file); err != nil { @@ -2347,6 +2346,8 @@ func IsTwoNodeFencing(ctx context.Context, configClient clientconfigv1.Interface return infrastructure.Status.ControlPlaneTopology == configv1.DualReplicaTopologyMode } +var ClusterDegraded bool + func groupName(groupVersionName string) string { return strings.Split(groupVersionName, "/")[0] }