diff --git a/.tekton/pipelines/common-operator-build.yaml b/.tekton/pipelines/common-operator-build.yaml index 30b1fb7789ff..9db7e0d380be 100644 --- a/.tekton/pipelines/common-operator-build.yaml +++ b/.tekton/pipelines/common-operator-build.yaml @@ -89,7 +89,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:bbf313b09740fb39b3343bc69ee94b2a2c21d16a9304f9b7c111c305558fc346 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:75b88ee5e134a22ee35eb974808dfe6a63693115fa445208a9060a7175b448cf - name: kind value: task resolver: bundles @@ -110,7 +110,7 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:3a920a83fc0135aaae2730fe9d446eb2da2ffc9d63a34bceea04afd24653bdee + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:0a89e1a6304076525e9766f63a4cd006763d21d5aca6863281fc427537a23c6f - name: kind value: task resolver: bundles @@ -141,7 +141,7 @@ spec: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:970285e3b0495961199523b566e0dd92ec2e29bedbcf61d8fc67106b06d0f923 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.2@sha256:3fa0204a481044b21f0e784ce39cbd25e8fb49c664a5458f3eef351fff1c906e - name: kind value: task resolver: bundles @@ -183,7 +183,7 @@ spec: - name: name value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.5@sha256:4f348fa6e0b5d7f976ae6cbb75f4e93bc0be1188f074e39bcae3b5fd71796a3f + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.7@sha256:b24359805297760c87cbce7b4c378267bc83aa1b9a3ac8431829f80bc26ed5d7 - name: kind value: task resolver: bundles @@ -212,7 +212,7 @@ spec: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:79784d53749584bc5a8de32142ec4e2f01cdbf42c20d94e59280e0b927c8597d + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:d455c28a6ae9f6ed40504e65e69b75ab147bb685c9fd606e6ffc3bad6f722bf4 - name: kind value: task resolver: bundles @@ -235,7 +235,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:f59175d9a0a60411738228dfe568af4684af4aa5e7e05c832927cb917801d489 + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:462baed733dfc38aca5395499e92f19b6f13a74c2e88fe5d86c3cffa2f899b57 - name: kind value: task resolver: bundles @@ -257,7 +257,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:a7cc183967f89c4ac100d04ab8f81e54733beee60a0528208107c9a22d3c43af + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:ee558db6af779ab162163ec88f288a5c1b2d5f70c3361f3690a474866e3bdc74 - name: kind value: task resolver: bundles @@ -281,7 +281,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:9568c51a5158d534248908b9b561cf67d2826ed4ea164ffd95628bb42380e6ec + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:04f75593558f79a27da2336400bc63d460bf0c5669e3c13f40ee2fb650b1ad1e - name: kind value: task resolver: bundles @@ -307,7 +307,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:181d63c126e3119a9d57b8feed4eb66a875b5208c3e90724c22758e65dca8733 + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:8ad28b7783837a24acbc9a8494c935e796e591ce476085ad5899bebd7e53f077 - name: kind value: task resolver: bundles @@ -333,7 +333,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:b0bd59748cda4a7abf311e4f448e6c1d00c6b6d8c0ecc1c2eb33e08dc0e0b802 + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:f3d2d179cddcc07d0228d9f52959a233037a3afa2619d0a8b2effbb467db80c3 - name: kind value: task resolver: bundles @@ -394,7 +394,7 @@ spec: - name: name value: coverity-availability-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:db2b267dc15e4ed17f704ee91b8e9b38068e1a35b1018a328fdca621819d74c6 + value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:36400873d3031df128c55aa71ee11d322c3e55fd8f13dc5779098fbc117c0aa3 - name: kind value: task resolver: bundles @@ -420,7 +420,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:bf7bdde00b7212f730c1356672290af6f38d070da2c8a316987b5c32fd49e0b9 + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:d44336d7bcbd1f7cedee639357a493bd1f661e2859e49e11a34644bdf6819c4e - name: kind value: task resolver: bundles @@ -446,7 +446,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.3@sha256:a2bde66f6b4164620298c7d709b8f08515409404000fa1dc2260d2508b135651 + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.3@sha256:e5a8d3e8e7be7246a1460385b95c084ea6e8fe7520d40fe4389deb90f1bf5176 - name: kind value: task resolver: bundles @@ -468,7 +468,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.2@sha256:f44be1bf0262471f2f503f5e19da5f0628dcaf968c86272a2ad6b4871e708448 + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.2@sha256:e4017ec351a0891ef95989f35bd20b8c3f091fa1a3da364c4d4e975e99f3063c - name: kind value: task resolver: bundles @@ -491,7 +491,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:14fba04580b236e4206a904b86ee2fd8eeaa4163f7619a9c2602d361e4f74c51 + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:08bba4a659ecd48f871bef00b80af58954e5a09fcbb28a1783ddd640c4f6535e - name: kind value: task resolver: bundles @@ -508,7 +508,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:1b6c20ab3dbfb0972803d3ebcb2fa72642e59400c77bd66dfd82028bdd09e120 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:20eb21c60522a12198205f70b9c58cb5d71db561a255a3ba1ced56ae7b4af270 - name: kind value: task resolver: bundles diff --git a/test/e2e/cilium_network_policy_test.go b/test/e2e/cilium_network_policy_test.go new file mode 100644 index 000000000000..18766be3bdf9 --- /dev/null +++ b/test/e2e/cilium_network_policy_test.go @@ -0,0 +1,52 @@ +//go:build e2e + +package e2e + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/azureutil" + e2eutil "github.com/openshift/hypershift/test/e2e/util" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// TestCiliumConnectivity validates Cilium connectivity using the official Cilium connectivity test suite. +// This test is specifically for ARO HCP clusters with Cilium as the network provider. +// +// The test performs the following steps: +// 1.Check cilium agent pods ready +// 2. Creates cilium-test namespace with appropriate labels +// 3. Deploys Cilium connectivity test pods +// 4. Waits for all test pods to be ready and running +// 5. Cleans up test resources. + +func TestCiliumConnectivity(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(testContext) + defer cancel() + + clusterOpts := globalOpts.DefaultClusterOptions(t) + + if globalOpts.Platform != hyperv1.AzurePlatform { + t.Skip("Skipping test because it requires Azure platform") + } + + if globalOpts.ExternalCNIProvider != "cilium" { + t.Skipf("skip cilium connection test if e2e.external-cni-provider is not cilium") + } + + e2eutil.NewHypershiftTest(t, ctx, func(t *testing.T, g Gomega, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster) { + if !azureutil.IsAroHCP() { + t.Skip("test only supported on ARO HCP clusters") + } + guestClient := e2eutil.WaitForGuestClient(t, ctx, mgtClient, hostedCluster) + + cleanup := e2eutil.EnsureCiliumConnectivityTestResources(t, ctx, guestClient) + defer cleanup() + }).Execute(&clusterOpts, globalOpts.Platform, globalOpts.ArtifactDir, "cilium-connectivity", globalOpts.ServiceAccountSigningKey) +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 3f0917dc90f9..046c34635c40 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -90,6 +90,7 @@ func TestMain(m *testing.M) { flag.BoolVar(&globalOpts.HOInstallationOptions.EnableCIDebugOutput, "e2e.ho-enable-ci-debug-output", false, "Install the HyperShift Operator with extra CI debug output enabled. This is a HyperShift Operator installation option") flag.StringVar(&globalOpts.HOInstallationOptions.PlatformMonitoring, "e2e.platform-monitoring", "All", "The option for enabling platform cluster monitoring when installing the HyperShift Operator. Valid values are: None, OperatorOnly, All. This is a HyperShift Operator installation option") flag.BoolVar(&globalOpts.RunUpgradeTest, "upgrade.run-tests", false, "Run HyperShift Operator upgrade test") + flag.StringVar(&globalOpts.ExternalCNIProvider, "e2e.external-cni-provider", "", "the option supported third cni provider: cilium, calico") // external OIDC configuration flag.StringVar(&globalOpts.ExternalOIDCProvider, "e2e.external-oidc-provider", "", "if not null, enable external OIDC config with provider. supported value: keycloak, azure") diff --git a/test/e2e/util/cilium.go b/test/e2e/util/cilium.go new file mode 100644 index 000000000000..fc07f2dd56bb --- /dev/null +++ b/test/e2e/util/cilium.go @@ -0,0 +1,608 @@ +package util + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/azureutil" + hyperutil "github.com/openshift/hypershift/support/util" + + configv1 "github.com/openshift/api/config/v1" + securityv1 "github.com/openshift/api/security/v1" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + // CiliumVersion is read from CILIUM_VERSION at runtime. When empty, the Cilium tests should skip. + CiliumVersion = os.Getenv("CILIUM_VERSION") +) + +const ( + // Generic timeouts and intervals for Cilium tests + CiliumDefaultTimeout = 10 * time.Minute + CiliumLongTimeout = 20 * time.Minute + CiliumShortTimeout = 2 * time.Minute + CiliumDefaultPollInterval = 10 * time.Second + CiliumLongPollInterval = 15 * time.Second + CiliumConnectivityWaitDuration = 60 * time.Second +) + +const ( + // CiliumNamespace is the namespace where Cilium agent pods run. + CiliumNamespace = "cilium" + // CiliumTestNamespace is the namespace created for Cilium connectivity tests. + CiliumTestNamespace = "cilium-test" + // CiliumTestServiceAccount is the name of the service account used for Cilium connectivity tests. + CiliumTestServiceAccount = "default" + // CiliumConfigGroup is the group for CiliumConfig. + CiliumConfigGroup = "cilium.io" + // CiliumConfigVersion is the version for CiliumConfig. + CiliumConfigVersion = "v1alpha1" + // CiliumConfigKind is the kind for CiliumConfig. + CiliumConfigKind = "CiliumConfig" + // CiliumConfigName is the name of the CiliumConfig resource. + CiliumConfigName = "cilium" +) + +// CiliumNamespaceManifest returns the cilium namespace with the required PodSecurity labels. +func CiliumNamespaceManifest() *corev1.Namespace { + return &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: CiliumNamespace, + Labels: map[string]string{ + "security.openshift.io/scc.podSecurityLabelSync": "false", + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } +} + +// CiliumSCCManifest returns the SecurityContextConstraints for Cilium. +func CiliumSCCManifest() *securityv1.SecurityContextConstraints { + return &securityv1.SecurityContextConstraints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cilium-scc", + }, + AllowHostPorts: true, + AllowHostNetwork: true, + AllowHostDirVolumePlugin: true, + AllowHostIPC: false, + AllowHostPID: false, + AllowPrivilegeEscalation: ptr.To(true), + AllowPrivilegedContainer: true, + ReadOnlyRootFilesystem: false, + RequiredDropCapabilities: []corev1.Capability{}, + AllowedCapabilities: []corev1.Capability{ + "CHOWN", "KILL", "NET_ADMIN", "NET_RAW", "IPC_LOCK", + "SYS_MODULE", "SYS_ADMIN", "SYS_RESOURCE", "DAC_OVERRIDE", + "FOWNER", "SETGID", "SETUID", "SYS_CHROOT", "SYS_PTRACE", + }, + RunAsUser: securityv1.RunAsUserStrategyOptions{Type: securityv1.RunAsUserStrategyRunAsAny}, + SELinuxContext: securityv1.SELinuxContextStrategyOptions{Type: securityv1.SELinuxStrategyRunAsAny}, + Volumes: []securityv1.FSType{ + securityv1.FSTypeHostPath, + securityv1.FSTypeEmptyDir, + securityv1.FSTypeSecret, + securityv1.FSTypeConfigMap, + securityv1.FSProjected, + }, + Users: []string{ + "system:serviceaccount:cilium:cilium", + "system:serviceaccount:cilium:cilium-operator", + }, + } +} + +// CiliumManifestURLs returns the list of Cilium manifest URLs for a given version. +func CiliumManifestURLs(version string) []string { + return []string{ + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-03-cilium-ciliumconfigs-crd.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00000-cilium-namespace.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00001-cilium-olm-serviceaccount.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00002-cilium-olm-deployment.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00003-cilium-olm-service.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00004-cilium-olm-leader-election-role.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00005-cilium-olm-role.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00006-leader-election-rolebinding.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00007-cilium-olm-rolebinding.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00008-cilium-cilium-olm-clusterrole.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00009-cilium-cilium-clusterrole.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00010-cilium-cilium-olm-clusterrolebinding.yaml", version), + fmt.Sprintf("https://raw.githubusercontent.com/isovalent/olm-for-cilium/main/manifests/cilium.v%s/cluster-network-06-cilium-00011-cilium-cilium-clusterrolebinding.yaml", version), + } +} + +// InstallCilium validates that Cilium network policies are properly enforced +// in ARO HCP guest clusters. This test covers:Verifying Cilium installation +func InstallCilium(t *testing.T, ctx context.Context, guestClient crclient.Client, hostedCluster *hyperv1.HostedCluster) { + t.Run("InstallCilium", func(t *testing.T) { + if !azureutil.IsAroHCP() { + t.Skip("test only supported on ARO HCP clusters") + } + + t.Run("InstallCiliumNameSpace", func(t *testing.T) { + g := NewWithT(t) + t.Log("Configuring cilium namespace with PodSecurity labels") + ciliumNs := &corev1.Namespace{} + err := guestClient.Get(ctx, crclient.ObjectKey{Name: "cilium"}, ciliumNs) + + manifest := CiliumNamespaceManifest() + + if apierrors.IsNotFound(err) { + // Namespace doesn't exist, create it with labels + t.Log("Creating cilium namespace") + ciliumNs = manifest + err = guestClient.Create(ctx, ciliumNs) + g.Expect(err).ToNot(HaveOccurred(), "failed to create cilium namespace") + } else { + // Namespace exists, update labels + g.Expect(err).ToNot(HaveOccurred(), "unexpected error checking cilium namespace") + + t.Log("Updating existing cilium namespace with PodSecurity labels") + if ciliumNs.Labels == nil { + ciliumNs.Labels = make(map[string]string) + } + for k, v := range manifest.Labels { + ciliumNs.Labels[k] = v + } + + err = guestClient.Update(ctx, ciliumNs) + g.Expect(err).ToNot(HaveOccurred(), "failed to update cilium namespace labels") + } + t.Log("Updated cilium namespace PodSecurity to 'privileged'") + }) + + t.Run("CreateSecurityContextConstraints", func(t *testing.T) { + g := NewWithT(t) + + t.Log("Creating SecurityContextConstraints for Cilium") + // Create SCC for Cilium namespace + ciliumSCC := CiliumSCCManifest() + + err := guestClient.Create(ctx, ciliumSCC) + if err != nil && !apierrors.IsAlreadyExists(err) { + g.Expect(err).ToNot(HaveOccurred(), "failed to create Cilium SCC") + } + + t.Log("Cilium SecurityContextConstraints created successfully") + }) + + t.Run("InstallCilium", func(t *testing.T) { + g := NewWithT(t) + t.Logf("Installing Cilium operator version %s", CiliumVersion) + // Install Cilium operator manifests + manifestURLs := CiliumManifestURLs(CiliumVersion) + + for _, url := range manifestURLs { + t.Logf("Applying manifest from %s", url) + err := ApplyYAMLFromURL(ctx, guestClient, url) + g.Expect(err).ToNot(HaveOccurred(), "failed to apply manifest from %s", url) + } + + // Verify critical resources were created successfully + t.Log("Verifying Cilium resources creation") + // Verify Deployment exists + deployment := &appsv1.Deployment{} + err := guestClient.Get(ctx, crclient.ObjectKey{Name: "cilium-olm", Namespace: "cilium"}, deployment) + if err != nil { + t.Logf("Failed to get Deployment cilium-olm: %v", err) + } + // Wait for cilium-olm deployment to be ready + WaitForDeploymentAvailable(ctx, t, guestClient, "cilium-olm", "cilium", CiliumDefaultTimeout, CiliumDefaultPollInterval) + + // Get cluster network configuration from guest cluster + podCIDR, hostPrefix := GetCiliumNetworkConfig(ctx, guestClient) + + // Create CiliumConfig + t.Log("Creating CiliumConfig with pod CIDR and host prefix") + ciliumConfig := CreateCiliumConfig(podCIDR, hostPrefix) + err = guestClient.Create(ctx, ciliumConfig) + if err != nil { + if apierrors.IsAlreadyExists(err) { + // CiliumConfig already exists (likely created by cilium-olm operator), update it + t.Log("CiliumConfig already exists, updating it with correct configuration") + existingConfig := &unstructured.Unstructured{} + existingConfig.SetGroupVersionKind(schema.GroupVersionKind{ + Group: CiliumConfigGroup, + Version: CiliumConfigVersion, + Kind: CiliumConfigKind, + }) + err = guestClient.Get(ctx, crclient.ObjectKey{Name: CiliumConfigName, Namespace: CiliumNamespace}, existingConfig) + g.Expect(err).ToNot(HaveOccurred(), "failed to get existing CiliumConfig") + + // Update the spec with our desired configuration + existingConfig.Object["spec"] = ciliumConfig.Object["spec"] + err = guestClient.Update(ctx, existingConfig) + g.Expect(err).ToNot(HaveOccurred(), "failed to update CiliumConfig") + t.Log("Successfully updated CiliumConfig with pod CIDR and host prefix") + } else { + g.Expect(err).ToNot(HaveOccurred(), "failed to create CiliumConfig") + } + } + + // Verify CiliumConfig has correct configuration + t.Log("Verifying CiliumConfig has correct IPAM configuration") + g.Eventually(func() bool { + ciliumConfig := &unstructured.Unstructured{} + ciliumConfig.SetGroupVersionKind(schema.GroupVersionKind{ + Group: CiliumConfigGroup, + Version: CiliumConfigVersion, + Kind: CiliumConfigKind, + }) + err := guestClient.Get(ctx, crclient.ObjectKey{Name: CiliumConfigName, Namespace: CiliumNamespace}, ciliumConfig) + if err != nil { + t.Logf("CiliumConfig not found yet: %v", err) + return false + } + + // Verify the clusterPoolIPv4MaskSize is set correctly + spec, found, err := unstructured.NestedMap(ciliumConfig.Object, "spec") + if err != nil || !found { + t.Logf("Failed to get spec from CiliumConfig: %v", err) + return false + } + ipam, found, err := unstructured.NestedMap(spec, "ipam") + if err != nil || !found { + t.Logf("Failed to get ipam from CiliumConfig spec: %v", err) + return false + } + operator, found, err := unstructured.NestedMap(ipam, "operator") + if err != nil || !found { + t.Logf("Failed to get operator from CiliumConfig ipam: %v", err) + return false + } + maskSize, found, err := unstructured.NestedInt64(operator, "clusterPoolIPv4MaskSize") + if err != nil || !found { + t.Logf("Failed to get clusterPoolIPv4MaskSize: %v", err) + return false + } + if maskSize != int64(hostPrefix) { + t.Logf("CiliumConfig clusterPoolIPv4MaskSize is %d, expected %d. Updating...", maskSize, hostPrefix) + // Update it again if the operator overwrote it + desiredConfig := CreateCiliumConfig(podCIDR, hostPrefix) + ciliumConfig.Object["spec"] = desiredConfig.Object["spec"] + err = guestClient.Update(ctx, ciliumConfig) + if err != nil { + t.Logf("Failed to update CiliumConfig: %v", err) + } + return false + } + + t.Logf("CiliumConfig has correct clusterPoolIPv4MaskSize: %d", maskSize) + return true + }, CiliumShortTimeout, CiliumDefaultPollInterval).Should(BeTrue(), "CiliumConfig should have correct configuration") + + // Wait for operator to create DaemonSet + t.Log("Waiting for Cilium DaemonSet to be created by operator") + var ciliumDaemonSet *appsv1.DaemonSet + g.Eventually(func() bool { + dsList := &appsv1.DaemonSetList{} + err := guestClient.List(ctx, dsList, crclient.InNamespace(CiliumNamespace)) + if err != nil { + t.Logf("Failed to list DaemonSets: %v", err) + return false + } + + t.Logf("Found %d DaemonSets in cilium namespace", len(dsList.Items)) + for i, ds := range dsList.Items { + t.Logf(" DaemonSet %d: %s", i+1, ds.Name) + // Look for the main Cilium DaemonSet (usually named "cilium") + if ds.Name == "cilium" || strings.HasPrefix(ds.Name, "cilium-") && !strings.Contains(ds.Name, "operator") { + ciliumDaemonSet = &dsList.Items[i] + t.Logf("Found Cilium DaemonSet: %s", ds.Name) + return true + } + } + + t.Log("Cilium DaemonSet not created by operator yet") + return false + }, CiliumDefaultTimeout, CiliumDefaultPollInterval).Should(BeTrue(), "cilium-olm operator should create Cilium DaemonSet") + + // Now wait for DaemonSet pods to be ready + t.Log("Waiting for Cilium agent pods from DaemonSet to be ready") + WaitForDaemonSetReady(ctx, t, guestClient, ciliumDaemonSet.Name, ciliumDaemonSet.Namespace, CiliumLongTimeout, CiliumLongPollInterval) + + t.Log("Cilium installation completed successfully") + }) + }) +} + +// EnsureCiliumConnectivityTestResources performs the Cilium connectivity tests. +// It returns a cleanup function. +func EnsureCiliumConnectivityTestResources(t *testing.T, ctx context.Context, guestClient crclient.Client) func() { + t.Run("CheckCiliumPodsRunning", func(t *testing.T) { + g := NewWithT(t) + + t.Log("Check cilium pods to be running") + g.Eventually(func(g Gomega) { + podList := &corev1.PodList{} + err := guestClient.List(ctx, podList, crclient.InNamespace(CiliumNamespace)) + g.Expect(err).NotTo(HaveOccurred(), "failed to list Cilium pods") + + g.Expect(podList.Items).NotTo(BeEmpty(), "no Cilium pods found yet") + t.Logf("Found %d Cilium pods", len(podList.Items)) + + // Check all pods are Ready + g.Expect(podList.Items).To(HaveEach( + HaveField("Status.Conditions", ContainElement( + And( + HaveField("Type", corev1.PodReady), + HaveField("Status", corev1.ConditionTrue), + ), + )), + ), "not all Cilium pods are ready") + }, CiliumLongTimeout, CiliumDefaultPollInterval).Should(Succeed(), "all Cilium pods should be running") + + t.Log("All Cilium pods are running") + }) + + t.Run("CreateSecurityContextConstraints", func(t *testing.T) { + g := NewWithT(t) + + t.Log("Creating SecurityContextConstraints for Cilium connectivity test") + scc := &securityv1.SecurityContextConstraints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cilium-test", + }, + AllowHostPorts: true, + AllowHostNetwork: true, + AllowHostDirVolumePlugin: false, + AllowHostIPC: false, + AllowHostPID: false, + AllowPrivilegeEscalation: ptrBool(false), + AllowPrivilegedContainer: false, + ReadOnlyRootFilesystem: false, + RequiredDropCapabilities: []corev1.Capability{}, + RunAsUser: securityv1.RunAsUserStrategyOptions{Type: securityv1.RunAsUserStrategyMustRunAsRange}, + SELinuxContext: securityv1.SELinuxContextStrategyOptions{Type: securityv1.SELinuxStrategyMustRunAs}, + Users: []string{fmt.Sprintf("system:serviceaccount:%s:%s", CiliumTestNamespace, CiliumTestServiceAccount)}, + } + + err := guestClient.Create(ctx, scc) + if err != nil && !apierrors.IsAlreadyExists(err) { + g.Expect(err).ToNot(HaveOccurred(), "failed to create SCC") + } + + t.Log("SecurityContextConstraints created successfully") + }) + + t.Run("CreateTestNamespace", func(t *testing.T) { + g := NewWithT(t) + + t.Log("Creating cilium-test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: CiliumTestNamespace, + Labels: map[string]string{"security.openshift.io/scc.podSecurityLabelSync": "false", + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + + err := guestClient.Create(ctx, ns) + if err != nil && !apierrors.IsAlreadyExists(err) { + g.Expect(err).ToNot(HaveOccurred(), "failed to create namespace") + } + + t.Log("Test namespace created successfully") + }) + + t.Run("DeployCiliumConnectivityTest", func(t *testing.T) { + g := NewWithT(t) + + t.Logf("Deploying Cilium connectivity test from version %s", CiliumVersion) + connectivityTestURL := fmt.Sprintf("https://raw.githubusercontent.com/cilium/cilium/%s/examples/kubernetes/connectivity-check/connectivity-check.yaml", CiliumVersion) + + err := ApplyYAMLFromURL(ctx, guestClient, connectivityTestURL, CiliumTestNamespace) + g.Expect(err).ToNot(HaveOccurred(), "failed to apply connectivity test manifest") + + t.Log("Connectivity test manifests applied successfully") + }) + + t.Run("WaitForConnectivityTestPodsReady", func(t *testing.T) { + g := NewWithT(t) + + t.Log("Waiting for connectivity test pods to be ready") + g.Eventually(func(g Gomega) { + podList := &corev1.PodList{} + err := guestClient.List(ctx, podList, crclient.InNamespace(CiliumTestNamespace)) + g.Expect(err).NotTo(HaveOccurred(), "failed to list pods") + g.Expect(podList.Items).NotTo(BeEmpty(), "no pods found in cilium-test namespace yet") + + t.Logf("Found %d pods in cilium-test namespace", len(podList.Items)) + + // Check all pods are Ready + g.Expect(podList.Items).To(HaveEach( + HaveField("Status.Conditions", ContainElement( + And( + HaveField("Type", corev1.PodReady), + HaveField("Status", corev1.ConditionTrue), + ), + )), + ), "some pods are not ready") + }, CiliumDefaultTimeout, CiliumLongPollInterval).Should(Succeed(), "all connectivity test pods should be ready") + + t.Log("All connectivity test pods are ready") + }) + + t.Run("WaitForConnectivityTestCompletion", func(t *testing.T) { + g := NewWithT(t) + + t.Logf("Waiting %v for connectivity tests to run", CiliumConnectivityWaitDuration) + time.Sleep(CiliumConnectivityWaitDuration) + + t.Log("Verifying all test pods are still running") + podList := &corev1.PodList{} + err := guestClient.List(ctx, podList, crclient.InNamespace(CiliumTestNamespace)) + g.Expect(err).ToNot(HaveOccurred(), "should be able to list test pods") + + failedPods := []string{} + for _, pod := range podList.Items { + if pod.Status.Phase != corev1.PodRunning { + failedPods = append(failedPods, fmt.Sprintf("%s (phase: %s)", pod.Name, pod.Status.Phase)) + } + } + + if len(failedPods) > 0 { + t.Errorf("Found %d pods not in Running phase: %v", len(failedPods), failedPods) + } else { + t.Logf("All %d connectivity test pods are running successfully", len(podList.Items)) + } + }) // Closes the anonymous function for "WaitForConnectivityTestCompletion" + + t.Log("Cilium connectivity test completed successfully") + + return func() { + CleanupCiliumConnectivityTestResources(ctx, t, guestClient) + } +} + +// GetCiliumNetworkConfig extracts pod CIDR and host prefix from guest cluster +// In dual-stack environments, this function ensures we return the IPv4 network +func GetCiliumNetworkConfig(ctx context.Context, guestClient crclient.Client) (podCIDR string, hostPrefix int32) { + podCIDR = "10.132.0.0/14" + hostPrefix = 23 + + // Get Network resource from guest cluster + network := &configv1.Network{} + err := guestClient.Get(ctx, types.NamespacedName{Name: "cluster"}, network) + if err != nil { + // Return defaults if we can't get the network config + return podCIDR, hostPrefix + } + + // Extract IPv4 CIDR from ClusterNetwork status + for _, clusterNet := range network.Status.ClusterNetwork { + if strings.Contains(clusterNet.CIDR, ".") { + podCIDR = clusterNet.CIDR + // Only use HostPrefix if it's non-zero, otherwise keep the default + if clusterNet.HostPrefix != 0 { + hostPrefix = int32(clusterNet.HostPrefix) + } + break + } + } + return podCIDR, hostPrefix +} + +func CleanupCiliumConnectivityTestResources(ctx context.Context, t *testing.T, guestClient crclient.Client) { + t.Log("Cleaning up Cilium connectivity test resources") + + // Delete SCC + scc := &securityv1.SecurityContextConstraints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cilium-test", + }, + } + if _, err := hyperutil.DeleteIfNeeded(ctx, guestClient, scc); err != nil { + t.Logf("Warning: failed to delete SCC: %v", err) + } + + // Delete namespace + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: CiliumTestNamespace, + }, + } + if _, err := hyperutil.DeleteIfNeeded(ctx, guestClient, ns); err != nil { + t.Logf("Warning: failed to delete namespace: %v", err) + } + + t.Log("Cleanup completed") +} + +func ptrBool(b bool) *bool { + return &b +} + +// CreateCiliumConfig creates a CiliumConfig custom resource +func CreateCiliumConfig(podCIDR string, hostPrefix int32) *unstructured.Unstructured { + ciliumConfig := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": fmt.Sprintf("%s/%s", CiliumConfigGroup, CiliumConfigVersion), + "kind": CiliumConfigKind, + "metadata": map[string]interface{}{ + "name": CiliumConfigName, + "namespace": CiliumNamespace, + }, + "spec": map[string]interface{}{ + "debug": map[string]interface{}{ + "enabled": true, + }, + "k8s": map[string]interface{}{ + "requireIPv4PodCIDR": true, + }, + "logSystemLoad": true, + "bpf": map[string]interface{}{ + "preallocateMaps": true, + }, + "etcd": map[string]interface{}{ + "leaseTTL": "30s", + }, + "ipv4": map[string]interface{}{ + "enabled": true, + }, + "ipv6": map[string]interface{}{ + "enabled": false, + }, + "identityChangeGracePeriod": "0s", + "ipam": map[string]interface{}{ + "mode": "cluster-pool", + "operator": map[string]interface{}{ + "clusterPoolIPv4PodCIDRList": []string{podCIDR}, + "clusterPoolIPv4MaskSize": hostPrefix, + }, + }, + "nativeRoutingCIDR": podCIDR, + "endpointRoutes": map[string]interface{}{ + "enabled": true, + }, + "clusterHealthPort": 9940, + "tunnelPort": 4789, + "cni": map[string]interface{}{ + "binPath": "/var/lib/cni/bin", + "confPath": "/var/run/multus/cni/net.d", + "chainingMode": "portmap", + }, + "prometheus": map[string]interface{}{ + "serviceMonitor": map[string]interface{}{ + "enabled": false, + }, + }, + "hubble": map[string]interface{}{ + "tls": map[string]interface{}{ + "enabled": false, + }, + }, + "sessionAffinity": true, + "tolerations": []map[string]interface{}{ + { + "operator": "Exists", + }, + }, + }, + }, + } + return ciliumConfig +} diff --git a/test/e2e/util/hypershift_framework.go b/test/e2e/util/hypershift_framework.go index c8ce268580ac..433414768e1c 100644 --- a/test/e2e/util/hypershift_framework.go +++ b/test/e2e/util/hypershift_framework.go @@ -48,7 +48,8 @@ type PlatformAgnosticOptions struct { PowerVSPlatform powervs.RawCreateOptions OpenStackPlatform openstack.RawCreateOptions - ExtOIDCConfig *ExtOIDCConfig + ExtOIDCConfig *ExtOIDCConfig + ExternalCNIProvider string } type hypershiftTestFunc func(t *testing.T, g Gomega, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster) @@ -124,7 +125,7 @@ func (h *hypershiftTest) Execute(opts *PlatformAgnosticOptions, platform hyperv1 // runs before each test. func (h *hypershiftTest) before(hostedCluster *hyperv1.HostedCluster, opts *PlatformAgnosticOptions, platform hyperv1.PlatformType) { h.Run("ValidateHostedCluster", func(t *testing.T) { - if platform != hyperv1.NonePlatform { + if platform != hyperv1.NonePlatform && hostedCluster.Spec.Networking.NetworkType != hyperv1.Other { if opts.AWSPlatform.EndpointAccess == string(hyperv1.Private) { ValidatePrivateCluster(t, h.ctx, h.client, hostedCluster, opts) } else { @@ -137,6 +138,25 @@ func (h *hypershiftTest) before(hostedCluster *hyperv1.HostedCluster, opts *Plat ValidateAuthenticationSpec(t, h.ctx, h.client, hostedCluster, opts.ExtOIDCConfig) } } + + if opts.ExternalCNIProvider == "cilium" { + // Only install Cilium when there are worker nodes configured. + // The cilium-olm deployment requires worker nodes to schedule its pods. + // TestNodePool sets NodePoolReplicas=0 and creates NodePools later in individual tests, + // so we skip Cilium installation during the initial cluster validation phase. + if !util.IsPrivateHC(hostedCluster) { + if opts.NodePoolReplicas == 0 { + t.Fatal("NodePool replicas must be positive for Cilium to install.") + } + guestClient := WaitForGuestClient(t, context.Background(), h.client, hostedCluster) + InstallCilium(t, context.Background(), guestClient, hostedCluster) + // wait hosted cluster ready + WaitForNReadyNodes(t, context.Background(), guestClient, opts.NodePoolReplicas, platform) + WaitForImageRollout(t, context.Background(), h.client, hostedCluster) + ValidateHostedClusterConditions(t, context.Background(), h.client, hostedCluster, true, 10*time.Minute) + } + + } }) } diff --git a/test/e2e/util/options.go b/test/e2e/util/options.go index d434cd01d6b4..6d4d74852bc2 100644 --- a/test/e2e/util/options.go +++ b/test/e2e/util/options.go @@ -91,6 +91,9 @@ type Options struct { ExternalOIDCConsoleSecret string ExternalOIDCCABundleFile string ExternalOIDCTestUsers string + + // ExternalCNIProvider specifies the third-party CNI provider (e.g., "cilium", "calico") + ExternalCNIProvider string } type HyperShiftOperatorInstallOptions struct { @@ -201,12 +204,13 @@ func (o *Options) DefaultClusterOptions(t *testing.T) PlatformAgnosticOptions { }, EtcdStorageClass: o.ConfigurableClusterOptions.EtcdStorageClass, }, - NonePlatform: o.DefaultNoneOptions(), - AWSPlatform: o.DefaultAWSOptions(), - KubevirtPlatform: o.DefaultKubeVirtOptions(), - AzurePlatform: o.DefaultAzureOptions(), - PowerVSPlatform: o.DefaultPowerVSOptions(), - OpenStackPlatform: o.DefaultOpenStackOptions(), + NonePlatform: o.DefaultNoneOptions(), + AWSPlatform: o.DefaultAWSOptions(), + KubevirtPlatform: o.DefaultKubeVirtOptions(), + AzurePlatform: o.DefaultAzureOptions(), + PowerVSPlatform: o.DefaultPowerVSOptions(), + OpenStackPlatform: o.DefaultOpenStackOptions(), + ExternalCNIProvider: o.ExternalCNIProvider, } switch o.Platform { diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index ef6ccadd4643..3292b7f38aa6 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -6,7 +6,9 @@ import ( "crypto/x509" "crypto/x509/pkix" "fmt" + "io" "net" + "net/http" "os" "reflect" "slices" @@ -57,6 +59,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/cli-runtime/pkg/genericclioptions" kubeclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -173,6 +176,36 @@ func UpdateObject[T crclient.Object](t *testing.T, ctx context.Context, client c }) } +// WaitForDeploymentAvailable waits for a deployment to be ready. +func WaitForDeploymentAvailable(ctx context.Context, t *testing.T, client crclient.Client, name, namespace string, timeout, interval time.Duration) { + g := NewWithT(t) + t.Logf("Waiting for deployment %s/%s to be ready", namespace, name) + g.Eventually(func() bool { + deployment := &appsv1.Deployment{} + err := client.Get(ctx, crclient.ObjectKey{Name: name, Namespace: namespace}, deployment) + if err != nil { + t.Logf("Failed to get deployment %s/%s: %v", namespace, name, err) + return false + } + return deployment.Status.ReadyReplicas > 0 + }, timeout, interval).Should(BeTrue(), fmt.Sprintf("deployment %s/%s should be ready", namespace, name)) +} + +// WaitForDaemonSetReady waits for a DaemonSet to be ready. +func WaitForDaemonSetReady(ctx context.Context, t *testing.T, client crclient.Client, name, namespace string, timeout, interval time.Duration) { + g := NewWithT(t) + t.Logf("Waiting for DaemonSet %s/%s to be ready", namespace, name) + g.Eventually(func() bool { + ds := &appsv1.DaemonSet{} + err := client.Get(ctx, crclient.ObjectKey{Name: name, Namespace: namespace}, ds) + if err != nil { + t.Logf("Failed to get DaemonSet %s/%s: %v", namespace, name, err) + return false + } + return ds.Status.NumberReady > 0 + }, timeout, interval).Should(BeTrue(), fmt.Sprintf("DaemonSet %s/%s should be ready", namespace, name)) +} + // DeleteNamespace deletes and finalizes the given namespace, logging any failures // along the way. func DeleteNamespace(t *testing.T, ctx context.Context, client crclient.Client, namespace string) error { @@ -1167,6 +1200,77 @@ func EnsureNetworkPolicies(t *testing.T, ctx context.Context, c crclient.Client, }) } +// ApplyYAMLFromURL downloads YAML content from a URL and applies it to the cluster. +// If defaultNamespace is provided, it will be used for namespaced resources that don't have a namespace set. +func ApplyYAMLFromURL(ctx context.Context, c crclient.Client, url string, defaultNamespace ...string) error { + // Download YAML content + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("failed to download manifest from %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download manifest from %s: HTTP %d", url, resp.StatusCode) + } + + yamlContent, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read manifest content: %w", err) + } + + // Split multi-document YAML using yaml decoder + decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(yamlContent), 4096) + + for { + obj := &unstructured.Unstructured{} + err := decoder.Decode(obj) + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to decode YAML: %w", err) + } + + if len(obj.Object) == 0 { + continue + } + + // Set default namespace if provided and the resource doesn't have one + if len(defaultNamespace) > 0 && obj.GetNamespace() == "" { + isNamespaced := false + gvk := obj.GroupVersionKind() + + // Check if the resource is namespaced using RESTMapper + mapping, err := c.RESTMapper().RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return fmt.Errorf("failed to get mapping for %s: %w", gvk, err) + } + + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + isNamespaced = true + } + + if isNamespaced { + obj.SetNamespace(defaultNamespace[0]) + } + } + + // Apply the resource using Server-Side Apply + patchOpts := []crclient.PatchOption{ + crclient.ForceOwnership, + crclient.FieldOwner("hypershift-e2e"), + } + + err = c.Patch(ctx, obj, crclient.Apply, patchOpts...) + if err != nil { + return fmt.Errorf("failed to apply %s/%s: %w", obj.GetKind(), obj.GetName(), err) + } + } + + return nil +} + // EnsureNodesRuntime ensures that all nodes in the NodePool have the expected runtime handlers. // This is only supported on 4.18+ when the default runtime is changed to crun. func EnsureNodesRuntime(t *testing.T, nodes []corev1.Node) {