CNTRLPLANE-1137: Add e2e test for ARO HCP with Cilium Network Policies

test/e2e/cilium_network_policy_test.go

@@ -0,0 +1,257 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	. "github.com/onsi/gomega"
+	securityv1 "github.com/openshift/api/security/v1"
+	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
+	"github.com/openshift/hypershift/support/azureutil"
+	e2eutil "github.com/openshift/hypershift/test/e2e/util"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	crclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// TestCiliumConnectivity validates Cilium connectivity using the official Cilium connectivity test suite.
+// This test is specifically for ARO HCP clusters with Cilium as the network provider.
+//
+// The test performs the following steps:
+// 1.Check cilium agent pods ready
+// 2. Creates cilium-test namespace with appropriate labels
+// 3. Deploys Cilium connectivity test pods
+// 4. Waits for all test pods to be ready and running
+// 5. Cleans up test resources.
+func TestCiliumConnectivity(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithCancel(testContext)
+	defer cancel()
+
+	clusterOpts := globalOpts.DefaultClusterOptions(t)
+
+	if globalOpts.Platform != hyperv1.AzurePlatform {
+		t.Skip("Skipping test because it requires Azure platform")
+	}
+
+	if globalOpts.ExternalCNIProvider != "cilium" {
+		t.Skipf("skip cilium connection test if e2e.external-cni-provider is not cilium")
+	}
+
+	e2eutil.NewHypershiftTest(t, ctx, func(t *testing.T, g Gomega, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster) {
+		if !azureutil.IsAroHCP() {
+			t.Skip("test only supported on ARO HCP clusters")
+		}
+		guestClient := e2eutil.WaitForGuestClient(t, ctx, mgtClient, hostedCluster)
+
+		testNamespace := "cilium-test"
+
+		// Cleanup function
+		cleanup := func() {
+			t.Log("Cleaning up Cilium connectivity test resources")
+
+			// Delete SCC
+			scc := &securityv1.SecurityContextConstraints{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "cilium-test",
+				},
+			}
+			if err := guestClient.Delete(ctx, scc); err != nil && !apierrors.IsNotFound(err) {
+				t.Logf("Warning: failed to delete SCC: %v", err)
+			}
+
+			// Delete namespace
+			ns := &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: testNamespace,
+				},
+			}
+			if err := guestClient.Delete(ctx, ns); err != nil && !apierrors.IsNotFound(err) {
+				t.Logf("Warning: failed to delete namespace: %v", err)
+			}
+
+			t.Log("Cleanup completed")
+		}
+		defer cleanup()
+
+		t.Run("CheckCiliumPodsRunning", func(t *testing.T) {
+			g := NewWithT(t)
+
+			t.Log("Check cilium pods to be running")
+			g.Eventually(func() bool {
+				podList := &corev1.PodList{}
+				err := guestClient.List(ctx, podList, crclient.InNamespace("cilium"))
+				if err != nil {
+					t.Logf("Failed to list Cilium pods: %v", err)
+					return false
+				}
+				t.Logf("Found %d Cilium pods", len(podList.Items))
+
+				// Check all pods are Ready
+				notReadyPods := []string{}
+				for _, pod := range podList.Items {
+					ready := false
+					for _, condition := range pod.Status.Conditions {
+						if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
+							ready = true
+							break
+						}
+					}
+					if !ready {
+						notReadyPods = append(notReadyPods, fmt.Sprintf("%s (phase: %s)", pod.Name, pod.Status.Phase))
+					}
+				}
+				if len(notReadyPods) > 0 {
+					t.Logf("Pods not ready: %v", notReadyPods)
+					return false
+				}
+				return true
+			}, e2eutil.CiliumPodsReadyTimeout, e2eutil.CiliumPodsReadyPollInterval).Should(BeTrue(), "all Cilium pods should be running")
+
+			t.Log("All Cilium pods are running")
+		})
+
+		t.Run("CreateSecurityContextConstraints", func(t *testing.T) {
+			g := NewWithT(t)
+
+			t.Log("Creating SecurityContextConstraints for Cilium connectivity test")
+			scc := &securityv1.SecurityContextConstraints{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "cilium-test",
+				},
+				AllowHostPorts:           true,
+				AllowHostNetwork:         true,
+				AllowHostDirVolumePlugin: false,
+				AllowHostIPC:             false,
+				AllowHostPID:             false,
+				AllowPrivilegeEscalation: ptrBool(false),
+				AllowPrivilegedContainer: false,
+				ReadOnlyRootFilesystem:   false,
+				RequiredDropCapabilities: []corev1.Capability{},
+				RunAsUser:                securityv1.RunAsUserStrategyOptions{Type: securityv1.RunAsUserStrategyMustRunAsRange},
+				SELinuxContext:           securityv1.SELinuxContextStrategyOptions{Type: securityv1.SELinuxStrategyMustRunAs},
+				Users:                    []string{"system:serviceaccount:cilium-test:default"},
+			}
+
+			err := guestClient.Create(ctx, scc)
+			if err != nil && !apierrors.IsAlreadyExists(err) {
+				g.Expect(err).ToNot(HaveOccurred(), "failed to create SCC")
+			}
+
+			t.Log("SecurityContextConstraints created successfully")
+		})
+
+		t.Run("CreateTestNamespace", func(t *testing.T) {
+			g := NewWithT(t)
+
+			t.Log("Creating cilium-test namespace")
+			ns := &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: testNamespace,
+					Labels: map[string]string{
+						"security.openshift.io/scc.podSecurityLabelSync": "false",
+						"pod-security.kubernetes.io/enforce":             "privileged",
+						"pod-security.kubernetes.io/audit":               "privileged",
+						"pod-security.kubernetes.io/warn":                "privileged",
+					},
+				},
+			}
+
+			err := guestClient.Create(ctx, ns)
+			if err != nil && !apierrors.IsAlreadyExists(err) {
+				g.Expect(err).ToNot(HaveOccurred(), "failed to create namespace")
+			}
+
+			t.Log("Test namespace created successfully")
+		})
+
+		t.Run("DeployCiliumConnectivityTest", func(t *testing.T) {
+			g := NewWithT(t)
+
+			t.Logf("Deploying Cilium connectivity test from version %s", e2eutil.CiliumVersion)
+			connectivityTestURL := fmt.Sprintf("https://raw.githubusercontent.com/cilium/cilium/%s/examples/kubernetes/connectivity-check/connectivity-check.yaml", e2eutil.CiliumVersion)
+
+			err := e2eutil.ApplyYAMLFromURL(ctx, guestClient, connectivityTestURL, testNamespace)
+			g.Expect(err).ToNot(HaveOccurred(), "failed to apply connectivity test manifest")
+
+			t.Log("Connectivity test manifests applied successfully")
+		})
+
+		t.Run("WaitForConnectivityTestPodsReady", func(t *testing.T) {
+			g := NewWithT(t)
+
+			t.Log("Waiting for connectivity test pods to be ready")
+			g.Eventually(func() bool {
+				podList := &corev1.PodList{}
+				err := guestClient.List(ctx, podList, crclient.InNamespace(testNamespace))
+				if err != nil {
+					t.Logf("Failed to list pods: %v", err)
+					return false
+				}
+				if len(podList.Items) == 0 {
+					t.Log("No pods found in cilium-test namespace yet")
+					return false
+				}
+
+				t.Logf("Found %d pods in cilium-test namespace", len(podList.Items))
+
+				// Check all pods are Ready
+				for _, pod := range podList.Items {
+					ready := false
+					for _, condition := range pod.Status.Conditions {
+						if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
+							ready = true
+							break
+						}
+					}
+					if !ready {
+						t.Logf("Pod %s is not ready yet, phase: %s", pod.Name, pod.Status.Phase)
+						return false
+					}
+				}
+				return true
+			}, e2eutil.CiliumConnectivityTestPodsTimeout, e2eutil.CiliumConnectivityTestPodsPollInterval).Should(BeTrue(), "all connectivity test pods should be ready")
+
+			t.Log("All connectivity test pods are ready")
+		})
+
+		t.Run("WaitForConnectivityTestCompletion", func(t *testing.T) {
+			g := NewWithT(t)
+
+			t.Logf("Waiting %v for connectivity tests to run", e2eutil.CiliumConnectivityTestDuration)
+			time.Sleep(e2eutil.CiliumConnectivityTestDuration)
+
+			t.Log("Verifying all test pods are still running")
+			podList := &corev1.PodList{}
+			err := guestClient.List(ctx, podList, crclient.InNamespace(testNamespace))
+			g.Expect(err).ToNot(HaveOccurred(), "should be able to list test pods")
+
+			failedPods := []string{}
+			for _, pod := range podList.Items {
+				if pod.Status.Phase != corev1.PodRunning {
+					failedPods = append(failedPods, fmt.Sprintf("%s (phase: %s)", pod.Name, pod.Status.Phase))
+				}
+			}
+
+			if len(failedPods) > 0 {
+				t.Errorf("Found %d pods not in Running phase: %v", len(failedPods), failedPods)
+			} else {
+				t.Logf("All %d connectivity test pods are running successfully", len(podList.Items))
+			}
+		})
+
+		t.Log("Cilium connectivity test completed successfully")
+	}).Execute(&clusterOpts, globalOpts.Platform, globalOpts.ArtifactDir, "cilium-connectivity", globalOpts.ServiceAccountSigningKey)
+}
+
+func ptrBool(b bool) *bool {
+	return &b
+}

test/e2e/e2e_test.go

@@ -90,6 +90,7 @@ func TestMain(m *testing.M) {
 	flag.BoolVar(&globalOpts.HOInstallationOptions.EnableCIDebugOutput, "e2e.ho-enable-ci-debug-output", false, "Install the HyperShift Operator with extra CI debug output enabled. This is a HyperShift Operator installation option")
 	flag.StringVar(&globalOpts.HOInstallationOptions.PlatformMonitoring, "e2e.platform-monitoring", "All", "The option for enabling platform cluster monitoring when installing the HyperShift Operator. Valid values are: None, OperatorOnly, All. This is a HyperShift Operator installation option")
 	flag.BoolVar(&globalOpts.RunUpgradeTest, "upgrade.run-tests", false, "Run HyperShift Operator upgrade test")
+	flag.StringVar(&globalOpts.ExternalCNIProvider, "e2e.external-cni-provider", "", "the option supported third cni provider: cilium, calico")
 
 	// external OIDC configuration
 	flag.StringVar(&globalOpts.ExternalOIDCProvider, "e2e.external-oidc-provider", "", "if not null, enable external OIDC config with provider. supported value: keycloak, azure")

test/e2e/util/hypershift_framework.go

@@ -48,7 +48,8 @@ type PlatformAgnosticOptions struct {
 	PowerVSPlatform   powervs.RawCreateOptions
 	OpenStackPlatform openstack.RawCreateOptions
 
-	ExtOIDCConfig *ExtOIDCConfig
+	ExtOIDCConfig       *ExtOIDCConfig
+	ExternalCNIProvider string
 }
 
 type hypershiftTestFunc func(t *testing.T, g Gomega, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster)
@@ -124,7 +125,7 @@ func (h *hypershiftTest) Execute(opts *PlatformAgnosticOptions, platform hyperv1
 // runs before each test.
 func (h *hypershiftTest) before(hostedCluster *hyperv1.HostedCluster, opts *PlatformAgnosticOptions, platform hyperv1.PlatformType) {
 	h.Run("ValidateHostedCluster", func(t *testing.T) {
-		if platform != hyperv1.NonePlatform {
+		if platform != hyperv1.NonePlatform && hostedCluster.Spec.Networking.NetworkType != hyperv1.Other {
 			if opts.AWSPlatform.EndpointAccess == string(hyperv1.Private) {
 				ValidatePrivateCluster(t, h.ctx, h.client, hostedCluster, opts)
 			} else {
@@ -137,6 +138,25 @@ func (h *hypershiftTest) before(hostedCluster *hyperv1.HostedCluster, opts *Plat
 				ValidateAuthenticationSpec(t, h.ctx, h.client, hostedCluster, opts.ExtOIDCConfig)
 			}
 		}
+
+		if opts.ExternalCNIProvider == "cilium" {
+			// Only install Cilium when there are worker nodes configured.
+			// The cilium-olm deployment requires worker nodes to schedule its pods.
+			// TestNodePool sets NodePoolReplicas=0 and creates NodePools later in individual tests,
+			// so we skip Cilium installation during the initial cluster validation phase.
+			if !util.IsPrivateHC(hostedCluster) {
+				if opts.NodePoolReplicas == 0 {
+					t.Fatal("NodePool replicas must be positive for Cilium to install.")
+				}
+				guestClient := WaitForGuestClient(t, context.Background(), h.client, hostedCluster)
+				InstallCilium(t, context.Background(), guestClient, hostedCluster)
+				// wait hosted cluster ready
+				WaitForNReadyNodes(t, context.Background(), guestClient, opts.NodePoolReplicas, platform)
+				WaitForImageRollout(t, context.Background(), h.client, hostedCluster)
+				ValidateHostedClusterConditions(t, context.Background(), h.client, hostedCluster, true, 10*time.Minute)
+			}
+
+		}
 	})
 }
 

test/e2e/util/options.go

@@ -91,6 +91,9 @@ type Options struct {
 	ExternalOIDCConsoleSecret   string
 	ExternalOIDCCABundleFile    string
 	ExternalOIDCTestUsers       string
+
+	// ExternalCNIProvider specifies the third-party CNI provider (e.g., "cilium", "calico")
+	ExternalCNIProvider string
 }
 
 type HyperShiftOperatorInstallOptions struct {
@@ -201,12 +204,13 @@ func (o *Options) DefaultClusterOptions(t *testing.T) PlatformAgnosticOptions {
 			},
 			EtcdStorageClass: o.ConfigurableClusterOptions.EtcdStorageClass,
 		},
-		NonePlatform:      o.DefaultNoneOptions(),
-		AWSPlatform:       o.DefaultAWSOptions(),
-		KubevirtPlatform:  o.DefaultKubeVirtOptions(),
-		AzurePlatform:     o.DefaultAzureOptions(),
-		PowerVSPlatform:   o.DefaultPowerVSOptions(),
-		OpenStackPlatform: o.DefaultOpenStackOptions(),
+		NonePlatform:        o.DefaultNoneOptions(),
+		AWSPlatform:         o.DefaultAWSOptions(),
+		KubevirtPlatform:    o.DefaultKubeVirtOptions(),
+		AzurePlatform:       o.DefaultAzureOptions(),
+		PowerVSPlatform:     o.DefaultPowerVSOptions(),
+		OpenStackPlatform:   o.DefaultOpenStackOptions(),
+		ExternalCNIProvider: o.ExternalCNIProvider,
 	}
 
 	switch o.Platform {