add deploy-test beforeSuite && pre-setting parameters in options.yaml

cicd-scripts/run-local-e2e-test.sh

@@ -36,20 +36,27 @@ container_node_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPA
 # container nonk8s api server
 hub_nonk8s_api_server="https://${container_node_ip}:30080"
 
-# container postgres uri
-container_pg_port="32432"
-database_uri=$(kubectl get secret storage-secret -n $hub_namespace --kubeconfig ${hub_kubeconfig} -ojsonpath='{.data.database_uri}' | base64 -d)
-container_pg_uri=$(echo $database_uri | sed "s|@.*hoh|@${container_node_ip}:${container_pg_port}/hoh|g")
-
 printf "options:" > $OPTIONS_FILE
 printf "\n  hub:" >> $OPTIONS_FILE
 printf "\n    name: $HUB_OF_HUB_NAME" >> $OPTIONS_FILE
 printf "\n    namespace: ${hub_namespace}" >> $OPTIONS_FILE
 printf "\n    apiServer: ${hub_api_server}" >> $OPTIONS_FILE
 printf "\n    nonk8sApiServer: ${hub_nonk8s_api_server}" >> $OPTIONS_FILE
-printf "\n    kubeconfig: ${hub_kubeconfig}" >> $OPTIONS_FILE
-printf "\n    kubecontext: ${hub_kubecontext}" >> $OPTIONS_FILE
-printf '\n    databaseURI: %s' ${container_pg_uri} >> $OPTIONS_FILE # contain $ need to use %s
+if [ ! -f "IS_CANARY_ENV" ];then
+  printf "\n    kubeconfig: ${ROOT_DIR}/test/setup/config/kubeconfig" >> $OPTIONS_FILE
+  printf "\n    crdsDir: ${ROOT_DIR}/pkg/testdata/crds" >> $OPTIONS_FILE
+  printf "\n    storagePath: ${ROOT_DIR}/test/setup/hoh/postgres_setup.sh" >> $OPTIONS_FILE
+  printf "\n    transportPath: ${ROOT_DIR}/test/setup/hoh/kafka_setup.sh" >> $OPTIONS_FILE
+else
+  printf "\n    kubeconfig: ${hub_kubeconfig}" >> $OPTIONS_FILE
+  printf "\n    storagePath: ${ROOT_DIR}/operator/config/samples/storage/deploy_postgres.sh" >> $OPTIONS_FILE
+  printf "\n    transportPath: ${ROOT_DIR}/operator/config/samples/transport/deploy_kafka.sh" >> $OPTIONS_FILE
+fi
+  printf "\n    kubecontext: ${hub_kubecontext}" >> $OPTIONS_FILE
+  printf "\n    databaseExternalHost: ${container_node_ip}" >> $OPTIONS_FILE
+  printf "\n    databaseExternalPort: 32432" >> $OPTIONS_FILE
+  printf "\n    mghManagerImageREF: quay.io/stolostron/multicluster-global-hub-manager:latest" >> $OPTIONS_FILE
+  printf "\n    mghAgentImageREF: quay.io/stolostron/multicluster-global-hub-agent:latest" >> $OPTIONS_FILE
 printf "\n  clusters:" >> $OPTIONS_FILE
 
 for i in $(seq 1 "${HUB_CLUSTER_NUM}"); do

cicd-scripts/run-prow-e2e-test.sh

@@ -27,6 +27,5 @@ ssh "${OPT[@]}" "$HOST" sudo yum install gcc git wget jq -y
 ssh "${OPT[@]}" "$HOST" "sudo sh -c 'echo \"fs.inotify.max_user_watches=524288\" >> /etc/sysctl.conf && echo \"fs.inotify.max_user_instances=1024\" >> /etc/sysctl.conf && sysctl -p /etc/sysctl.conf'"
 echo "setup e2e environment"
 ssh "${OPT[@]}" "$HOST" "cd $HOST_DIR && . test/resources/env.list && sudo make e2e-setup-dependencies && make e2e-setup-start" > >(tee "$ARTIFACT_DIR/run-e2e-setup.log") 2>&1
-
 echo "runn e2e tests"
 ssh "${OPT[@]}" "$HOST" "cd $HOST_DIR && . test/resources/env.list && make e2e-tests-all && make e2e-tests-prune" > >(tee "$ARTIFACT_DIR/run-e2e-test.log") 2>&1

test/pkg/e2e/hoh-application_test.go

@@ -40,7 +40,7 @@ var _ = Describe("Deploy the application to the managed cluster", Label("e2e-tes
 			transport := &http.Transport{
 					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 			}
-			httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+			httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 			var err error
 			managedClusters, err = getManagedCluster(httpClient, httpToken)
 			if err != nil {
@@ -50,7 +50,7 @@ var _ = Describe("Deploy the application to the managed cluster", Label("e2e-tes
 				return fmt.Errorf("managed cluster is not exist")
 			}
 			return nil
-		}, 3*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+		}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 
 		By("Get the appsubreport client")
 		scheme := runtime.NewScheme()

test/pkg/e2e/hoh-e2e-test_suite_test.go

@@ -47,6 +47,7 @@ func init() {
 
 var _ = BeforeSuite(func() {
 	initVars()
+	deployGlobalHub()
 
 	By("Init the kubernetes client")
 	clients = utils.NewTestClient(testOptionsContainer.Options)
@@ -72,15 +73,15 @@ var _ = BeforeSuite(func() {
 	transport := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 	}
-	httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+	httpClient = &http.Client{Timeout: time.Second * 50, Transport: transport}
 })
 
 var _ = AfterSuite(func() {
 	utils.DeleteTestingRBAC(testOptionsContainer.Options)
 })
 
 func initVars() {
-	testTimeout = time.Second * 30
+	testTimeout = time.Second * 50
 
 	klog.V(6).Infof("Options Path: %s", optionsFile)
 	data, err := os.ReadFile(optionsFile)

test/pkg/e2e/hoh-label_test.go

@@ -2,13 +2,12 @@ package tests
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
-	"strings"
 	"time"
-	"crypto/tls"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -28,9 +27,9 @@ var _ = Describe("Updating cluster label from HoH manager", Label("e2e-tests-lab
 		Eventually(func() error {
 			By("Config request of the api")
 			transport := &http.Transport{
-					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 			}
-			httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+			httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 			var err error
 			managedClusters, err = getManagedCluster(httpClient, httpToken)
 			if err != nil {
@@ -40,7 +39,7 @@ var _ = Describe("Updating cluster label from HoH manager", Label("e2e-tests-lab
 				return fmt.Errorf("managed cluster is not exist")
 			}
 			return nil
-		}, 3*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+		}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 	})
 
 	It("add the label to the managed cluster", func() {
@@ -72,7 +71,7 @@ var _ = Describe("Updating cluster label from HoH manager", Label("e2e-tests-lab
 	})
 
 	It("add the label to the managed cluster", func() {
-		for i:=1; i<len(managedClusters); i++ {
+		for i := 1; i < len(managedClusters); i++ {
 			patches := []patch{
 				{
 					Op:    "add", // or remove
@@ -139,16 +138,6 @@ type patch struct {
 	Value string `json:"value"`
 }
 
-func getLeafHubName(managedClusterName string) string {
-	result := ""
-	for _, cluster := range testOptions.ManagedClusters {
-		if strings.Compare(cluster.Name, managedClusterName) == 0 {
-			result = cluster.LeafHubName
-		}
-	}
-	return result
-}
-
 func getManagedCluster(client *http.Client, token string) ([]clusterv1.ManagedCluster, error) {
 	managedClusterUrl := fmt.Sprintf("%s/global-hub-api/v1/managedclusters", testOptions.HubCluster.Nonk8sApiServer)
 	req, err := http.NewRequest("GET", managedClusterUrl, nil)
@@ -161,7 +150,6 @@ func getManagedCluster(client *http.Client, token string) ([]clusterv1.ManagedCl
 		return nil, err
 	}
 	defer resp.Body.Close()
-
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err

test/pkg/e2e/hoh-local-policy_test.go

@@ -51,7 +51,7 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 				transport := &http.Transport{
 					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 				}
-				httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+				httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 				managedClusters, err = getManagedCluster(httpClient, httpToken)
 				if err != nil {
 					return err
@@ -60,7 +60,7 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 					return fmt.Errorf("managed cluster number error")
 				}
 				return nil
-			}, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred())
+			}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 
 			By("Create runtime client")
 			scheme := runtime.NewScheme()
@@ -205,10 +205,10 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 							return err
 						}
 						fmt.Printf("local_spec.policies: %s/%s \n", policy.Namespace, policy.Name)
-							for _, leafhubName := range LeafHubNames {
+						for _, leafhubName := range LeafHubNames {
 							if leafhub == leafhubName && policy.Name == LOCAL_POLICY_NAME && policy.Namespace == LOCAL_POLICY_NAMESPACE {
 								policies[leafhub] = policy
-							}						
+							}
 						}
 					}
 					if len(policies) != len(LeafHubNames) {
@@ -220,7 +220,7 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 				By("Verify the local policy is synchronized to the global hub status table")
 				Eventually(func() error {
 					rows, err := postgresConn.Query(context.TODO(),
-						"SELECT id,cluster_name,leaf_hub_name FROM local_status.compliance")
+						"SELECT policy_id,cluster_name,leaf_hub_name FROM local_status.compliance")
 					if err != nil {
 						return err
 					}
@@ -391,7 +391,7 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 				By("Verify the local policy is deleted from the global hub status table")
 				Eventually(func() error {
 					rows, err := postgresConn.Query(context.TODO(),
-						"SELECT id,cluster_name,leaf_hub_name FROM local_status.compliance")
+						"SELECT policy_id,cluster_name,leaf_hub_name FROM local_status.compliance")
 					if err != nil {
 						return err
 					}
@@ -421,4 +421,4 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 			By("Close the postgresql connection")
 			Expect(postgresConn.Close(context.Background())).Should(Succeed())
 		})
-	})
+	})

test/pkg/e2e/hoh-placement_test.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"strings"
 	"time"
+	"crypto/tls"
+	"net/http"
 
 	"github.com/jackc/pgx/v4"
 	. "github.com/onsi/ginkgo/v2"
@@ -32,7 +34,7 @@ const (
 	PLACEMENT_APP_SUB_YAML      = "../../resources/app/app-helloworld-appsub-placement.yaml"
 	PLACEMENT_LOCAL_POLICY_YAML = "../../resources/policy/local-inform-limitrange-policy-placement.yaml"
 
-	PLACEMENT_APP    = "../../resources/policy/enforce-limitrange-policy.yaml"
+	PLACEMENT_APP        = "../../resources/policy/enforce-limitrange-policy.yaml"
 	CLUSTERSET_LABEL_KEY = "cluster.open-cluster-management.io/clusterset"
 )
 
@@ -58,6 +60,9 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 
 		policyClusterset = "clusterset1"
 		Eventually(func() error {
+			transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			}
+			httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 			managedClusters, err = getManagedCluster(httpClient, httpToken)
 			if err != nil {
 				return err
@@ -66,7 +71,7 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 				return fmt.Errorf("managed cluster is not exist")
 			}
 			return nil
-		}, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred())
+		}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 
 		By("Init the client")
 		scheme := runtime.NewScheme()
@@ -81,7 +86,7 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 		globalClient, err = clients.ControllerRuntimeClient(GlobalHubName, scheme)
 		Expect(err).ShouldNot(HaveOccurred())
 
-		for _, leafhubName := range LeafHubNames{
+		for _, leafhubName := range LeafHubNames {
 			leafhubClient, err := clients.ControllerRuntimeClient(leafhubName, scheme)
 			Expect(err).ShouldNot(HaveOccurred())
 			// create local namespace on each leafhub
@@ -95,7 +100,7 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 	})
 
 	Context("When apply local policy with placement on the regional hub", func() {
-		It("deploy local policy on the regional hub", func() {	
+		It("deploy local policy on the regional hub", func() {
 			By("Add local policy test label")
 			patches := []patch{
 				{
@@ -124,7 +129,7 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 				defer rows.Close()
 				for rows.Next() {
 					policy := &policiesv1.Policy{}
-					leafhub := ""	
+					leafhub := ""
 					if err := rows.Scan(&leafhub, policy); err != nil {
 						return err
 					}
@@ -145,12 +150,12 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 			By("Verify the local policy is synchronized to the global hub status table")
 			Eventually(func() error {
 				rows, err := postgresConn.Query(context.TODO(),
-					"SELECT id,cluster_name,leaf_hub_name FROM local_status.compliance")
+					"SELECT policy_id,cluster_name,leaf_hub_name FROM local_status.compliance")
 				if err != nil {
 					return err
 				}
 				defer rows.Close()
-			
+
 				// policies, if leahfubname check remove the kv
 				for rows.Next() {
 					columnValues, _ := rows.Values()
@@ -235,7 +240,7 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 					return err
 				}
 				fmt.Println("Verify the local policy(placement) is deleted from the spec tabl")
-				defer rows.Close()				
+				defer rows.Close()
 				for rows.Next() {
 					policy := &policiesv1.Policy{}
 					if err := rows.Scan(policy); err != nil {
@@ -250,7 +255,7 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 
 			By("Verify the local policy(placement) is deleted from the global hub status table")
 			Eventually(func() error {
-				rows, err := postgresConn.Query(context.TODO(), "SELECT id,cluster_name,leaf_hub_name FROM local_status.compliance")
+				rows, err := postgresConn.Query(context.TODO(), "SELECT policy_id,cluster_name,leaf_hub_name FROM local_status.compliance")
 				if err != nil {
 					fmt.Println(err)
 					return err
@@ -484,4 +489,4 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 		By("Close the postgresql connection")
 		Expect(postgresConn.Close(context.Background())).Should(Succeed())
 	})
-})
+})

test/pkg/e2e/hoh-policy_test.go

@@ -49,7 +49,7 @@ var _ = Describe("Apply policy to the managed clusters", Ordered, Label("e2e-tes
 			By("Config request of the api")
 			transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 			}
-			httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+			httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 			managedClusters, err = getManagedCluster(httpClient, httpToken)
 			if err != nil {
 				return err
@@ -58,7 +58,7 @@ var _ = Describe("Apply policy to the managed clusters", Ordered, Label("e2e-tes
 				return fmt.Errorf("managed cluster is not exist")
 			}
 			return nil
-		}, 3*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+		}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 
 		By("Get the appsubreport client")
 		scheme := runtime.NewScheme()