From 78cce8671b8e559bfc7a2a6ac3569c91a2c66ad2 Mon Sep 17 00:00:00 2001
From: MMMsc <790253229@qq.com>
Date: Tue, 18 Apr 2023 22:27:49 -0400
Subject: [PATCH 1/2] fix verify local policy

Signed-off-by: MMMsc <790253229@qq.com>
---
 test/pkg/e2e/hoh-local-policy_test.go | 15 ++++++++-------
 test/pkg/e2e/hoh-placement_test.go    | 15 ++++++++-------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/test/pkg/e2e/hoh-local-policy_test.go b/test/pkg/e2e/hoh-local-policy_test.go
index 259b208c6b..43e5620a11 100644
--- a/test/pkg/e2e/hoh-local-policy_test.go
+++ b/test/pkg/e2e/hoh-local-policy_test.go
@@ -204,14 +204,15 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 						if err := rows.Scan(&leafhub, policy); err != nil {
 							return err
 						}
-						// if _, ok := policies[leafhub]; ok {
-						// 	return fmt.Errorf("expect leafhub [%s] is already exist", leafhub)
-						// }
-						fmt.Printf("local_spec.policies: %s/%s \n", policy.Namespace, policy.Name)
-						if policy.Name != LOCAL_POLICY_NAME || policy.Namespace != LOCAL_POLICY_NAMESPACE {
-							return fmt.Errorf("expect policy [%s/%s] but got [%s/%s]", LOCAL_POLICY_NAMESPACE, LOCAL_POLICY_NAME, policy.Namespace, policy.Name)
+						for _, leafhubName := range LeafHubNames {
+							if leafhub == leafhubName {
+								fmt.Printf("local_spec.policies: %s/%s \n", policy.Namespace, policy.Name)
+								if policy.Name != LOCAL_POLICY_NAME || policy.Namespace != LOCAL_POLICY_NAMESPACE {
+									return fmt.Errorf("expect policy [%s/%s] but got [%s/%s]", LOCAL_POLICY_NAMESPACE, LOCAL_POLICY_NAME, policy.Namespace, policy.Name)
+								}
+								policies[leafhub] = policy
+							}
 						}
-						policies[leafhub] = policy
 					}
 					if len(policies) != len(LeafHubNames) {
 						return fmt.Errorf("expect policy has not synchronized")
diff --git a/test/pkg/e2e/hoh-placement_test.go b/test/pkg/e2e/hoh-placement_test.go
index db1f951d42..28c298a165 100644
--- a/test/pkg/e2e/hoh-placement_test.go
+++ b/test/pkg/e2e/hoh-placement_test.go
@@ -128,14 +128,15 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 					if err := rows.Scan(&leafhub, policy); err != nil {
 						return err
 					}
-					// if _, ok := policies[leafhub]; ok {
-					// 	return fmt.Errorf("expect leafhub [%s] is already exist", leafhub)
-					// }
-					fmt.Printf("local_spec.policies: %s/%s \n", policy.Namespace, policy.Name)
-					if policy.Name != localPolicyName || policy.Namespace != localPolicyNamespace {
-						return fmt.Errorf("expect policy(placement) [%s/%s] but got [%s/%s]", localPolicyNamespace, localPolicyName, policy.Namespace, policy.Name)
+					for _, leafhubName := range LeafHubNames {
+						if leafhub == leafhubName {
+							fmt.Printf("local_spec.policies: %s/%s \n", policy.Namespace, policy.Name)
+							if policy.Name != LOCAL_POLICY_NAME || policy.Namespace != LOCAL_POLICY_NAMESPACE {
+								return fmt.Errorf("expect policy [%s/%s] but got [%s/%s]", LOCAL_POLICY_NAMESPACE, LOCAL_POLICY_NAME, policy.Namespace, policy.Name)
+							}
+							policies[leafhub] = policy
+						}
 					}
-					policies[leafhub] = policy
 				}
 				if len(policies) != len(LeafHubNames) {
 					return fmt.Errorf("expect policy has not synchronized")

From 2b87f1ba56c9e7d4a1ccc50177dcd1844a005f2d Mon Sep 17 00:00:00 2001
From: MMMsc <790253229@qq.com>
Date: Wed, 7 Jun 2023 01:09:04 -0400
Subject: [PATCH 2/2] add deploy-test beforeSuite && pre-setting parameters in
 options.yaml

Signed-off-by: MMMsc <790253229@qq.com>

add deploy-test beforeSuite && pre-setting parameters in options.yaml

Signed-off-by: MMMsc <790253229@qq.com>

add deploy-test beforeSuite && pre-setting parameters in options.yaml

Signed-off-by: MMMsc <790253229@qq.com>

fix

Signed-off-by: MMMsc <790253229@qq.com>
---
 cicd-scripts/run-local-e2e-test.sh        |  23 +-
 cicd-scripts/run-ocp-e2e-test.sh          |   0
 cicd-scripts/run-prow-e2e-test.sh         |   1 -
 test/pkg/e2e/hoh-application_test.go      |   4 +-
 test/pkg/e2e/hoh-deploy_test.go           | 373 ++++++++++++++++++++++
 test/pkg/e2e/hoh-e2e-test_suite_test.go   |  12 +-
 test/pkg/e2e/hoh-label_test.go            |   5 +-
 test/pkg/e2e/hoh-local-policy_test.go     |   6 +-
 test/pkg/e2e/hoh-placement_test.go        |   7 +-
 test/pkg/e2e/hoh-policy_test.go           |   4 +-
 test/pkg/e2e/hoh-validation_test.go       |  74 ++---
 test/pkg/utils/client.go                  |  51 +--
 test/pkg/utils/options.go                 |   7 +
 test/setup/common.sh                      |   0
 test/setup/e2e_clean.sh                   |   0
 test/setup/e2e_dependencies.sh            |   0
 test/setup/e2e_setup.sh                   |  14 +-
 test/setup/hoh/hoh_setup.sh               |   5 +-
 test/setup/hoh/kafka_setup.sh             |   0
 test/setup/hoh/postgres_setup.sh          |   4 -
 test/setup/leafhub_clean.sh               |   0
 test/setup/leafhub_setup.sh               |   0
 test/setup/microshift/microshift_setup.sh |   0
 23 files changed, 469 insertions(+), 121 deletions(-)
 mode change 100755 => 100644 cicd-scripts/run-local-e2e-test.sh
 mode change 100755 => 100644 cicd-scripts/run-ocp-e2e-test.sh
 mode change 100755 => 100644 cicd-scripts/run-prow-e2e-test.sh
 create mode 100644 test/pkg/e2e/hoh-deploy_test.go
 mode change 100755 => 100644 test/setup/common.sh
 mode change 100755 => 100644 test/setup/e2e_clean.sh
 mode change 100755 => 100644 test/setup/e2e_dependencies.sh
 mode change 100755 => 100644 test/setup/e2e_setup.sh
 mode change 100755 => 100644 test/setup/hoh/hoh_setup.sh
 mode change 100755 => 100644 test/setup/hoh/kafka_setup.sh
 mode change 100755 => 100644 test/setup/hoh/postgres_setup.sh
 mode change 100755 => 100644 test/setup/leafhub_clean.sh
 mode change 100755 => 100644 test/setup/leafhub_setup.sh
 mode change 100755 => 100644 test/setup/microshift/microshift_setup.sh

diff --git a/cicd-scripts/run-local-e2e-test.sh b/cicd-scripts/run-local-e2e-test.sh
old mode 100755
new mode 100644
index 65909dec27..07b5313390
--- a/cicd-scripts/run-local-e2e-test.sh
+++ b/cicd-scripts/run-local-e2e-test.sh
@@ -36,20 +36,27 @@ container_node_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPA
 # container nonk8s api server
 hub_nonk8s_api_server="https://${container_node_ip}:30080"
 
-# container postgres uri
-container_pg_port="32432"
-database_uri=$(kubectl get secret storage-secret -n $hub_namespace --kubeconfig ${hub_kubeconfig} -ojsonpath='{.data.database_uri}' | base64 -d)
-container_pg_uri=$(echo $database_uri | sed "s|@.*hoh|@${container_node_ip}:${container_pg_port}/hoh|g")
-
 printf "options:" > $OPTIONS_FILE
 printf "\n  hub:" >> $OPTIONS_FILE
 printf "\n    name: $HUB_OF_HUB_NAME" >> $OPTIONS_FILE
 printf "\n    namespace: ${hub_namespace}" >> $OPTIONS_FILE
 printf "\n    apiServer: ${hub_api_server}" >> $OPTIONS_FILE
 printf "\n    nonk8sApiServer: ${hub_nonk8s_api_server}" >> $OPTIONS_FILE
-printf "\n    kubeconfig: ${hub_kubeconfig}" >> $OPTIONS_FILE
-printf "\n    kubecontext: ${hub_kubecontext}" >> $OPTIONS_FILE
-printf '\n    databaseURI: %s' ${container_pg_uri} >> $OPTIONS_FILE # contain $ need to use %s
+if [ ! -f "IS_CANARY_ENV" ];then
+  printf "\n    kubeconfig: ${ROOT_DIR}/test/setup/config/kubeconfig" >> $OPTIONS_FILE
+  printf "\n    crdsDir: ${ROOT_DIR}/pkg/testdata/crds" >> $OPTIONS_FILE
+  printf "\n    storagePath: ${ROOT_DIR}/test/setup/hoh/postgres_setup.sh" >> $OPTIONS_FILE
+  printf "\n    transportPath: ${ROOT_DIR}/test/setup/hoh/kafka_setup.sh" >> $OPTIONS_FILE
+else
+  printf "\n    kubeconfig: ${hub_kubeconfig}" >> $OPTIONS_FILE
+  printf "\n    storagePath: ${ROOT_DIR}/operator/config/samples/storage/deploy_postgres.sh" >> $OPTIONS_FILE
+  printf "\n    transportPath: ${ROOT_DIR}/operator/config/samples/transport/deploy_kafka.sh" >> $OPTIONS_FILE
+fi
+  printf "\n    kubecontext: ${hub_kubecontext}" >> $OPTIONS_FILE
+  printf "\n    databaseExternalHost: ${container_node_ip}" >> $OPTIONS_FILE
+  printf "\n    databaseExternalPort: 32432" >> $OPTIONS_FILE
+  printf "\n    mghManagerImageREF: quay.io/stolostron/multicluster-global-hub-manager:latest" >> $OPTIONS_FILE
+  printf "\n    mghAgentImageREF: quay.io/stolostron/multicluster-global-hub-agent:latest" >> $OPTIONS_FILE
 printf "\n  clusters:" >> $OPTIONS_FILE
 
 for i in $(seq 1 "${HUB_CLUSTER_NUM}"); do
diff --git a/cicd-scripts/run-ocp-e2e-test.sh b/cicd-scripts/run-ocp-e2e-test.sh
old mode 100755
new mode 100644
diff --git a/cicd-scripts/run-prow-e2e-test.sh b/cicd-scripts/run-prow-e2e-test.sh
old mode 100755
new mode 100644
index 2c1894f7d4..1d3a3301bb
--- a/cicd-scripts/run-prow-e2e-test.sh
+++ b/cicd-scripts/run-prow-e2e-test.sh
@@ -27,6 +27,5 @@ ssh "${OPT[@]}" "$HOST" sudo yum install gcc git wget jq -y
 ssh "${OPT[@]}" "$HOST" "sudo sh -c 'echo \"fs.inotify.max_user_watches=524288\" >> /etc/sysctl.conf && echo \"fs.inotify.max_user_instances=1024\" >> /etc/sysctl.conf && sysctl -p /etc/sysctl.conf'"
 echo "setup e2e environment"
 ssh "${OPT[@]}" "$HOST" "cd $HOST_DIR && . test/resources/env.list && sudo make e2e-setup-dependencies && make e2e-setup-start" > >(tee "$ARTIFACT_DIR/run-e2e-setup.log") 2>&1
-
 echo "runn e2e tests"
 ssh "${OPT[@]}" "$HOST" "cd $HOST_DIR && . test/resources/env.list && make e2e-tests-all && make e2e-tests-prune" > >(tee "$ARTIFACT_DIR/run-e2e-test.log") 2>&1
diff --git a/test/pkg/e2e/hoh-application_test.go b/test/pkg/e2e/hoh-application_test.go
index 5c23dbbfc6..e8028f2dd5 100644
--- a/test/pkg/e2e/hoh-application_test.go
+++ b/test/pkg/e2e/hoh-application_test.go
@@ -40,7 +40,7 @@ var _ = Describe("Deploy the application to the managed cluster", Label("e2e-tes
 			transport := &http.Transport{
 					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 			}
-			httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+			httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 			var err error
 			managedClusters, err = getManagedCluster(httpClient, httpToken)
 			if err != nil {
@@ -50,7 +50,7 @@ var _ = Describe("Deploy the application to the managed cluster", Label("e2e-tes
 				return fmt.Errorf("managed cluster is not exist")
 			}
 			return nil
-		}, 3*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+		}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 
 		By("Get the appsubreport client")
 		scheme := runtime.NewScheme()
diff --git a/test/pkg/e2e/hoh-deploy_test.go b/test/pkg/e2e/hoh-deploy_test.go
new file mode 100644
index 0000000000..03e3f5e873
--- /dev/null
+++ b/test/pkg/e2e/hoh-deploy_test.go
@@ -0,0 +1,373 @@
+package tests
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+	"strings"
+	"regexp"
+	"strconv"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"	
+)
+
+func deployGlobalHub() error {
+	currentDir, err := os.Getwd()
+	Expect(err).NotTo(HaveOccurred())
+	rootDir := fmt.Sprintf("%s/../../..", currentDir)
+
+	fmt.Println(testOptions.HubCluster.KubeConfig)
+
+	// Create the dynamic client
+	config, err := clientcmd.BuildConfigFromFlags("", testOptions.HubCluster.KubeConfig)
+	if err != nil {
+		Expect(err).ShouldNot(HaveOccurred())
+	}
+	dynClient, err := dynamic.NewForConfig(config)
+	Expect(err).ShouldNot(HaveOccurred())
+
+	clientset, err := kubernetes.NewForConfig(config)
+	Expect(err).ShouldNot(HaveOccurred())
+
+	if os.Getenv("IS_CANARY_ENV") != "true" {
+		By("deploy globalbub for e2e ENV")
+		Eventually(func() error {
+			for i := 1; i <= ExpectedLeafHubNum; i++ {
+				cmd := exec.Command("kubectl", "--context", "kind-hub"+strconv.Itoa(i), "apply", "-f", fmt.Sprintf("%s/0000_01_operator.open-cluster-management.io_multiclusterhubs.crd.yaml", testOptions.HubCluster.CrdsDir))
+				cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", testOptions.HubCluster.KubeConfig))
+				output, err := cmd.CombinedOutput()
+				fmt.Println(string(output))
+				if err != nil {
+					return err
+				}
+			}
+			return nil 
+		}, 3*time.Minute, 5*time.Second).Should(Succeed())
+	}
+
+	By("checking postgresql is ready")
+	cmd := exec.Command("/bin/bash", testOptions.HubCluster.StoragePath)
+	cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", rootDir+"/test/setup/config/kubeconfig-hub-microshift"))
+	output, err := cmd.CombinedOutput()
+	fmt.Println(string(output))
+	Expect(err).ShouldNot(HaveOccurred())
+	
+	Eventually(func() error {
+		// check postgres is ready by checking proxyReadyReplicas && proxyReadyReplicas
+		name := "hoh"
+		namespace := "hoh-postgres"
+
+		postgresCluster, err := dynClient.Resource(
+			schema.GroupVersionResource{
+				Group:    "postgres-operator.crunchydata.com",
+				Version:  "v1beta1",
+				Resource: "postgresclusters",
+			},
+		).Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{})
+		if err != nil {
+			if errors.IsNotFound(err) {
+				return fmt.Errorf("Postgres cluster %s/%s not found\n", namespace, name)
+			}
+			return err
+		}
+
+		// transferring Unstructured into Map
+		postgresClusterMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(postgresCluster)
+		if err != nil {
+			return err
+		}
+		// get and check status.proxy.readyReplicas and status.instances.readyReplicas larget than zero
+		proxyReadyReplicas, proxyOk := postgresClusterMap["status"].(map[string]interface{})["proxy"].(map[string]interface{})["pgBouncer"].(map[string]interface{})["readyReplicas"].(int64)
+		instances := postgresClusterMap["status"].(map[string]interface{})["instances"].([]interface{})
+		instanceReadyReplicas := int64(0)
+		for _, instance := range instances {
+			instanceMap := instance.(map[string]interface{})
+			if instanceMap["readyReplicas"] != nil && instanceMap["readyReplicas"].(int64) > 0 {
+				fmt.Println(instanceReadyReplicas)
+				instanceReadyReplicas++
+			}
+		}
+		if proxyOk && proxyReadyReplicas > 0 && instanceReadyReplicas > 0{
+			return nil
+		}
+		return fmt.Errorf("postgres is not ready")
+	}, 5*time.Minute, 5*time.Second).Should(Succeed())
+
+	Eventually(func() error {
+		// Execute kubectl command to get secret value
+		cmd = exec.Command("bash", "-c", fmt.Sprintf("kubectl get secret storage-secret -n open-cluster-management --kubeconfig %s/test/resources/kubeconfig/kubeconfig-hub-of-hubs -ojsonpath='{.data.database_uri}' | base64 -d", rootDir))
+		output, err = cmd.CombinedOutput()
+		fmt.Println(string(output))
+		if err != nil {
+			return err
+		}
+
+		databaseUri := strings.TrimSpace(string(output))
+		// Replace container node IP and port in database URI
+		containerPgURI := strings.Replace(databaseUri, "@.*hoh", fmt.Sprintf("@%s:%d/hoh", testOptions.HubCluster.DatabaseExternalHost, testOptions.HubCluster.DatabaseExternalPort), -1)
+		
+		pattern := `@.*hoh`
+		replacement := fmt.Sprintf("@%s:%d/hoh", testOptions.HubCluster.DatabaseExternalHost, testOptions.HubCluster.DatabaseExternalPort)
+		re := regexp.MustCompile(pattern)
+		modifiedURI := re.ReplaceAllString(containerPgURI, replacement)
+		fmt.Println(modifiedURI)
+		// add DatabaseURI in options
+		testOptions.HubCluster.DatabaseURI = modifiedURI
+
+		return nil
+	}, 3*time.Minute, 5*time.Second).Should(Succeed())
+
+	By("checking kafka is ready")
+	Eventually(func() error {
+		cmd := exec.Command("/bin/bash", testOptions.HubCluster.TransportPath)
+		cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", rootDir+"/test/setup/config/kubeconfig-hub-microshift"))
+		output, err := cmd.CombinedOutput()
+		fmt.Println(string(output))
+		if err != nil {
+			Expect(err).ShouldNot(HaveOccurred())
+		}
+
+		// check kafka is ready by checking condition status
+		namespace := "kafka"
+		name := "kafka-brokers-cluster"
+
+		kafka, err := dynClient.Resource(
+			schema.GroupVersionResource{
+				Group:    "kafka.strimzi.io",
+				Version:  "v1beta2",
+				Resource: "kafkas",
+			},
+		).Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{})
+		if err != nil {
+			if errors.IsNotFound(err) {
+				return fmt.Errorf("Kafka cluster %s/%s not found\n", namespace, name)
+			} else {
+				return err
+			}
+		}
+
+		conditions, ok := kafka.Object["status"].(map[string]interface{})["conditions"].([]interface{})
+		if !ok && len(conditions) == 0{
+			return fmt.Errorf("Failed to extract conditions from Kafka object")
+		}
+		// check the status conditions for the Kafka cluster.
+		for _, c := range conditions {
+			condition := c.(map[string]interface{})
+			if condition["type"].(string) == "Ready" && condition["status"].(string) == "True" {
+				return nil
+			}
+		}
+		return fmt.Errorf("The condition from kafka object is not ready")
+	}, 5*time.Minute, 5*time.Second).Should(Succeed())
+	
+	// wait deployment is ready
+	// check global hub operator / pod is running
+	By("deploying operator")
+	fmt.Println(testOptions.HubCluster.MghManagerImageREF)
+	if testOptions.HubCluster.MghManagerImageREF != "" {
+		cmd := exec.Command("sed", "-i", fmt.Sprintf("s|quay.io/stolostron/multicluster-global-hub-manager:latest|%s|g", testOptions.HubCluster.MghManagerImageREF), rootDir+"/operator/config/manager/manager.yaml")
+		output, err := cmd.CombinedOutput()
+		fmt.Println(string(output))
+		if err != nil {
+			fmt.Println(err)
+			Expect(err).NotTo(HaveOccurred())
+		}
+	}
+	fmt.Println(testOptions.HubCluster.MghAgentImageREF)
+	if testOptions.HubCluster.MghAgentImageREF != "" {
+		cmd := exec.Command("sed", "-i", fmt.Sprintf("s|quay.io/stolostron/multicluster-global-hub-agent:latest|%s|g", testOptions.HubCluster.MghAgentImageREF), rootDir+"/operator/config/manager/manager.yaml")
+		output, err := cmd.CombinedOutput()
+		fmt.Println(string(output))
+		if err != nil {
+			fmt.Println(err)
+			Expect(err).NotTo(HaveOccurred())
+		}
+	}
+
+	cmd = exec.Command("make", "-C", "../../../operator", "deploy")
+	cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", testOptions.HubCluster.KubeConfig))
+	output, err = cmd.CombinedOutput()
+	fmt.Println(string(output))
+	if err != nil {
+		fmt.Println(err)
+		Expect(fmt.Errorf("failed to execute make deploy-operator: %v\n", err)).NotTo(HaveOccurred())
+	}
+
+	Eventually(func() error {
+		deploymentList, err := clientset.AppsV1().Deployments("open-cluster-management").List(context.Background(), metav1.ListOptions{})
+		if err != nil {
+			fmt.Println(err)
+			return err
+		}
+
+		for _, deployment := range deploymentList.Items {
+			fmt.Println(deployment.Labels["name"])
+			if deployment.Labels["name"] == "multicluster-global-hub-operator" {
+				fmt.Println(deployment.Status.ReadyReplicas)
+				if deployment.Status.ReadyReplicas > 0 {
+					return nil
+				}
+			}
+		}
+		return fmt.Errorf("postgres is not ready")
+	}, 5*time.Minute, 5*time.Second).Should(Succeed())
+	
+	// deploy CR by unstructured.Unstructured
+	By("deploying CR")
+	transportSecretName := "transport-secret"
+	storageSecretName := "storage-secret"
+
+	resource := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "operator.open-cluster-management.io/v1alpha2",
+			"kind":       "MulticlusterGlobalHub",
+			"metadata": map[string]interface{}{
+				"name": "multiclusterglobalhub",
+			},
+			"spec": map[string]interface{}{
+				"dataLayer": map[string]interface{}{
+					"type": "largeScale",
+					"largeScale": map[string]interface{}{
+						"kafka": map[string]interface{}{
+							"name": transportSecretName,
+							"transportFormat": "cloudEvents",
+						},
+						"postgres": map[string]interface{}{
+							"name": storageSecretName,
+						},
+					},
+				},
+			},
+		},
+	}
+	_, err = dynClient.Resource(
+		schema.GroupVersionResource{
+			// apis Gourpversion
+			Group:    "operator.open-cluster-management.io",
+			Version:  "v1alpha2",
+			Resource: "multiclusterglobalhubs",
+		},
+	).Namespace("open-cluster-management").Create(context.TODO(), resource, metav1.CreateOptions{})
+	
+	Eventually(func() error {
+		grafanaPodName := "multicluster-global-hub-grafana"
+		managerPodName := "multicluster-global-hub-manager"
+
+		deploymentList, err := clientset.AppsV1().Deployments("open-cluster-management").List(context.Background(), metav1.ListOptions{
+			LabelSelector: labels.Everything().String(),
+		})
+		if err != nil || deploymentList == nil {
+			return fmt.Errorf("failed to list Pods: %v\n", err)
+		}
+		
+		expectResCount := 2
+		for _, deployment := range deploymentList.Items {
+			if deployment.Labels["name"] == grafanaPodName || deployment.Labels["name"] == managerPodName {
+				if deployment.Status.UnavailableReplicas != 0 {
+					return fmt.Errorf("multicluster global hub is not running")
+				}
+				expectResCount -= 1
+			}
+		}
+		
+		if expectResCount != 0 {
+			return fmt.Errorf("deploy multicluster-global-hub-grafana/manager failed")
+		}
+		return nil
+	}, 3*time.Minute, 5*time.Second).Should(Succeed())
+	
+	// globalhub setup for e2e
+	if os.Getenv("IS_CANARY_ENV") != "true" {
+		By("deploy globalbub for e2e ENV")
+		namespace := "open-cluster-management"	
+
+		By("updating deployment && cluster-manager")
+		cmd := exec.Command("kubectl", "patch", "deployment", "governance-policy-propagator", "-n", "open-cluster-management", "-p", "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"governance-policy-propagator\",\"image\":\"quay.io/open-cluster-management-hub-of-hubs/governance-policy-propagator:v0.5.0\"}]}}}}")		// cmd.Run()
+		cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", testOptions.HubCluster.KubeConfig))
+		err := cmd.Run()
+		if err != nil {
+			Expect(err).Should(Succeed())
+		}
+		
+
+		cmd = exec.Command("kubectl", "patch", "clustermanager", "cluster-manager", "--type", "merge", "-p", "{\"spec\":{\"placementImagePullSpec\":\"quay.io/open-cluster-management/placement:latest\"}}")
+		cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", testOptions.HubCluster.KubeConfig))
+		err = cmd.Run()
+		if err != nil {
+			Expect(err).Should(Succeed())
+		}
+
+		cmd = exec.Command("kubectl", "apply", "-f" ,fmt.Sprintf("%s/test/setup/hoh/components/manager-service-local.yaml", rootDir), "-n", namespace)
+		cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", testOptions.HubCluster.KubeConfig))
+		err = cmd.Run()
+		if err != nil {
+			Expect(err).Should(Succeed())
+		}
+
+		By("waiting for core components to be ready")
+		Eventually(func() error {
+			deployment, err := dynClient.Resource(
+				schema.GroupVersionResource{
+					Group: "apps",
+					Version: "v1",
+					Resource: "deployments",
+				},
+			).Namespace("open-cluster-management-global-hub-system").Get(context.Background(),  "multicluster-global-hub-agent", metav1.GetOptions{})
+			if err != nil {
+				return fmt.Errorf("GlobalHub Agent is not running: %v\n", err)
+			}
+
+			conditions, ok := deployment.Object["status"].(map[string]interface{})["conditions"].([]interface{})
+			if !ok && len(conditions) == 0{
+				return fmt.Errorf("Failed to extract conditions from Kafka object")
+			}
+			for _, c := range conditions {
+				condition := c.(map[string]interface{})
+				if condition["status"].(string) != "True" {
+					return fmt.Errorf("agent is not running")
+				}
+			}
+			return nil
+		})
+
+		Eventually(func() error {
+			cmd = exec.Command("kubectl", "annotate", "mutatingwebhookconfiguration", "multicluster-global-hub-mutator", "service.beta.openshift.io/inject-cabundle-")
+			cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", testOptions.HubCluster.KubeConfig))
+			output, err := cmd.CombinedOutput()
+			if err == nil {
+				fmt.Println(string(output))
+			} else {
+				fmt.Println(string(output))
+				return err
+			}
+
+			cmd = exec.Command("kubectl", "get", "secret", "multicluster-global-hub-webhook-certs", "-n", namespace, "-o", "jsonpath={.data.tls\\.crt}")
+			cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", testOptions.HubCluster.KubeConfig))
+			ca, _ := cmd.Output()
+			fmt.Println(string(ca))
+
+			cmd = exec.Command("kubectl", "patch", "mutatingwebhookconfiguration", "multicluster-global-hub-mutator", "-n", namespace, "-p", fmt.Sprintf("{\"webhooks\":[{\"name\":\"global-hub.open-cluster-management.io\",\"clientConfig\":{\"caBundle\":\"%s\"}}]}", ca))
+			cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", testOptions.HubCluster.KubeConfig))
+			output, err = cmd.CombinedOutput()
+			fmt.Println(string(output))
+			if err != nil {
+				return err 
+			}
+
+			return nil
+		}, 3*time.Minute, 5*time.Second).Should(Succeed())
+	}
+	return nil
+}	
\ No newline at end of file
diff --git a/test/pkg/e2e/hoh-e2e-test_suite_test.go b/test/pkg/e2e/hoh-e2e-test_suite_test.go
index 9b30476fe7..09318ff75d 100644
--- a/test/pkg/e2e/hoh-e2e-test_suite_test.go
+++ b/test/pkg/e2e/hoh-e2e-test_suite_test.go
@@ -47,6 +47,7 @@ func init() {
 
 var _ = BeforeSuite(func() {
 	initVars()
+	deployGlobalHub()
 
 	By("Init the kubernetes client")
 	clients = utils.NewTestClient(testOptionsContainer.Options)
@@ -72,7 +73,7 @@ var _ = BeforeSuite(func() {
 	transport := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 	}
-	httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+	httpClient = &http.Client{Timeout: time.Second * 50, Transport: transport}
 })
 
 var _ = AfterSuite(func() {
@@ -80,7 +81,8 @@ var _ = AfterSuite(func() {
 })
 
 func initVars() {
-	testTimeout = time.Second * 30
+	testTimeout = time.Second * 50
+	rootDir := fmt.Sprintf("%s/../../..", os.Getwd())
 
 	klog.V(6).Infof("Options Path: %s", optionsFile)
 	data, err := os.ReadFile(optionsFile)
@@ -117,4 +119,10 @@ func initVars() {
 			ExpectedManagedClusterNum += 1
 		}
 	}
+
+	if os.Getenv("IS_CANARY_ENV") {
+		testOptions.HubCluster.KubeConfig = 
+		testOptions.HubCluster.StoragePath = rootDir+"/operator/config/samples/storage/deploy_postgres.sh"
+		testOptions.HubCluster.Transport = rootDir+"/operator/config/samples/storage/deploy_kafka.sh"
+	}
 }
diff --git a/test/pkg/e2e/hoh-label_test.go b/test/pkg/e2e/hoh-label_test.go
index a23d166611..76d65451ea 100644
--- a/test/pkg/e2e/hoh-label_test.go
+++ b/test/pkg/e2e/hoh-label_test.go
@@ -30,7 +30,7 @@ var _ = Describe("Updating cluster label from HoH manager", Label("e2e-tests-lab
 			transport := &http.Transport{
 					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 			}
-			httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+			httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 			var err error
 			managedClusters, err = getManagedCluster(httpClient, httpToken)
 			if err != nil {
@@ -40,7 +40,7 @@ var _ = Describe("Updating cluster label from HoH manager", Label("e2e-tests-lab
 				return fmt.Errorf("managed cluster is not exist")
 			}
 			return nil
-		}, 3*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+		}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 	})
 
 	It("add the label to the managed cluster", func() {
@@ -161,7 +161,6 @@ func getManagedCluster(client *http.Client, token string) ([]clusterv1.ManagedCl
 		return nil, err
 	}
 	defer resp.Body.Close()
-
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
diff --git a/test/pkg/e2e/hoh-local-policy_test.go b/test/pkg/e2e/hoh-local-policy_test.go
index fd9b281e56..c47bb57748 100644
--- a/test/pkg/e2e/hoh-local-policy_test.go
+++ b/test/pkg/e2e/hoh-local-policy_test.go
@@ -51,7 +51,7 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 				transport := &http.Transport{
 					TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 				}
-				httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+				httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 				managedClusters, err = getManagedCluster(httpClient, httpToken)
 				if err != nil {
 					return err
@@ -60,7 +60,7 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 					return fmt.Errorf("managed cluster number error")
 				}
 				return nil
-			}, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred())
+			}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 
 			By("Create runtime client")
 			scheme := runtime.NewScheme()
@@ -205,7 +205,7 @@ var _ = Describe("Apply local policy to the managed clusters", Ordered,
 							return err
 						}
 						fmt.Printf("local_spec.policies: %s/%s \n", policy.Namespace, policy.Name)
-							for _, leafhubName := range LeafHubNames {
+						for _, leafhubName := range LeafHubNames {
 							if leafhub == leafhubName && policy.Name == LOCAL_POLICY_NAME && policy.Namespace == LOCAL_POLICY_NAMESPACE {
 								policies[leafhub] = policy
 							}						
diff --git a/test/pkg/e2e/hoh-placement_test.go b/test/pkg/e2e/hoh-placement_test.go
index a767969355..86044baea8 100644
--- a/test/pkg/e2e/hoh-placement_test.go
+++ b/test/pkg/e2e/hoh-placement_test.go
@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"strings"
 	"time"
+	"crypto/tls"
+	"net/http"
 
 	"github.com/jackc/pgx/v4"
 	. "github.com/onsi/ginkgo/v2"
@@ -58,6 +60,9 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 
 		policyClusterset = "clusterset1"
 		Eventually(func() error {
+			transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			}
+			httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 			managedClusters, err = getManagedCluster(httpClient, httpToken)
 			if err != nil {
 				return err
@@ -66,7 +71,7 @@ var _ = Describe("Apply policy/app with placement on the global hub", Ordered, L
 				return fmt.Errorf("managed cluster is not exist")
 			}
 			return nil
-		}, 1*time.Minute, 1*time.Second).ShouldNot(HaveOccurred())
+		}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 
 		By("Init the client")
 		scheme := runtime.NewScheme()
diff --git a/test/pkg/e2e/hoh-policy_test.go b/test/pkg/e2e/hoh-policy_test.go
index 6afdf9a8af..c582c7a421 100644
--- a/test/pkg/e2e/hoh-policy_test.go
+++ b/test/pkg/e2e/hoh-policy_test.go
@@ -49,7 +49,7 @@ var _ = Describe("Apply policy to the managed clusters", Ordered, Label("e2e-tes
 			By("Config request of the api")
 			transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 			}
-			httpClient = &http.Client{Timeout: time.Second * 20, Transport: transport}
+			httpClient = &http.Client{Timeout: time.Second * 60, Transport: transport}
 			managedClusters, err = getManagedCluster(httpClient, httpToken)
 			if err != nil {
 				return err
@@ -58,7 +58,7 @@ var _ = Describe("Apply policy to the managed clusters", Ordered, Label("e2e-tes
 				return fmt.Errorf("managed cluster is not exist")
 			}
 			return nil
-		}, 3*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
+		}, 5*time.Minute, 5*time.Second).ShouldNot(HaveOccurred())
 
 		By("Get the appsubreport client")
 		scheme := runtime.NewScheme()
diff --git a/test/pkg/e2e/hoh-validation_test.go b/test/pkg/e2e/hoh-validation_test.go
index 36c88b1336..dc7f23d8bb 100644
--- a/test/pkg/e2e/hoh-validation_test.go
+++ b/test/pkg/e2e/hoh-validation_test.go
@@ -17,6 +17,43 @@ import (
 )
 
 var _ = Describe("Check all the connection of clients and necessary parameter validation", Label("e2e-tests-validation"), func() {
+	Context("Check all the parameters for e2e-tests", func() {	
+		var ManagedClusterNum = 1
+		It("Check the num of hub clusters and managed clusters on options-local.yaml", func() {
+			opt := testOptions.ManagedClusters
+			var leafhubClusters []string
+			var managedClusters []string
+
+			for _, c := range opt {
+				if c.Name == c.LeafHubName {
+					leafhubClusters = append(leafhubClusters, c.Name)
+				} else {
+					managedClusters = append(managedClusters, c.Name)
+				}
+			}
+			if len(leafhubClusters) != ExpectedLeafHubNum || len(managedClusters) != ManagedClusterNum*ExpectedLeafHubNum {
+				Expect(fmt.Errorf("generate %d hub cluster and %d managed cluster error", ExpectedLeafHubNum, ExpectedManagedClusterNum)).Should(Succeed())
+			}
+		})
+
+		It("Check the num of hub clusters and managed clusters in the kubeconfig", func() {
+			for i := 1; i <= ExpectedLeafHubNum; i++ {
+				hubFileName := fmt.Sprintf("../../resources/kubeconfig/kubeconfig-hub%d", i)
+				_, err := os.Stat(hubFileName)
+				if os.IsNotExist(err) {
+					Expect(fmt.Errorf("kubeconfig-hub%d is not exist", i)).Should(Succeed())
+				}
+				for j := 1; j <= ManagedClusterNum; j++ {
+					managedFileName := fmt.Sprintf("../../resources/kubeconfig/kubeconfig-hub%d-cluster%d", i, j)
+					_, err := os.Stat(managedFileName)
+					if os.IsNotExist(err) {
+						Expect(fmt.Errorf("kubeconfig-hub%d-cluster%d is not exist", i, j)).Should(Succeed())
+					}
+				}
+			}
+		})
+	})
+
 	Context("Check all the clients could connect to the HoH servers", func() {
 		It("connect to the apiserver with kubernetes interface", func() {
 			hubClient := clients.KubeClient()
@@ -64,41 +101,4 @@ var _ = Describe("Check all the connection of clients and necessary parameter va
 			Expect(users).To(ContainElement(result["metadata"].(map[string]interface{})["name"].(string)))
 		})
 	})
-
-	Context("Check all the parameters for e2e-tests", func() {	
-		var ManagedClusterNum = 1
-		It("Check the num of hub clusters and managed clusters on options-local.yaml", func() {
-			opt := testOptions.ManagedClusters
-			var leafhubClusters []string
-			var managedClusters []string
-
-			for _, c := range opt {
-				if c.Name == c.LeafHubName {
-					leafhubClusters = append(leafhubClusters, c.Name)
-				} else {
-					managedClusters = append(managedClusters, c.Name)
-				}
-			}
-			if len(leafhubClusters) != ExpectedLeafHubNum || len(managedClusters) != ManagedClusterNum*ExpectedLeafHubNum {
-				Expect(fmt.Errorf("generate %d hub cluster and %d managed cluster error", ExpectedLeafHubNum, ExpectedManagedClusterNum)).Should(Succeed())
-			}
-		})
-
-		It("Check the num of hub clusters and managed clusters in the kubeconfig", func() {
-			for i := 1; i <= ExpectedLeafHubNum; i++ {
-				hubFileName := fmt.Sprintf("../../resources/kubeconfig/kubeconfig-hub%d", i)
-				_, err := os.Stat(hubFileName)
-				if os.IsNotExist(err) {
-					Expect(fmt.Errorf("kubeconfig-hub%d is not exist", i)).Should(Succeed())
-				}
-				for j := 1; j <= ManagedClusterNum; j++ {
-					managedFileName := fmt.Sprintf("../../resources/kubeconfig/kubeconfig-hub%d-cluster%d", i, j)
-					_, err := os.Stat(managedFileName)
-					if os.IsNotExist(err) {
-						Expect(fmt.Errorf("kubeconfig-hub%d-cluster%d is not exist", i, j)).Should(Succeed())
-					}
-				}
-			}
-		})
-	})
 })
diff --git a/test/pkg/utils/client.go b/test/pkg/utils/client.go
index 823cbb9b12..add44cb7e6 100644
--- a/test/pkg/utils/client.go
+++ b/test/pkg/utils/client.go
@@ -22,11 +22,6 @@ type Client interface {
 	ControllerRuntimeClient(clusterName string, scheme *runtime.Scheme) (runClient.Client, error)
 	Kubectl(clusterName string, args ...string) (string, error)
 	RestConfig(clusterName string) (*rest.Config, error)
-	// HubClusterName() string
-	// LeafHubClusterName() string
-	// GetLeafHubClusterNames() []string
-	// LeafHubClusterNumber() int
-	// ManagedClusterNumber() int
 }
 
 type client struct {
@@ -92,7 +87,6 @@ func (c *client) Kubectl(clusterName string, args ...string) (string, error) {
 		if cluster.Name == clusterName {
 			args = append([]string{"--context", cluster.KubeContext}, args...)
 			args = append([]string{"--kubeconfig", cluster.KubeConfig}, args...)
-			fmt.Println(args)
 			output, err := exec.Command("kubectl", args...).CombinedOutput()
 			return string(output), err
 		}
@@ -144,47 +138,4 @@ func LoadConfig(url, kubeconfig, context string) (*rest.Config, error) {
 	}
 
 	return nil, fmt.Errorf("could not create a valid kubeconfig")
-}
-
-// func (c *client) HubClusterName() string {
-// 	return c.options.HubCluster.Name
-// }
-
-// func (c *client) GetLeafHubClusterNames() []string {
-// 	var clusters []string
-// 	for _, cluster := range c.options.ManagedClusters {
-// 		if cluster.Name == cluster.LeafHubName {
-// 			clusters = append(clusters, cluster.Name)
-// 		}
-// 	}
-// 	return clusters
-// }
-
-// func (c *client) LeafHubClusterName() string {
-// 	for _, cluster := range c.options.ManagedClusters {
-// 		if cluster.Name == cluster.LeafHubName {
-// 			return cluster.Name
-// 		}
-// 	}
-// 	return ""
-// }
-
-// func (c *client) LeafHubClusterNumber() int {
-// 	leafhubNum := 0
-// 	for _, cluster := range c.options.ManagedClusters {
-// 		if cluster.Name == cluster.LeafHubName {
-// 			leafhubNum += 1
-// 		}
-// 	}
-// 	return leafhubNum
-// }
-
-// func (c *client) ManagedClusterNumber() int {
-// 	managedClusterNum := 0
-// 	for _, cluster := range c.options.ManagedClusters {
-// 		if cluster.Name != cluster.LeafHubName {
-// 			managedClusterNum += 1
-// 		}
-// 	}
-// 	return managedClusterNum
-// }
\ No newline at end of file
+}
\ No newline at end of file
diff --git a/test/pkg/utils/options.go b/test/pkg/utils/options.go
index 224183657d..bb5a3816b6 100644
--- a/test/pkg/utils/options.go
+++ b/test/pkg/utils/options.go
@@ -19,6 +19,13 @@ type HOHCluster struct {
 	KubeContext     string `yaml:"kubecontext,omitempty"`
 	Nonk8sApiServer string `yaml:"nonk8sApiServer,omitempty"`
 	DatabaseURI     string `yaml:"databaseURI,omitempty"`
+	StoragePath     string `yaml:"storagePath,omitempty"`
+	TransportPath   string `yaml:"transportPath,omitempty"`
+	CrdsDir         string `yaml:"crdsDir,omitempty"`
+	DatabaseExternalHost string `yaml:"databaseExternalHost,omitempty"`
+	DatabaseExternalPort int    `yaml:"databaseExternalPort,omitempty"`
+	MghManagerImageREF   string `yaml:"mghManagerImageREF,omitempty"`
+	MghAgentImageREF     string `yaml:"mghAgentImageREF,omitempty"`
 }
 
 type ManagedCluster struct {
diff --git a/test/setup/common.sh b/test/setup/common.sh
old mode 100755
new mode 100644
diff --git a/test/setup/e2e_clean.sh b/test/setup/e2e_clean.sh
old mode 100755
new mode 100644
diff --git a/test/setup/e2e_dependencies.sh b/test/setup/e2e_dependencies.sh
old mode 100755
new mode 100644
diff --git a/test/setup/e2e_setup.sh b/test/setup/e2e_setup.sh
old mode 100755
new mode 100644
index 1af6789bd3..0429f61576
--- a/test/setup/e2e_setup.sh
+++ b/test/setup/e2e_setup.sh
@@ -37,8 +37,8 @@ enableOLM $CTX_HUB 2>&1 >> $LOG &
 hover $! "  Enable OLM for $CTX_HUB"
 
 # install some component in microshift in detached mode
-bash ${CURRENT_DIR}/hoh/postgres_setup.sh $HOH_KUBECONFIG 2>&1 >> $LOG &
-bash ${CURRENT_DIR}/hoh/kafka_setup.sh $HOH_KUBECONFIG 2>&1 >> $LOG &
+# bash ${CURRENT_DIR}/hoh/postgres_setup.sh $HOH_KUBECONFIG 2>&1 >> $LOG &
+# bash ${CURRENT_DIR}/hoh/kafka_setup.sh $HOH_KUBECONFIG 2>&1 >> $LOG &
 initHub $CTX_HUB 2>&1 >> $LOG &
 
 # init leafhub 
@@ -69,19 +69,19 @@ for i in $(seq 1 "${HUB_CLUSTER_NUM}"); do
 done
 
 # wait kafka to be ready
-waitAppear "kubectl get pods -n kafka -l name=strimzi-cluster-operator --ignore-not-found | grep Running || true" 1200
-waitAppear "kubectl get kafka kafka-brokers-cluster -n kafka -o jsonpath='{.status.listeners[1].certificates[0]}' --ignore-not-found=true" 1200
+# waitAppear "kubectl get pods -n kafka -l name=strimzi-cluster-operator --ignore-not-found | grep Running || true" 1200
+# waitAppear "kubectl get kafka kafka-brokers-cluster -n kafka -o jsonpath='{.status.listeners[1].certificates[0]}' --ignore-not-found=true" 1200
 
 # wait postgres to be ready
-waitAppear "kubectl get secret hoh-pguser-postgres -n hoh-postgres --ignore-not-found=true"
+# waitAppear "kubectl get secret hoh-pguser-postgres -n hoh-postgres --ignore-not-found=true"
 
 #deploy hoh
 #need the following labels to enable deploying agent in leaf hub cluster
 for i in $(seq 1 "${HUB_CLUSTER_NUM}"); do
     kubectl label managedcluster kind-$LEAF_HUB_NAME$i vendor=OpenShift --overwrite 2>&1 >> $LOG 
 done
-source ${CURRENT_DIR}/hoh/hoh_setup.sh >> $LOG 2>&1 &
-hover $! "6 Deploy hub-of-hubs with $TAG" 
+# source ${CURRENT_DIR}/hoh/hoh_setup.sh >> $LOG 2>&1 &
+# hover $! "6 Deploy hub-of-hubs with $TAG" 
 
 export KUBECONFIG=$KUBECONFIG
 printf "%s\033[0;32m%s\n\033[0m" "[Access the Clusters]: " "export KUBECONFIG=$KUBECONFIG"
diff --git a/test/setup/hoh/hoh_setup.sh b/test/setup/hoh/hoh_setup.sh
old mode 100755
new mode 100644
index d616f57a86..10634c3ea4
--- a/test/setup/hoh/hoh_setup.sh
+++ b/test/setup/hoh/hoh_setup.sh
@@ -2,7 +2,10 @@
 
 set -euo pipefail
 
-export TAG=${TAG:-"latest"}
+branch=$TAG
+if [ $TAG == "latest" ]; then
+  branch="main"
+fi
 export OPENSHIFT_CI=${OPENSHIFT_CI:-"false"}
 export REGISTRY=${REGISTRY:-"quay.io/stolostron"}
 
diff --git a/test/setup/hoh/kafka_setup.sh b/test/setup/hoh/kafka_setup.sh
old mode 100755
new mode 100644
diff --git a/test/setup/hoh/postgres_setup.sh b/test/setup/hoh/postgres_setup.sh
old mode 100755
new mode 100644
index d57f422e9f..a661b0aa7f
--- a/test/setup/hoh/postgres_setup.sh
+++ b/test/setup/hoh/postgres_setup.sh
@@ -10,10 +10,6 @@ bash $postgresDir/deploy_postgres.sh $KUBECONFIG
 # expose the postgres service as NodePort
 pgnamespace="hoh-postgres"
 kubectl patch postgrescluster hoh -n $pgnamespace -p '{"spec":{"service":{"type":"NodePort", "nodePort": 32432}}}'  --type merge
-# unset HA
-kubectl patch configmap pgo-config -n hoh-postgres --type json -p='[{"op": "replace", "path": "/data/DisableAutofail", "value": "true"}]'
-kubectl patch postgrescluster hoh -n hoh-postgres --type json -p='[{"op": "replace", "path": "/spec/instances/0/replicas", "value": 1}]'
-kubectl patch postgrescluster hoh -n hoh-postgres --type merge --patch '{"spec":{"proxy":{"pgBouncer":{"replicas":1}}}}'
 
 stss=$(kubectl get statefulset -n $pgnamespace -o jsonpath={.items..metadata.name})
 for sts in ${stss}; do
diff --git a/test/setup/leafhub_clean.sh b/test/setup/leafhub_clean.sh
old mode 100755
new mode 100644
diff --git a/test/setup/leafhub_setup.sh b/test/setup/leafhub_setup.sh
old mode 100755
new mode 100644
diff --git a/test/setup/microshift/microshift_setup.sh b/test/setup/microshift/microshift_setup.sh
old mode 100755
new mode 100644