fix post-completion/failure longhorn prometheus scale code (#4887)

* fix post-failure longhorn prometheus scale code

* the longhorn reset scale process should never block continuation of the script if the migration succeeded

* include prometheus in longhorn to openebs migration test

Author: laverya

pkg/cli/longhorn.go

@@ -2,7 +2,6 @@ package cli
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"log"
 	"strconv"
@@ -20,16 +19,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 
 	lhv1b1 "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta1"
-	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	"github.com/spf13/cobra"
 )
 
 var scaleDownReplicasWaitTime = 5 * time.Minute
 
 const (
 	prometheusNamespace          = "monitoring"
-	prometheusName               = "k8s"
-	prometheusStatefulSetName    = "prometheus-k8s"
 	ekcoNamespace                = "kurl"
 	ekcoDeploymentName           = "ekc-operator"
 	pvmigrateScaleDownAnnotation = "kurl.sh/pvcmigrate-scale"
@@ -64,7 +60,6 @@ func NewLonghornRollbackMigrationReplicas(cli CLI) *cobra.Command {
 				return fmt.Errorf("error creating client: %s", err)
 			}
 			lhv1b1.AddToScheme(cli.Scheme())
-			promv1.AddToScheme(cli.Scheme())
 
 			var l1b1Volumes lhv1b1.VolumeList
 			if err := cli.List(cmd.Context(), &l1b1Volumes, client.InNamespace(longhornNamespace)); err != nil {
@@ -136,7 +131,6 @@ func NewLonghornPrepareForMigration(cli CLI) *cobra.Command {
 				return fmt.Errorf("error creating client: %s", err)
 			}
 			lhv1b1.AddToScheme(cli.Scheme())
-			promv1.AddToScheme(cli.Scheme())
 
 			var scaledDown bool
 			var nodes corev1.NodeList
@@ -256,46 +250,22 @@ func isPrometheusInstalled(ctx context.Context, cli client.Client) (bool, error)
 	return true, nil
 }
 
-// scaleUpPrometheus scales up prometheus.
+// scaleUpPrometheus scales up the prometheus operator.
 func scaleUpPrometheus(ctx context.Context, cli client.Client) error {
 	if installed, err := isPrometheusInstalled(ctx, cli); err != nil {
-		return fmt.Errorf("error scaling down prometheus: %w", err)
+		return fmt.Errorf("error scaling up prometheus: %w", err)
 	} else if !installed {
 		return nil
 	}
 
-	nsn := types.NamespacedName{Namespace: prometheusNamespace, Name: prometheusName}
-	var prometheus promv1.Prometheus
-	if err := cli.Get(ctx, nsn, &prometheus); err != nil {
-		if errors.IsNotFound(err) {
-			return nil
-		}
-		return fmt.Errorf("error getting prometheus: %w", err)
-	}
-	replicasStr, ok := prometheus.Annotations[pvmigrateScaleDownAnnotation]
-	if !ok {
-		return fmt.Errorf("error reading original replicas from the prometheus annotation: not found")
-	}
-	origReplicas, err := strconv.Atoi(replicasStr)
-	if err != nil {
-		return fmt.Errorf("error converting replicas annotation to integer: %w", err)
-	}
-	patch := map[string]interface{}{
-		"metadata": map[string]interface{}{
-			"annotations": map[string]interface{}{
-				pvmigrateScaleDownAnnotation: nil,
-			},
-		},
-		"spec": map[string]interface{}{
-			"replicas": origReplicas,
-		},
-	}
-	rawPatch, err := json.Marshal(patch)
-	if err != nil {
-		return fmt.Errorf("error creating prometheus patch: %w", err)
+	var dep appsv1.Deployment
+	if err := cli.Get(ctx, types.NamespacedName{Namespace: prometheusNamespace, Name: "prometheus-operator"}, &dep); err != nil {
+		return fmt.Errorf("error getting prometheus operator deployment: %w", err)
 	}
-	if err := cli.Patch(ctx, &prometheus, client.RawPatch(types.MergePatchType, rawPatch)); err != nil {
-		return fmt.Errorf("error scaling prometheus: %w", err)
+
+	dep.Spec.Replicas = ptr.To(int32(1))
+	if err := cli.Update(ctx, &dep); err != nil {
+		return fmt.Errorf("error scaling up prometheus operator: %w", err)
 	}
 	return nil
 }

scripts/common/longhorn.sh

@@ -157,7 +157,9 @@ function longhorn_to_sc_migration() {
         kubectl annotate storageclass "$longhornStorageClass" storageclass.kubernetes.io/is-default-class-
     done
 
-    longhorn_restore_migration_replicas
+    if ! longhorn_restore_migration_replicas; then
+        log "Failed to restore the scale of Longhorn volumes and replicas. All data was successfully migrated to $destStorageClass and no action needs to be taken."
+    fi
 
     # reset ekco scale
     if [ "$ekcoScaledDown" = "1" ] ; then

testgrid/specs/storage-migration.yaml

@@ -162,6 +162,7 @@
     fi
 - name: Migrate from Longhorn + Minio to OpenEBS + Minio
   flags: "yes"
+  cpu: 6
   installerSpec:
     kubernetes:
       version: 1.24.x
@@ -179,6 +180,8 @@
       version: 1.3.x
     minio:
       version: latest
+    prometheus:
+      version: latest
   upgradeSpec:
     kubernetes:
       version: 1.24.x
@@ -193,11 +196,13 @@
     ekco:
       version: latest
     openebs:
-      version: 3.4.x
+      version: latest
       isLocalPVEnabled: true
       localPVStorageClassName: local
     minio:
       version: latest
+    prometheus:
+      version: latest
   postInstallScript: |
     source /opt/kurl-testgrid/testhelpers.sh
     create_deployment_with_mounted_volume "migration-test" "default" "/data" "registry:2.8.1"