Stop clusters (#4644)

* stop database * add debt status record * debt system skip not running kb cluster * optimize account controller logger info --------- Co-authored-by: jiahui <[email protected]>
labring · May 14, 2024 · 16c8524 · 16c8524
1 parent 13bd347
commit 16c8524
Show file tree

Hide file tree

Showing 13 changed files with 771 additions and 135 deletions.
diff --git a/controllers/account/api/v1/debt_types.go b/controllers/account/api/v1/debt_types.go
@@ -72,7 +72,7 @@ type DebtStatus struct {
 type DebtStatusRecord struct {
 	LastStatus    DebtStatusType `json:"lastDebtStatus,omitempty"`
 	CurrentStatus DebtStatusType `json:"currentStatus,omitempty"`
-	UpdateTime    int64          `json:"updateTime,omitempty"`
+	UpdateTime    metav1.Time    `json:"updateTime,omitempty"`
 }
 
 //+kubebuilder:object:root=true

diff --git a/controllers/account/config/crd/bases/account.sealos.io_debts.yaml b/controllers/account/config/crd/bases/account.sealos.io_debts.yaml
@@ -59,6 +59,19 @@ spec:
           status:
             description: DebtStatus defines the observed state of Debt
             properties:
+              debtStatusRecords:
+                items:
+                  description: DebtStatusRecord defines the observed state of Debt
+                  properties:
+                    currentStatus:
+                      type: string
+                    lastDebtStatus:
+                      type: string
+                    updateTime:
+                      format: date-time
+                      type: string
+                  type: object
+                type: array
               lastUpdateTimestamp:
                 format: int64
                 type: integer

diff --git a/controllers/account/config/rbac/role.yaml b/controllers/account/config/rbac/role.yaml
@@ -38,17 +38,17 @@ rules:
   verbs:
   - get
 - apiGroups:
-    - ""
+  - ""
   resources:
-    - secrets
+  - secrets
   verbs:
-    - create
-    - delete
-    - get
-    - list
-    - patch
-    - update
-    - watch
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - account.sealos.io
   resources:
@@ -232,9 +232,9 @@ rules:
   - patch
   - update
 - apiGroups:
-  - account.sealos.io
+  - apps
   resources:
-  - transfers
+  - daemonsets
   verbs:
   - create
   - delete
@@ -244,23 +244,21 @@ rules:
   - update
   - watch
 - apiGroups:
-  - account.sealos.io
-  resources:
-  - transfers/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - account.sealos.io
+  - apps
   resources:
-  - transfers/status
+  - deployments
   verbs:
+  - create
+  - delete
   - get
+  - list
   - patch
   - update
+  - watch
 - apiGroups:
   - apps
   resources:
-  - daemonsets
+  - replicasets
   verbs:
   - create
   - delete
@@ -272,7 +270,7 @@ rules:
 - apiGroups:
   - apps
   resources:
-  - deployments
+  - statefulsets
   verbs:
   - create
   - delete
@@ -282,9 +280,9 @@ rules:
   - update
   - watch
 - apiGroups:
-  - apps
+  - apps.kubeblocks.io
   resources:
-  - replicasets
+  - clusters
   verbs:
   - create
   - delete
@@ -294,9 +292,17 @@ rules:
   - update
   - watch
 - apiGroups:
-  - apps
+  - apps.kubeblocks.io
   resources:
-  - statefulsets
+  - clusters/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - apps.kubeblocks.io
+  resources:
+  - opsrequests
   verbs:
   - create
   - delete
@@ -305,6 +311,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - apps.kubeblocks.io
+  resources:
+  - opsrequests/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - ""
   resources:

diff --git a/controllers/account/controllers/debt_controller.go b/controllers/account/controllers/debt_controller.go
@@ -156,14 +156,14 @@ func (r *DebtReconciler) reconcile(ctx context.Context, owner string) error {
 	// In a multi-region scenario, select the region where the account is created for SMS notification
 	smsEnable := account.CreateRegionID == r.LocalRegionID
 
-	r.Logger.Info("reconcile debt", "account", owner, "balance", account.Balance, "deduction balance", account.DeductionBalance)
+	//r.Logger.Info("reconcile debt", "account", owner, "balance", account.Balance, "deduction balance", account.DeductionBalance)
 	if err := r.Get(ctx, client.ObjectKey{Name: GetDebtName(owner), Namespace: r.accountSystemNamespace}, debt); client.IgnoreNotFound(err) != nil {
 		return err
 	} else if err != nil {
 		if err := r.syncDebt(ctx, owner, debt); err != nil {
 			return err
 		}
-		r.Logger.Info("create or update debt success", "debt", debt)
+		//r.Logger.Info("create or update debt success", "debt", debt)
 	}
 
 	nsList, err := getOwnNsList(r.Client, getUsername(owner))
@@ -210,7 +210,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 		if oweamount >= 0 {
 			return nil
 		}
-		update = SetDebtStatus(debt, accountv1.WarningPeriod)
+		update = SetDebtStatus(debt, accountv1.NormalPeriod, accountv1.WarningPeriod)
 		if err := r.sendWarningNotice(ctx, debt.Spec.UserName, oweamount, userNamespaceList, smsEnable); err != nil {
 			r.Logger.Error(err, "send warning notice error")
 		}
@@ -225,7 +225,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 			  警告期 -> 临近删除期： 更新status 状态approachingDeletion事件及更新时间，发送临近删除消息通知
 		*/
 		if oweamount >= 0 {
-			update = SetDebtStatus(debt, accountv1.NormalPeriod)
+			update = SetDebtStatus(debt, accountv1.WarningPeriod, accountv1.NormalPeriod)
 			if err := r.readNotice(ctx, userNamespaceList, WarningNotice); err != nil {
 				r.Logger.Error(err, "readNotice WarningNotice error")
 			}
@@ -235,7 +235,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 		if updateIntervalSeconds < DebtConfig[accountv1.ApproachingDeletionPeriod] && (account.Balance/2)+oweamount > 0 {
 			return nil
 		}
-		update = SetDebtStatus(debt, accountv1.ApproachingDeletionPeriod)
+		update = SetDebtStatus(debt, accountv1.WarningPeriod, accountv1.ApproachingDeletionPeriod)
 		if err := r.sendApproachingDeletionNotice(ctx, debt.Spec.UserName, oweamount, userNamespaceList, smsEnable); err != nil {
 			r.Logger.Error(err, "sendApproachingDeletionNotice error")
 		}
@@ -251,7 +251,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 			  临近删除期 -> 即刻删除期： 执行暂停用户资源，更新status 状态imminentDeletionPeriod事件及更新时间，发送最终删除消息通知
 		*/
 		if oweamount >= 0 {
-			update = SetDebtStatus(debt, accountv1.NormalPeriod)
+			update = SetDebtStatus(debt, accountv1.ApproachingDeletionPeriod, accountv1.NormalPeriod)
 			if err := r.readNotice(ctx, userNamespaceList, ApproachingDeletionNotice, WarningNotice); err != nil {
 				r.Logger.Error(err, "readNotice ApproachingDeletionNotice error")
 			}
@@ -260,7 +260,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 		if updateIntervalSeconds < DebtConfig[accountv1.ImminentDeletionPeriod] && account.Balance+oweamount > 0 {
 			return nil
 		}
-		update = SetDebtStatus(debt, accountv1.ImminentDeletionPeriod)
+		update = SetDebtStatus(debt, accountv1.ApproachingDeletionPeriod, accountv1.ImminentDeletionPeriod)
 		if err := r.sendImminentDeletionNotice(ctx, debt.Spec.UserName, oweamount, userNamespaceList, smsEnable); err != nil {
 			r.Logger.Error(err, "sendImminentDeletionNotice error")
 		}
@@ -277,7 +277,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 			  即刻删除期 -> 最终删除期： 删除用户全部资源，更新status 状态finalDeletionPeriod事件及更新时间。发生最终删除消息通知
 		*/
 		if oweamount >= 0 {
-			update = SetDebtStatus(debt, accountv1.NormalPeriod)
+			update = SetDebtStatus(debt, accountv1.ImminentDeletionPeriod, accountv1.NormalPeriod)
 			// 恢复用户资源
 			if err := r.ResumeUserResource(ctx, userNamespaceList); err != nil {
 				return err
@@ -292,7 +292,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 			return nil
 		}
 		// TODO 暂时只暂停资源，后续会添加真正删除全部资源逻辑, 或直接删除namespace
-		update = SetDebtStatus(debt, accountv1.FinalDeletionPeriod)
+		update = SetDebtStatus(debt, accountv1.ImminentDeletionPeriod, accountv1.FinalDeletionPeriod)
 		if err := r.sendFinalDeletionNotice(ctx, debt.Spec.UserName, oweamount, userNamespaceList, smsEnable); err != nil {
 			r.Error(err, "sendFinalDeletionNotice error")
 		}
@@ -309,7 +309,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 				r.Logger.Error(err, "readNotice FinalDeletionNotice error")
 			}
 			//TODO 用户从欠费到正常，是否需要发送消息通知
-			update = SetDebtStatus(debt, accountv1.NormalPeriod)
+			update = SetDebtStatus(debt, accountv1.FinalDeletionPeriod, accountv1.NormalPeriod)
 
 			// TODO 暂时非真正完全删除，仍可恢复用户资源，后续会添加真正删除全部资源逻辑，不在执行恢复逻辑
 			if err := r.ResumeUserResource(ctx, userNamespaceList); err != nil {
@@ -326,7 +326,7 @@ func (r *DebtReconciler) reconcileDebtStatus(ctx context.Context, debt *accountv
 	}
 
 	if update {
-		r.Logger.Info("update debt status", "account", debt.Spec.UserName,
+		r.Logger.V(1).Info("update debt status", "account", debt.Spec.UserName,
 			"last status", lastStatus, "last update time", time.Unix(debt.Status.LastUpdateTimestamp, 0).Format(time.RFC3339),
 			"current status", debt.Status.AccountDebtStatus, "time", time.Now().UTC().Format(time.RFC3339))
 		return r.Status().Update(ctx, debt)
@@ -346,9 +346,24 @@ func (r *DebtReconciler) syncDebt(ctx context.Context, owner string, debt *accou
 	return nil
 }
 
-func SetDebtStatus(debt *accountv1.Debt, status accountv1.DebtStatusType) bool {
-	debt.Status.AccountDebtStatus = status
-	debt.Status.LastUpdateTimestamp = time.Now().UTC().Unix()
+var MaxDebtHistoryStatusLength = env.GetIntEnvWithDefault("MAX_DEBT_HISTORY_STATUS_LENGTH", 10)
+
+func SetDebtStatus(debt *accountv1.Debt, lastStatus, currentStatus accountv1.DebtStatusType) bool {
+	debt.Status.AccountDebtStatus = currentStatus
+	now := time.Now().UTC()
+	debt.Status.LastUpdateTimestamp = now.Unix()
+	length := len(debt.Status.DebtStatusRecords)
+	statusRecord := accountv1.DebtStatusRecord{
+		LastStatus:    lastStatus,
+		CurrentStatus: currentStatus,
+		UpdateTime:    metav1.NewTime(now),
+	}
+	if length == 0 {
+		debt.Status.DebtStatusRecords = make([]accountv1.DebtStatusRecord, 0)
+	} else if length == MaxDebtHistoryStatusLength {
+		debt.Status.DebtStatusRecords = debt.Status.DebtStatusRecords[1:]
+	}
+	debt.Status.DebtStatusRecords = append(debt.Status.DebtStatusRecords, statusRecord)
 	return true
 }
 

diff --git a/controllers/account/controllers/namespace_controller.go b/controllers/account/controllers/namespace_controller.go
@@ -23,6 +23,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/madmin-go/v3"
+
+	v1 "github.com/labring/sealos/controllers/account/api/v1"
+
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
@@ -31,13 +35,10 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/watch"
 
-	"github.com/go-logr/logr"
-	"github.com/minio/madmin-go/v3"
-
-	v1 "github.com/labring/sealos/controllers/account/api/v1"
-
 	objectstoragev1 "github/labring/sealos/controllers/objectstorage/api/v1"
 
+	kbv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1"
+	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -70,6 +71,10 @@ const (
 //+kubebuilder:rbac:groups=core,resources=namespaces/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=core,resources=namespaces/finalizers,verbs=update
 //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=apps.kubeblocks.io,resources=clusters,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=apps.kubeblocks.io,resources=clusters/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=apps.kubeblocks.io,resources=opsrequests,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=apps.kubeblocks.io,resources=opsrequests/status,verbs=get;update;patch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -94,16 +99,14 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		logger.Error(fmt.Errorf("no debt status"), "no debt status")
 		return ctrl.Result{}, nil
 	}
-	logger.Info("debt status", "status", debtStatus)
+	logger.V(1).Info("debt status", "status", debtStatus)
 	switch debtStatus {
 	case v1.SuspendDebtNamespaceAnnoStatus:
-		logger.Info("suspend namespace resources")
 		if err := r.SuspendUserResource(ctx, req.NamespacedName.Name); err != nil {
 			logger.Error(err, "suspend namespace resources failed")
 			return ctrl.Result{}, err
 		}
 	case v1.ResumeDebtNamespaceAnnoStatus:
-		logger.Info("resume namespace resources")
 		if err := r.ResumeUserResource(ctx, req.NamespacedName.Name); err != nil {
 			logger.Error(err, "resume namespace resources failed")
 			return ctrl.Result{}, err
@@ -130,6 +133,7 @@ func (r *NamespaceReconciler) SuspendUserResource(ctx context.Context, namespace
 	// suspend pod: deploy pod && clone unmanaged pod
 	// delete infra cr
 	pipelines := []func(context.Context, string) error{
+		r.suspendKBCluster,
 		r.suspendOrphanPod,
 		r.limitResourceQuotaCreate,
 		r.deleteControlledPod,
@@ -187,6 +191,29 @@ func GetLimit0ResourceQuota(namespace string) *corev1.ResourceQuota {
 	return &quota
 }
 
+func (r *NamespaceReconciler) suspendKBCluster(ctx context.Context, namespace string) error {
+	kbClusterList := kbv1alpha1.ClusterList{}
+	if err := r.Client.List(ctx, &kbClusterList, client.InNamespace(namespace)); err != nil {
+		return err
+	}
+	for _, kbCluster := range kbClusterList.Items {
+		if kbCluster.Status.Phase != kbv1alpha1.RunningClusterPhase {
+			continue
+		}
+		ops := kbv1alpha1.OpsRequest{}
+		ops.Namespace = kbCluster.Namespace
+		ops.ObjectMeta.Name = "stop-" + kbCluster.Name + "-" + time.Now().Format("2006-01-02-15")
+		ops.Spec.TTLSecondsAfterSucceed = 1
+		ops.Spec.ClusterRef = kbCluster.Name
+		ops.Spec.Type = "Stop"
+		err := r.Client.Create(ctx, &ops)
+		if err != nil {
+			r.Log.Error(err, "create ops request failed", "ops", ops.Name, "namespace", ops.Namespace)
+		}
+	}
+	return nil
+}
+
 func (r *NamespaceReconciler) suspendOrphanPod(ctx context.Context, namespace string) error {
 	podList := corev1.PodList{}
 	if err := r.Client.List(ctx, &podList, client.InNamespace(namespace)); err != nil {