From eb1cc0d3dca6ae19b8b7c110452a798008716140 Mon Sep 17 00:00:00 2001 From: yangw Date: Sat, 30 Mar 2024 21:34:27 +0800 Subject: [PATCH] feat: add redisreplication status masterNode (#849) * feat: add redisreplication status masterNode Signed-off-by: drivebyer * fix e2e Signed-off-by: drivebyer * fix lint Signed-off-by: drivebyer * fix lint Signed-off-by: drivebyer --------- Signed-off-by: drivebyer --- api/v1beta1/redisreplication_types.go | 4 ++- api/v1beta2/redisreplication_types.go | 4 ++- ...edis.opstreelabs.in_redisreplications.yaml | 6 ++++ controllers/redisreplication_controller.go | 26 +++++++++++++-- controllers/redissentinel_controller.go | 12 +++++++ k8sutils/redis.go | 32 ++++++++----------- .../redis-replication/chainsaw-test.yaml | 16 ++-------- .../redis-replication/ready-replication.yaml | 8 +++++ 8 files changed, 71 insertions(+), 37 deletions(-) create mode 100644 tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml diff --git a/api/v1beta1/redisreplication_types.go b/api/v1beta1/redisreplication_types.go index 4cca92403..d4be20456 100644 --- a/api/v1beta1/redisreplication_types.go +++ b/api/v1beta1/redisreplication_types.go @@ -31,7 +31,9 @@ func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 { } // RedisStatus defines the observed state of Redis -type RedisReplicationStatus struct{} +type RedisReplicationStatus struct { + MasterNode string `json:"masterNode,omitempty"` +} // +kubebuilder:object:root=true // +kubebuilder:subresource:status diff --git a/api/v1beta2/redisreplication_types.go b/api/v1beta2/redisreplication_types.go index 59248fb55..90bf70796 100644 --- a/api/v1beta2/redisreplication_types.go +++ b/api/v1beta2/redisreplication_types.go @@ -36,7 +36,9 @@ func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 { } // RedisStatus defines the observed state of Redis -type RedisReplicationStatus struct{} +type RedisReplicationStatus struct { + MasterNode string `json:"masterNode,omitempty"` +} // +kubebuilder:object:root=true // +kubebuilder:subresource:status diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml index e130d18c2..eb09d428f 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml @@ -4127,6 +4127,9 @@ spec: type: object status: description: RedisStatus defines the observed state of Redis + properties: + masterNode: + type: string type: object required: - spec @@ -8841,6 +8844,9 @@ spec: type: object status: description: RedisStatus defines the observed state of Redis + properties: + masterNode: + type: string type: object required: - spec diff --git a/controllers/redisreplication_controller.go b/controllers/redisreplication_controller.go index a57308e80..98a2b9fac 100644 --- a/controllers/redisreplication_controller.go +++ b/controllers/redisreplication_controller.go @@ -77,19 +77,39 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{RequeueAfter: time.Second * 60}, nil } - if len(k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master")) > int(leaderReplicas) { + var realMaster string + masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master") + if len(masterNodes) > int(leaderReplicas) { reqLogger.Info("Creating redis replication by executing replication creation commands", "Replication.Ready", strconv.Itoa(int(redisReplicationInfo.Status.ReadyReplicas))) - masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master") slaveNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "slave") - err := k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, r.Log, instance, masterNodes, slaveNodes) + realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes) + if len(slaveNodes) == 0 { + realMaster = masterNodes[0] + } + err := k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, r.Log, instance, masterNodes, realMaster) if err != nil { return ctrl.Result{RequeueAfter: time.Second * 60}, err } } + realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes) + if err := r.UpdateRedisReplicationMaster(ctx, instance, realMaster); err != nil { + return ctrl.Result{}, err + } reqLogger.Info("Will reconcile redis operator in again 10 seconds") return ctrl.Result{RequeueAfter: time.Second * 10}, nil } +func (r *RedisReplicationReconciler) UpdateRedisReplicationMaster(ctx context.Context, instance *redisv1beta2.RedisReplication, masterNode string) error { + if instance.Status.MasterNode == masterNode { + return nil + } + instance.Status.MasterNode = masterNode + if err := r.Client.Status().Update(ctx, instance); err != nil { + return err + } + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *RedisReplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/redissentinel_controller.go b/controllers/redissentinel_controller.go index 2bb206139..831dfaf16 100644 --- a/controllers/redissentinel_controller.go +++ b/controllers/redissentinel_controller.go @@ -11,8 +11,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" ) // RedisSentinelReconciler reconciles a RedisSentinel object @@ -83,5 +86,14 @@ func (r *RedisSentinelReconciler) Reconcile(ctx context.Context, req ctrl.Reques func (r *RedisSentinelReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&redisv1beta2.RedisSentinel{}). + Watches(&redisv1beta2.RedisReplication{}, &handler.Funcs{ + CreateFunc: nil, + UpdateFunc: func(ctx context.Context, event event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { + _ = event.ObjectNew.GetName() + _ = event.ObjectNew.GetNamespace() + }, + DeleteFunc: nil, + GenericFunc: nil, + }). Complete(r) } diff --git a/k8sutils/redis.go b/k8sutils/redis.go index b7f74ef05..bb983084c 100644 --- a/k8sutils/redis.go +++ b/k8sutils/redis.go @@ -554,25 +554,7 @@ func checkAttachedSlave(ctx context.Context, redisClient *redis.Client, logger l return 0 } -func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string, slavePods []string) error { - var realMasterPod string - - for _, podName := range masterPods { - redisClient := configureRedisReplicationClient(client, logger, cr, podName) - defer redisClient.Close() - - if checkAttachedSlave(ctx, redisClient, logger, podName) > 0 { - realMasterPod = podName - break - } - } - // realMasterPod = checkAttachedSlave(ctx, client, logger, cr, masterPods) - - if len(slavePods) < 1 { - realMasterPod = masterPods[0] - logger.V(1).Info("No Master Node Found with attached slave promoting the following pod to master", "pod", masterPods[0]) - } - +func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string, realMasterPod string) error { logger.V(1).Info("Redis Master Node is set to", "pod", realMasterPod) realMasterInfo := RedisDetails{ PodName: realMasterPod, @@ -596,3 +578,15 @@ func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interfa return nil } + +func GetRedisReplicationRealMaster(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string) string { + for _, podName := range masterPods { + redisClient := configureRedisReplicationClient(client, logger, cr, podName) + defer redisClient.Close() + + if checkAttachedSlave(ctx, redisClient, logger, podName) > 0 { + return podName + } + } + return "" +} diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml index ff7822ce7..0b536b458 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml @@ -15,6 +15,8 @@ spec: file: ready-svc.yaml - assert: file: ready-pvc.yaml + - assert: + file: ready-replication.yaml catch: - description: Redis Operator Logs podLogs: @@ -51,16 +53,4 @@ spec: content: | kubectl exec --namespace ${NAMESPACE} redis-replication-0 -- redis-cli -p 6379 set foo-0 bar-0 check: - ($stdout=='OK'): true - # - script: - # timeout: 10s - # content: | - # kubectl exec --namespace ${NAMESPACE} redis-replication-1 -- redis-cli -p 6379 set foo-1 bar-1 - # check: - # ($stdout==`READONLY You can't write against a read only replica.`): true - # - script: - # timeout: 10s - # content: | - # kubectl exec --namespace ${NAMESPACE} redis-replication-2 -- redis-cli -p 6379 set foo-2 bar-2 - # check: - # ($stdout==`READONLY You can't write against a read only replica.`): true \ No newline at end of file + ($stdout=='OK'): true \ No newline at end of file diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml new file mode 100644 index 000000000..f3a13d647 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisReplication +metadata: + name: redis-replication +status: + # by default, the first pod is being selected as master + masterNode: redis-replication-0 \ No newline at end of file