refactor: Enhance MySQL role switching with transaction coordination,…

… validation, and rollback
LumeWeb · Nov 24, 2024 · 052698b · 052698b
1 parent eabec5b
commit 052698b
Showing 1 changed file with 83 additions and 6 deletions.
diff --git a/lib/mysql-role.sh b/lib/mysql-role.sh
@@ -16,11 +16,18 @@ register_role_change_handler() {
     ROLE_CHANGE_HANDLERS[$handler_name]=$handler_function
 }
 
+# Track role change timing and status
+declare -g ROLE_CHANGE_START_TIME=""
+declare -g ROLE_CHANGE_STATUS="none"
+
 # Validate role change with enhanced checks
 validate_role_change() {
     local new_role=$1
     local current_role=$2
     local errors=0
+
+    ROLE_CHANGE_START_TIME=$(date +%s)
+    ROLE_CHANGE_STATUS="validating"
 
     # Don't allow invalid transitions
     case "$current_role:$new_role" in
@@ -59,12 +66,30 @@ validate_role_change() {
                 errors=$((errors + 1))
             fi
 
-            # Check for split-brain prevention
-            if ! verify_old_master_down; then
+            # Enhanced replication validation
+            local Exec_Master_Log_Pos=$(mysql -N -e "SHOW SLAVE STATUS\G" | grep "Exec_Master_Log_Pos:" | awk '{print $2}')
+            local Read_Master_Log_Pos=$(mysql -N -e "SHOW SLAVE STATUS\G" | grep "Read_Master_Log_Pos:" | awk '{print $2}')
+
+            if [ "$Exec_Master_Log_Pos" != "$Read_Master_Log_Pos" ]; then
+                log_error "Replication positions not in sync: Exec=$Exec_Master_Log_Pos Read=$Read_Master_Log_Pos"
+                errors=$((errors + 1))
+            fi
+
+            # Check for split-brain prevention with timeout
+            if ! timeout 30 verify_old_master_down; then
                 log_error "Old master may still be active - split-brain prevention"
                 errors=$((errors + 1))
             fi
 
+            # Verify no data drift
+            local checksum_master=$(mysql -N -e "CHECKSUM TABLE mysql.user" | awk '{print $2}')
+            local checksum_slave=$(mysql -h "$old_master_host" -P "$old_master_port" -N -e "CHECKSUM TABLE mysql.user" | awk '{print $2}')
+
+            if [ "$checksum_master" != "$checksum_slave" ]; then
+                log_error "Data drift detected between master and slave"
+                errors=$((errors + 1))
+            fi
+
             [ $errors -eq 0 ] && return 0 || return 1
             ;;
         "master:slave")
@@ -81,11 +106,42 @@ validate_role_change() {
     esac
 }
 
-# Notify handlers of role change
+# Rollback role change
+rollback_role_change() {
+    local role=$1
+    local old_role=$2
+
+    log_warn "Rolling back role change from $old_role to $role"
+
+    case "$old_role" in
+        "master")
+            mysql -e "
+                SET GLOBAL read_only = OFF;
+                SET GLOBAL super_read_only = OFF;
+                FLUSH TABLES WITH READ LOCK;
+                FLUSH LOGS;
+                UNLOCK TABLES;
+            "
+            ;;
+        "slave")
+            mysql -e "
+                SET GLOBAL super_read_only = ON;
+                SET GLOBAL read_only = ON;
+            "
+            ;;
+    esac
+
+    ROLE_CHANGE_STATUS="rolled_back"
+    return 0
+}
+
+# Notify handlers of role change with transaction coordination
 notify_role_change() {
     local new_role=$1
     local old_role=$2
 
+    ROLE_CHANGE_STATUS="in_progress"
+
     # Acquire role change lock
     (
         if ! flock -w 30 9; then
@@ -119,7 +175,13 @@ notify_role_change() {
             fi
         done
 
-        # Configure new role with enhanced error handling
+        # Start global transaction for atomic role change
+        mysql -e "SET GLOBAL TRANSACTION ISOLATION LEVEL SERIALIZABLE"
+        mysql -e "START TRANSACTION WITH CONSISTENT SNAPSHOT"
+
+        local role_change_success=1
+
+        # Configure new role with enhanced error handling and rollback
         case "$new_role" in
             "master")
                 local role_errors=0
@@ -149,8 +211,14 @@ notify_role_change() {
 
                 if [ $role_errors -gt 0 ]; then
                     log_error "Failed to configure master role ($role_errors errors)"
+                    mysql -e "ROLLBACK"
+                    rollback_role_change "$new_role" "$old_role"
                     return 1
                 fi
+
+                # Commit transaction if successful
+                mysql -e "COMMIT"
+                role_change_success=0
                 ;;
             "slave") 
                 mysql -e "
@@ -162,8 +230,17 @@ notify_role_change() {
                 ;;
         esac
 
-        log_info "Role change complete: $old_role -> $new_role"
-        return 0
+        # Calculate role change duration
+        local duration=$(($(date +%s) - ROLE_CHANGE_START_TIME))
+        log_info "Role change complete: $old_role -> $new_role (duration: ${duration}s)"
+
+        if [ $role_change_success -eq 0 ]; then
+            ROLE_CHANGE_STATUS="completed"
+            return 0
+        else
+            ROLE_CHANGE_STATUS="failed"
+            return 1
+        fi
     ) 9>"$ROLE_CHANGE_LOCK"
 }