[slack-22.0] forward-port: support a minority of lagging tablets in ERS (#677)

.github/actions/setup-mysql/action.yml

@@ -24,7 +24,7 @@ runs:
 
         # We have to install this old version of libaio1. See also:
         # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501
-        wget http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb && \
+        wget http://archive.ubuntu.com/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb && \
           sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb && \
           rm libaio1_0.3.112-13build1_amd64.deb
 
@@ -40,7 +40,7 @@ runs:
           sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
           sudo apt-get update
           # libtinfo5 is also needed for older MySQL 5.7 builds.
-          curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
+          curl -L -O http://archive.ubuntu.com/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
           sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb
           sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6
         elif [[ "${{ inputs.flavor }}" == "mysql-8.0" ]]; then

.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml

@@ -87,7 +87,7 @@ jobs:
 
         # Setup Percona Server for MySQL 8.0
         sudo apt-get -qq update
-        sudo apt-get -qq install -y lsb-release gnupg2
+        sudo apt-get -qq install -y lsb-release gnupg2 libdbd-mysql-perl
         wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
         sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo percona-release setup pdps8.0

.github/workflows/cluster_endtoend_xb_backup.yml

@@ -87,7 +87,7 @@ jobs:
 
         # Setup Percona Server for MySQL 8.0
         sudo apt-get -qq update
-        sudo apt-get -qq install -y lsb-release gnupg2
+        sudo apt-get -qq install -y lsb-release gnupg2 libdbd-mysql-perl
         wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
         sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo percona-release setup pdps8.0

.github/workflows/cluster_endtoend_xb_recovery.yml

@@ -87,7 +87,7 @@ jobs:
 
         # Setup Percona Server for MySQL 8.0
         sudo apt-get -qq update
-        sudo apt-get -qq install -y lsb-release gnupg2
+        sudo apt-get -qq install -y lsb-release gnupg2 libdbd-mysql-perl
         wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
         sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo percona-release setup pdps8.0

.github/workflows/upgrade_downgrade_test_backups_e2e.yml

@@ -87,7 +87,7 @@ jobs:
         sudo DEBIAN_FRONTEND="noninteractive" apt-get update
 
         # Install everything else we need, and configure
-        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep
+        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl
 
         sudo service etcd stop
 

.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml

@@ -91,7 +91,7 @@ jobs:
         sudo DEBIAN_FRONTEND="noninteractive" apt-get update
 
         # Install everything else we need, and configure
-        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep
+        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl
 
         sudo service etcd stop
 

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -95,8 +95,8 @@ jobs:
         sudo DEBIAN_FRONTEND="noninteractive" apt-get update
 
         # Install everything else we need, and configure
-        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep
-        
+        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl
+
         sudo service etcd stop
 
         wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb

.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml

@@ -97,8 +97,8 @@ jobs:
         sudo DEBIAN_FRONTEND="noninteractive" apt-get update
 
         # Install everything else we need, and configure
-        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep
-        
+        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl
+
         sudo service etcd stop
 
         wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb

.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml

@@ -96,8 +96,8 @@ jobs:
       run: |
         sudo DEBIAN_FRONTEND="noninteractive" apt-get update
         # Install everything else we need, and configure
-        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget
-        
+        sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget libdbd-mysql-perl
+
         sudo service etcd stop
 
         wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb

.github/workflows/upgrade_downgrade_test_semi_sync.yml

@@ -91,10 +91,10 @@ jobs:
           sudo DEBIAN_FRONTEND="noninteractive" apt-get update
 
           # Install everything else we need, and configure
-          sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep
+          sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl
 
           sudo service etcd stop
-          
+
           wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
           sudo apt-get install -y gnupg2
           sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb

docker/bootstrap/Dockerfile.mysql80

@@ -6,7 +6,9 @@ FROM --platform=linux/amd64 "${image}"
 USER root
 
 # Install MySQL 8.0
-RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends libdbd-mysql-perl && \
+    for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \
     for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com A8D3785C && break; done && \
     echo 'deb http://repo.mysql.com/apt/debian/ bookworm mysql-8.0' > /etc/apt/sources.list.d/mysql.list && \
     for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \
@@ -17,7 +19,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyser
         echo percona-server-server-8.0 percona-server-server/root_password_again password 'unused'; \
     } | debconf-set-selections && \
     apt-get update -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 libcurl4-openssl-dev percona-xtrabackup-80 && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev rsync libev4 libcurl4-openssl-dev percona-xtrabackup-80 && \
     rm -rf /var/lib/apt/lists/*
 
 USER vitess

docker/bootstrap/Dockerfile.mysql84

@@ -6,7 +6,9 @@ FROM --platform=linux/amd64 "${image}"
 USER root
 
 # Install MySQL 8.4
-RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends libdbd-mysql-perl && \
+    for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \
     for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com A8D3785C && break; done && \
     echo 'deb http://repo.mysql.com/apt/debian/ bookworm mysql-8.4-lts' > /etc/apt/sources.list.d/mysql.list && \
     for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \
@@ -17,7 +19,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyser
         echo percona-server-server-8.4 percona-server-server/root_password_again password 'unused'; \
     } | debconf-set-selections && \
     apt-get update -y && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 libcurl4-openssl-dev percona-xtrabackup-84 && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev rsync libev4 libcurl4-openssl-dev percona-xtrabackup-84 && \
     rm -rf /var/lib/apt/lists/*
 
 USER vitess

docker/bootstrap/Dockerfile.percona80

@@ -6,7 +6,9 @@ FROM --platform=linux/amd64 "${image}"
 USER root
 
 # Install Percona 8.0
-RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done \
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends libdbd-mysql-perl \
+    && for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done \
     && echo 'deb http://repo.percona.com/ps-80/apt bookworm main' > /etc/apt/sources.list.d/percona.list && \
     { \
         echo debconf debconf/frontend select Noninteractive; \
@@ -19,7 +21,6 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.c
 	libperconaserverclient21 \
 	percona-server-rocksdb \
 	bzip2 \
-	libdbd-mysql-perl \
 	rsync \
 	libev4 \
 #    && rm -f /etc/apt/sources.list.d/percona.list \

go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go

@@ -636,6 +636,12 @@ func TestDownPrimaryPromotionRule(t *testing.T) {
 // That is the replica which should be promoted in case of primary failure
 // It should also be caught up when it is promoted
 func TestDownPrimaryPromotionRuleWithLag(t *testing.T) {
+	// Slack prioritizes data safety over promotion preferences. PR #677 changed ERS
+	// to only consider the majority of most-advanced replicas for promotion, which
+	// excludes lagging replicas even if they have Prefer promotion rules. This test
+	// expects a lagging replica with a Prefer rule to be promoted after catching up,
+	// but the new behavior removes it from consideration before the catch-up phase.
+	t.Skip()
 	defer utils.PrintVTOrcLogsOnFailure(t, clusterInfo.ClusterInstance)
 	utils.SetupVttabletsAndVTOrcs(t, clusterInfo, 2, 1, nil, cluster.VTOrcConfiguration{
 		LockShardTimeoutSeconds: 5,
@@ -715,6 +721,8 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) {
 // We let a replica in our own cell lag. That is the replica which should be promoted in case of primary failure
 // It should also be caught up when it is promoted
 func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) {
+	// Slack does not use PreventCrossCellFailover configuration
+	t.Skip()
 	defer utils.PrintVTOrcLogsOnFailure(t, clusterInfo.ClusterInstance)
 	utils.SetupVttabletsAndVTOrcs(t, clusterInfo, 2, 1, nil, cluster.VTOrcConfiguration{
 		LockShardTimeoutSeconds:  5,

go/vt/vtctl/reparentutil/emergency_reparenter.go

@@ -221,7 +221,8 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve
 		return err
 	}
 	// Restrict the valid candidates list. We remove any tablet which is of the type DRAINED, RESTORE or BACKUP.
-	validCandidates, err = restrictValidCandidates(validCandidates, tabletMap)
+	// The remaining candidates are reduced to a majority with the most advanced relay log GTIDs.
+	validCandidates, err = restrictValidCandidates(validCandidates, tabletMap, erp.logger)
 	if err != nil {
 		return err
 	} else if len(validCandidates) == 0 {

go/vt/vtctl/reparentutil/replication.go

@@ -18,6 +18,7 @@ package reparentutil
 
 import (
 	"context"
+	"slices"
 	"sync"
 	"time"
 
@@ -62,9 +63,12 @@ func (rlp *RelayLogPositions) AtLeast(pos *RelayLogPositions) bool {
 		return false
 	}
 
+	// if two combined GTID sets are equal, sort by the executed GTID
+	// set so we pick a position with the most advanced SQL thread.
 	if rlp.Combined.Equal(pos.Combined) {
 		return rlp.Executed.AtLeast(pos.Executed)
 	}
+
 	return rlp.Combined.AtLeast(pos.Combined)
 }
 
@@ -82,6 +86,31 @@ func (rlp *RelayLogPositions) IsZero() bool {
 	return rlp.Combined.IsZero()
 }
 
+// CompareRelayLogPositions compares two RelayLogPositions, returning:
+// 0 if both a anb b are equal positions.
+// 1 if a is > than b.
+// -1 if a is < than b.
+// This can be used as a sort function via
+// slices.SortFunc and slices.SortFuncStable.
+func CompareRelayLogPositions(a, b *RelayLogPositions) int {
+	if a.Equal(b) {
+		return 0
+	}
+	if a.AtLeast(b) && !b.AtLeast(a) {
+		return -1
+	}
+	return 1
+}
+
+// sortRelayLogPositions sorts RelayLogPositions using replication positions.
+func sortRelayLogPositions(p []*RelayLogPositions) []*RelayLogPositions {
+	positions := p
+	slices.SortFunc(positions, func(a, b *RelayLogPositions) int {
+		return CompareRelayLogPositions(a, b)
+	})
+	return positions
+}
+
 // FindPositionsOfAllCandidates will find candidates for an emergency
 // reparent, and, if successful, return a mapping of those tablet aliases (as
 // raw strings) to their replication positions for later comparison.

go/vt/vtctl/reparentutil/replication_test.go

@@ -1570,3 +1570,93 @@ func TestRelayLogPositions_IsZero(t *testing.T) {
 	rlp.Executed = replication.Position{GTIDSet: gtidSet}
 	assert.False(t, rlp.IsZero())
 }
+
+func TestSortRelayLogPositions(t *testing.T) {
+	gtidSet1, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-7")
+	gtidSet2, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-6")
+	gtidSet3, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-5")
+	gtidSet4, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-3")
+	gtidSet5, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-2")
+	gtidSet6, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-3,3e11fa47-71ca-11e1-9e33-c80aa9429563:1-9999")
+	gtidSet7, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-1,3e11fa47-71ca-11e1-9e33-c80aa9429563:1-999")
+
+	testCases := []struct {
+		name   string
+		in     []*RelayLogPositions
+		wanted []*RelayLogPositions
+	}{
+		{
+			name: "default",
+			in: []*RelayLogPositions{
+				{
+					Combined: replication.Position{GTIDSet: gtidSet3},
+					Executed: replication.Position{GTIDSet: gtidSet4},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet3},
+					Executed: replication.Position{GTIDSet: gtidSet4},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet2},
+					Executed: replication.Position{GTIDSet: gtidSet3},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet4},
+					Executed: replication.Position{GTIDSet: gtidSet5},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet1},
+					Executed: replication.Position{GTIDSet: gtidSet5},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet2},
+					Executed: replication.Position{GTIDSet: gtidSet5},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet6},
+					Executed: replication.Position{GTIDSet: gtidSet7},
+				},
+			},
+			wanted: []*RelayLogPositions{
+				{
+					Combined: replication.Position{GTIDSet: gtidSet1},
+					Executed: replication.Position{GTIDSet: gtidSet5},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet2},
+					Executed: replication.Position{GTIDSet: gtidSet3},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet2},
+					Executed: replication.Position{GTIDSet: gtidSet5},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet3},
+					Executed: replication.Position{GTIDSet: gtidSet4},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet3},
+					Executed: replication.Position{GTIDSet: gtidSet4},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet6},
+					Executed: replication.Position{GTIDSet: gtidSet7},
+				},
+				{
+					Combined: replication.Position{GTIDSet: gtidSet4},
+					Executed: replication.Position{GTIDSet: gtidSet5},
+				},
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			positions := sortRelayLogPositions(testCase.in)
+			for i, wanted := range testCase.wanted {
+				require.Equal(t, wanted.Combined.String(), positions[i].Combined.String())
+				require.Equal(t, wanted.Executed.String(), positions[i].Executed.String())
+			}
+		})
+	}
+}

go/vt/vtctl/reparentutil/util.go

@@ -19,6 +19,8 @@ package reparentutil
 import (
 	"context"
 	"fmt"
+	"math"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -316,9 +318,20 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]*RelayLogPo
 	return validTablets, tabletPositions, nil
 }
 
+// getValidCandidatesMajorityCount returns a number equal to a majority of candidates. If
+// there are fewer than 3 candidates, all provided candidates are the majority.
+func getValidCandidatesMajorityCount(validCandidates map[string]*RelayLogPositions) int {
+	totalCandidates := len(validCandidates)
+	if totalCandidates < 3 {
+		return totalCandidates
+	}
+	return int(math.Floor(float64(totalCandidates)/2) + 1)
+}
+
 // restrictValidCandidates is used to restrict some candidates from being considered eligible for becoming the intermediate source or the final promotion candidate
-func restrictValidCandidates(validCandidates map[string]*RelayLogPositions, tabletMap map[string]*topo.TabletInfo) (map[string]*RelayLogPositions, error) {
+func restrictValidCandidates(validCandidates map[string]*RelayLogPositions, tabletMap map[string]*topo.TabletInfo, logger logutil.Logger) (map[string]*RelayLogPositions, error) {
 	restrictedValidCandidates := make(map[string]*RelayLogPositions)
+	validPositions := make([]*RelayLogPositions, 0, len(validCandidates))
 	for candidate, position := range validCandidates {
 		candidateInfo, ok := tabletMap[candidate]
 		if !ok {
@@ -329,6 +342,21 @@ func restrictValidCandidates(validCandidates map[string]*RelayLogPositions, tabl
 			continue
 		}
 		restrictedValidCandidates[candidate] = position
+		validPositions = append(validPositions, position)
+	}
+
+	// sort by replication positions with greatest GTID set first, then remove
+	// replicas that are not part of a majority of the most-advanced replicas.
+	validPositions = sortRelayLogPositions(validPositions)
+	majorityCandidatesCount := getValidCandidatesMajorityCount(restrictedValidCandidates)
+	validPositions = validPositions[:majorityCandidatesCount]
+	for tabletAlias, position := range restrictedValidCandidates {
+		if !slices.ContainsFunc(validPositions, func(rlp *RelayLogPositions) bool {
+			return position.Equal(rlp)
+		}) {
+			logger.Infof("Ignoring least-advanced tablet as a candidate: %s", tabletAlias)
+			delete(restrictedValidCandidates, tabletAlias)
+		}
 	}
 	return restrictedValidCandidates, nil
 }