diff --git a/.github/actions/setup-mysql/action.yml b/.github/actions/setup-mysql/action.yml index 444c2133a82..921e7733770 100644 --- a/.github/actions/setup-mysql/action.yml +++ b/.github/actions/setup-mysql/action.yml @@ -24,7 +24,7 @@ runs: # We have to install this old version of libaio1. See also: # https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501 - wget http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb && \ + wget http://archive.ubuntu.com/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb && \ sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb && \ rm libaio1_0.3.112-13build1_amd64.deb @@ -40,7 +40,7 @@ runs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update # libtinfo5 is also needed for older MySQL 5.7 builds. - curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb + curl -L -O http://archive.ubuntu.com/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses6 elif [[ "${{ inputs.flavor }}" == "mysql-8.0" ]]; then diff --git a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml index fcbe3563e12..6e8f36541de 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml @@ -87,7 +87,7 @@ jobs: # Setup Percona Server for MySQL 8.0 sudo apt-get -qq update - sudo apt-get -qq install -y lsb-release gnupg2 + sudo apt-get -qq install -y lsb-release gnupg2 libdbd-mysql-perl wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo percona-release setup pdps8.0 diff --git a/.github/workflows/cluster_endtoend_xb_backup.yml b/.github/workflows/cluster_endtoend_xb_backup.yml index 58aae272d7f..e185491b23a 100644 --- a/.github/workflows/cluster_endtoend_xb_backup.yml +++ b/.github/workflows/cluster_endtoend_xb_backup.yml @@ -87,7 +87,7 @@ jobs: # Setup Percona Server for MySQL 8.0 sudo apt-get -qq update - sudo apt-get -qq install -y lsb-release gnupg2 + sudo apt-get -qq install -y lsb-release gnupg2 libdbd-mysql-perl wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo percona-release setup pdps8.0 diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml index 75e559a1979..199acd12b4c 100644 --- a/.github/workflows/cluster_endtoend_xb_recovery.yml +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -87,7 +87,7 @@ jobs: # Setup Percona Server for MySQL 8.0 sudo apt-get -qq update - sudo apt-get -qq install -y lsb-release gnupg2 + sudo apt-get -qq install -y lsb-release gnupg2 libdbd-mysql-perl wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo percona-release setup pdps8.0 diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml index 16803958848..3bdbb601ad2 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml @@ -87,7 +87,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" apt-get update # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl sudo service etcd stop diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml index 371795c698f..4bbf7fdc215 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml @@ -91,7 +91,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" apt-get update # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl sudo service etcd stop diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml index a7d8559c211..227053846b3 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml @@ -95,8 +95,8 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" apt-get update # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep - + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl + sudo service etcd stop wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml index 6f21cb89c36..f46c91dbcb2 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml @@ -97,8 +97,8 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" apt-get update # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep - + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl + sudo service etcd stop wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml index 8584f0f3ab1..c69aecd7c43 100644 --- a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml @@ -96,8 +96,8 @@ jobs: run: | sudo DEBIAN_FRONTEND="noninteractive" apt-get update # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget - + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget libdbd-mysql-perl + sudo service etcd stop wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml index c5fb483d097..3fee85cf63f 100644 --- a/.github/workflows/upgrade_downgrade_test_semi_sync.yml +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -91,10 +91,10 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" apt-get update # Install everything else we need, and configure - sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep libdbd-mysql-perl sudo service etcd stop - + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo apt-get install -y gnupg2 sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 index e8ca365a704..65e590cfc60 100644 --- a/docker/bootstrap/Dockerfile.mysql80 +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -6,7 +6,9 @@ FROM --platform=linux/amd64 "${image}" USER root # Install MySQL 8.0 -RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \ +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends libdbd-mysql-perl && \ + for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \ for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com A8D3785C && break; done && \ echo 'deb http://repo.mysql.com/apt/debian/ bookworm mysql-8.0' > /etc/apt/sources.list.d/mysql.list && \ for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \ @@ -17,7 +19,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyser echo percona-server-server-8.0 percona-server-server/root_password_again password 'unused'; \ } | debconf-set-selections && \ apt-get update -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 libcurl4-openssl-dev percona-xtrabackup-80 && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev rsync libev4 libcurl4-openssl-dev percona-xtrabackup-80 && \ rm -rf /var/lib/apt/lists/* USER vitess diff --git a/docker/bootstrap/Dockerfile.mysql84 b/docker/bootstrap/Dockerfile.mysql84 index 2f90d588701..988dced756e 100644 --- a/docker/bootstrap/Dockerfile.mysql84 +++ b/docker/bootstrap/Dockerfile.mysql84 @@ -6,7 +6,9 @@ FROM --platform=linux/amd64 "${image}" USER root # Install MySQL 8.4 -RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \ +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends libdbd-mysql-perl && \ + for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \ for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com A8D3785C && break; done && \ echo 'deb http://repo.mysql.com/apt/debian/ bookworm mysql-8.4-lts' > /etc/apt/sources.list.d/mysql.list && \ for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \ @@ -17,7 +19,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyser echo percona-server-server-8.4 percona-server-server/root_password_again password 'unused'; \ } | debconf-set-selections && \ apt-get update -y && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev libdbd-mysql-perl rsync libev4 libcurl4-openssl-dev percona-xtrabackup-84 && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev rsync libev4 libcurl4-openssl-dev percona-xtrabackup-84 && \ rm -rf /var/lib/apt/lists/* USER vitess diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 index 53b16a8eb4b..c6d916b5311 100644 --- a/docker/bootstrap/Dockerfile.percona80 +++ b/docker/bootstrap/Dockerfile.percona80 @@ -6,7 +6,9 @@ FROM --platform=linux/amd64 "${image}" USER root # Install Percona 8.0 -RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends libdbd-mysql-perl \ + && for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done \ && echo 'deb http://repo.percona.com/ps-80/apt bookworm main' > /etc/apt/sources.list.d/percona.list && \ { \ echo debconf debconf/frontend select Noninteractive; \ @@ -19,7 +21,6 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.c libperconaserverclient21 \ percona-server-rocksdb \ bzip2 \ - libdbd-mysql-perl \ rsync \ libev4 \ # && rm -f /etc/apt/sources.list.d/percona.list \ diff --git a/go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go b/go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go index 5c04b133242..40c152b887a 100644 --- a/go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go +++ b/go/test/endtoend/vtorc/primaryfailure/primary_failure_test.go @@ -636,6 +636,12 @@ func TestDownPrimaryPromotionRule(t *testing.T) { // That is the replica which should be promoted in case of primary failure // It should also be caught up when it is promoted func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { + // Slack prioritizes data safety over promotion preferences. PR #677 changed ERS + // to only consider the majority of most-advanced replicas for promotion, which + // excludes lagging replicas even if they have Prefer promotion rules. This test + // expects a lagging replica with a Prefer rule to be promoted after catching up, + // but the new behavior removes it from consideration before the catch-up phase. + t.Skip() defer utils.PrintVTOrcLogsOnFailure(t, clusterInfo.ClusterInstance) utils.SetupVttabletsAndVTOrcs(t, clusterInfo, 2, 1, nil, cluster.VTOrcConfiguration{ LockShardTimeoutSeconds: 5, @@ -715,6 +721,8 @@ func TestDownPrimaryPromotionRuleWithLag(t *testing.T) { // We let a replica in our own cell lag. That is the replica which should be promoted in case of primary failure // It should also be caught up when it is promoted func TestDownPrimaryPromotionRuleWithLagCrossCenter(t *testing.T) { + // Slack does not use PreventCrossCellFailover configuration + t.Skip() defer utils.PrintVTOrcLogsOnFailure(t, clusterInfo.ClusterInstance) utils.SetupVttabletsAndVTOrcs(t, clusterInfo, 2, 1, nil, cluster.VTOrcConfiguration{ LockShardTimeoutSeconds: 5, diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index c702f9c43c8..23ee11573fa 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -221,7 +221,8 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve return err } // Restrict the valid candidates list. We remove any tablet which is of the type DRAINED, RESTORE or BACKUP. - validCandidates, err = restrictValidCandidates(validCandidates, tabletMap) + // The remaining candidates are reduced to a majority with the most advanced relay log GTIDs. + validCandidates, err = restrictValidCandidates(validCandidates, tabletMap, erp.logger) if err != nil { return err } else if len(validCandidates) == 0 { diff --git a/go/vt/vtctl/reparentutil/replication.go b/go/vt/vtctl/reparentutil/replication.go index 0fa60a4c71f..442073be65c 100644 --- a/go/vt/vtctl/reparentutil/replication.go +++ b/go/vt/vtctl/reparentutil/replication.go @@ -18,6 +18,7 @@ package reparentutil import ( "context" + "slices" "sync" "time" @@ -62,9 +63,12 @@ func (rlp *RelayLogPositions) AtLeast(pos *RelayLogPositions) bool { return false } + // if two combined GTID sets are equal, sort by the executed GTID + // set so we pick a position with the most advanced SQL thread. if rlp.Combined.Equal(pos.Combined) { return rlp.Executed.AtLeast(pos.Executed) } + return rlp.Combined.AtLeast(pos.Combined) } @@ -82,6 +86,31 @@ func (rlp *RelayLogPositions) IsZero() bool { return rlp.Combined.IsZero() } +// CompareRelayLogPositions compares two RelayLogPositions, returning: +// 0 if both a anb b are equal positions. +// 1 if a is > than b. +// -1 if a is < than b. +// This can be used as a sort function via +// slices.SortFunc and slices.SortFuncStable. +func CompareRelayLogPositions(a, b *RelayLogPositions) int { + if a.Equal(b) { + return 0 + } + if a.AtLeast(b) && !b.AtLeast(a) { + return -1 + } + return 1 +} + +// sortRelayLogPositions sorts RelayLogPositions using replication positions. +func sortRelayLogPositions(p []*RelayLogPositions) []*RelayLogPositions { + positions := p + slices.SortFunc(positions, func(a, b *RelayLogPositions) int { + return CompareRelayLogPositions(a, b) + }) + return positions +} + // FindPositionsOfAllCandidates will find candidates for an emergency // reparent, and, if successful, return a mapping of those tablet aliases (as // raw strings) to their replication positions for later comparison. diff --git a/go/vt/vtctl/reparentutil/replication_test.go b/go/vt/vtctl/reparentutil/replication_test.go index d0fedde0f8a..3b5c6b91229 100644 --- a/go/vt/vtctl/reparentutil/replication_test.go +++ b/go/vt/vtctl/reparentutil/replication_test.go @@ -1570,3 +1570,93 @@ func TestRelayLogPositions_IsZero(t *testing.T) { rlp.Executed = replication.Position{GTIDSet: gtidSet} assert.False(t, rlp.IsZero()) } + +func TestSortRelayLogPositions(t *testing.T) { + gtidSet1, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-7") + gtidSet2, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-6") + gtidSet3, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-5") + gtidSet4, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-3") + gtidSet5, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-2") + gtidSet6, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-3,3e11fa47-71ca-11e1-9e33-c80aa9429563:1-9999") + gtidSet7, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-1,3e11fa47-71ca-11e1-9e33-c80aa9429563:1-999") + + testCases := []struct { + name string + in []*RelayLogPositions + wanted []*RelayLogPositions + }{ + { + name: "default", + in: []*RelayLogPositions{ + { + Combined: replication.Position{GTIDSet: gtidSet3}, + Executed: replication.Position{GTIDSet: gtidSet4}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet3}, + Executed: replication.Position{GTIDSet: gtidSet4}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet2}, + Executed: replication.Position{GTIDSet: gtidSet3}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet4}, + Executed: replication.Position{GTIDSet: gtidSet5}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet1}, + Executed: replication.Position{GTIDSet: gtidSet5}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet2}, + Executed: replication.Position{GTIDSet: gtidSet5}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet6}, + Executed: replication.Position{GTIDSet: gtidSet7}, + }, + }, + wanted: []*RelayLogPositions{ + { + Combined: replication.Position{GTIDSet: gtidSet1}, + Executed: replication.Position{GTIDSet: gtidSet5}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet2}, + Executed: replication.Position{GTIDSet: gtidSet3}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet2}, + Executed: replication.Position{GTIDSet: gtidSet5}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet3}, + Executed: replication.Position{GTIDSet: gtidSet4}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet3}, + Executed: replication.Position{GTIDSet: gtidSet4}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet6}, + Executed: replication.Position{GTIDSet: gtidSet7}, + }, + { + Combined: replication.Position{GTIDSet: gtidSet4}, + Executed: replication.Position{GTIDSet: gtidSet5}, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + positions := sortRelayLogPositions(testCase.in) + for i, wanted := range testCase.wanted { + require.Equal(t, wanted.Combined.String(), positions[i].Combined.String()) + require.Equal(t, wanted.Executed.String(), positions[i].Executed.String()) + } + }) + } +} diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index 3ec5a907ff5..4893cdfaa11 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -19,6 +19,8 @@ package reparentutil import ( "context" "fmt" + "math" + "slices" "strings" "sync" "time" @@ -316,9 +318,20 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]*RelayLogPo return validTablets, tabletPositions, nil } +// getValidCandidatesMajorityCount returns a number equal to a majority of candidates. If +// there are fewer than 3 candidates, all provided candidates are the majority. +func getValidCandidatesMajorityCount(validCandidates map[string]*RelayLogPositions) int { + totalCandidates := len(validCandidates) + if totalCandidates < 3 { + return totalCandidates + } + return int(math.Floor(float64(totalCandidates)/2) + 1) +} + // restrictValidCandidates is used to restrict some candidates from being considered eligible for becoming the intermediate source or the final promotion candidate -func restrictValidCandidates(validCandidates map[string]*RelayLogPositions, tabletMap map[string]*topo.TabletInfo) (map[string]*RelayLogPositions, error) { +func restrictValidCandidates(validCandidates map[string]*RelayLogPositions, tabletMap map[string]*topo.TabletInfo, logger logutil.Logger) (map[string]*RelayLogPositions, error) { restrictedValidCandidates := make(map[string]*RelayLogPositions) + validPositions := make([]*RelayLogPositions, 0, len(validCandidates)) for candidate, position := range validCandidates { candidateInfo, ok := tabletMap[candidate] if !ok { @@ -329,6 +342,21 @@ func restrictValidCandidates(validCandidates map[string]*RelayLogPositions, tabl continue } restrictedValidCandidates[candidate] = position + validPositions = append(validPositions, position) + } + + // sort by replication positions with greatest GTID set first, then remove + // replicas that are not part of a majority of the most-advanced replicas. + validPositions = sortRelayLogPositions(validPositions) + majorityCandidatesCount := getValidCandidatesMajorityCount(restrictedValidCandidates) + validPositions = validPositions[:majorityCandidatesCount] + for tabletAlias, position := range restrictedValidCandidates { + if !slices.ContainsFunc(validPositions, func(rlp *RelayLogPositions) bool { + return position.Equal(rlp) + }) { + logger.Infof("Ignoring least-advanced tablet as a candidate: %s", tabletAlias) + delete(restrictedValidCandidates, tabletAlias) + } } return restrictedValidCandidates, nil } diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index 9e7d26b15e5..aa52831188b 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1764,6 +1764,10 @@ func TestWaitForCatchUp(t *testing.T) { } func TestRestrictValidCandidates(t *testing.T) { + gtidSet1, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-6") + gtidSet2, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-5") + gtidSet3, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-3") + gtidSet4, _ := replication.ParseMysql56GTIDSet("3e11fa47-71ca-11e1-9e33-c80aa9429562:1-2") tests := []struct { name string validCandidates map[string]*RelayLogPositions @@ -1773,12 +1777,34 @@ func TestRestrictValidCandidates(t *testing.T) { { name: "remove invalid tablets", validCandidates: map[string]*RelayLogPositions{ - "zone1-0000000100": {}, - "zone1-0000000101": {}, - "zone1-0000000102": {}, - "zone1-0000000103": {}, - "zone1-0000000104": {}, - "zone1-0000000105": {}, + "zone1-0000000100": { + Combined: replication.Position{GTIDSet: gtidSet1}, + Executed: replication.Position{GTIDSet: gtidSet2}, + }, + "zone1-0000000101": { + Combined: replication.Position{GTIDSet: gtidSet2}, + Executed: replication.Position{GTIDSet: gtidSet3}, + }, + "zone1-0000000102": { + Combined: replication.Position{GTIDSet: gtidSet2}, + Executed: replication.Position{GTIDSet: gtidSet3}, + }, + "zone1-0000000103": { + Combined: replication.Position{GTIDSet: gtidSet3}, + Executed: replication.Position{GTIDSet: gtidSet4}, + }, + "zone1-0000000104": { + Combined: replication.Position{GTIDSet: gtidSet3}, + Executed: replication.Position{GTIDSet: gtidSet4}, + }, + "zone1-0000000105": { + Combined: replication.Position{GTIDSet: gtidSet4}, + Executed: replication.Position{GTIDSet: gtidSet4}, + }, + "zone1-0000000106": { + Combined: replication.Position{GTIDSet: gtidSet2}, // == to zone1-0000000101 + Executed: replication.Position{GTIDSet: gtidSet4}, // == to zone1-0000000101 + 1 + }, }, tabletMap: map[string]*topo.TabletInfo{ "zone1-0000000100": { @@ -1830,23 +1856,42 @@ func TestRestrictValidCandidates(t *testing.T) { Tablet: &topodatapb.Tablet{ Alias: &topodatapb.TabletAlias{ Cell: "zone1", - Uid: 103, + Uid: 105, }, Type: topodatapb.TabletType_BACKUP, }, }, + "zone1-0000000106": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 106, + }, + Type: topodatapb.TabletType_REPLICA, + }, + }, }, result: map[string]*RelayLogPositions{ - "zone1-0000000100": {}, - "zone1-0000000101": {}, - "zone1-0000000104": {}, + "zone1-0000000100": { + Combined: replication.Position{GTIDSet: gtidSet1}, + Executed: replication.Position{GTIDSet: gtidSet2}, + }, + "zone1-0000000101": { + Combined: replication.Position{GTIDSet: gtidSet2}, + Executed: replication.Position{GTIDSet: gtidSet3}, + }, + "zone1-0000000106": { + Combined: replication.Position{GTIDSet: gtidSet2}, + Executed: replication.Position{GTIDSet: gtidSet4}, + }, }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - res, err := restrictValidCandidates(test.validCandidates, test.tabletMap) + logger := logutil.NewMemoryLogger() + res, err := restrictValidCandidates(test.validCandidates, test.tabletMap, logger) assert.NoError(t, err) assert.Equal(t, res, test.result) }) @@ -2116,3 +2161,18 @@ func TestGetBackupCandidates(t *testing.T) { }) } } + +func TestGetValidCandidatesMajorityCount(t *testing.T) { + buildCandidatesFunc := func(length int) map[string]*RelayLogPositions { + candidates := make(map[string]*RelayLogPositions, length) + for i := 1; i <= length; i++ { + candidates[fmt.Sprintf("candidate-%d", i)] = &RelayLogPositions{} + } + return candidates + } + require.Equal(t, 1, getValidCandidatesMajorityCount(buildCandidatesFunc(1))) + require.Equal(t, 2, getValidCandidatesMajorityCount(buildCandidatesFunc(2))) + require.Equal(t, 2, getValidCandidatesMajorityCount(buildCandidatesFunc(3))) + require.Equal(t, 3, getValidCandidatesMajorityCount(buildCandidatesFunc(5))) + require.Equal(t, 5, getValidCandidatesMajorityCount(buildCandidatesFunc(9))) +} diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index 663ca5b52f5..237d4124be8 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -112,7 +112,7 @@ jobs: # Setup Percona Server for MySQL 8.0 sudo apt-get -qq update - sudo apt-get -qq install -y lsb-release gnupg2 + sudo apt-get -qq install -y lsb-release gnupg2 libdbd-mysql-perl wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb sudo percona-release setup pdps8.0