diff --git a/go/test/endtoend/backup/vtbackup/backup_only_test.go b/go/test/endtoend/backup/vtbackup/backup_only_test.go index 39deef1b494..c841ef43a12 100644 --- a/go/test/endtoend/backup/vtbackup/backup_only_test.go +++ b/go/test/endtoend/backup/vtbackup/backup_only_test.go @@ -134,6 +134,11 @@ func prepareCluster(t *testing.T) { // Restore the Tablet restore(t, primary, "replica", "NOT_SERVING") + // restore() returns as soon as the vttablet's HTTP endpoint is responsive, + // but RestoreData runs in the background and holds the actionSema for the + // entire duration of the restore. We need to wait for it to finish before + // issuing RPCs like SetWritable that also need the actionSema. + waitForRestoreComplete(t, primary.VttabletProcess, 180*time.Second) // Vitess expects that the user has set the database into ReadWrite mode before calling // TabletExternallyReparented err = localCluster.VtctldClientProcess.ExecuteCommand( @@ -349,6 +354,28 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS require.NoError(t, err) } +// waitForRestoreComplete waits for a vttablet's background restore to finish. +// The tablet type transitions replica -> restore -> replica during a restore. +// This function polls until it observes "restore" and then sees "replica" again. +// If the restore completes faster than the polling interval and "restore" is +// never observed, the function returns since the restore is already done. +func waitForRestoreComplete(t *testing.T, vttablet *cluster.VttabletProcess, timeout time.Duration) { + t.Helper() + sawRestore := false + assert.Eventually(t, func() bool { + tabletType := vttablet.GetTabletType() + if tabletType == "restore" { + sawRestore = true + } + return sawRestore && tabletType == "replica" + }, timeout, 300*time.Millisecond) + if sawRestore { + require.Equal(t, "replica", vttablet.GetTabletType(), "timed out waiting for tablet restore to complete") + } + // If we never observed "restore" type, the restore likely completed + // before we started polling. Nothing to wait for. +} + func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet, initMysql bool) { extraArgs := []string{"--db-credentials-file", dbCredentialFile} tablet.MysqlctlProcess.ExtraArgs = extraArgs