e2e: fix race in TestFailingReplication

go/test/endtoend/backup/vtbackup/backup_only_test.go

@@ -132,6 +132,11 @@ func prepareCluster(t *testing.T) {
 
 	// Restore the Tablet
 	restore(t, primary, "replica", "NOT_SERVING")
+	// restore() returns as soon as the vttablet's HTTP endpoint is responsive,
+	// but RestoreData runs in the background and holds the actionSema for the
+	// entire duration of the restore. We need to wait for it to finish before
+	// issuing RPCs like SetWritable that also need the actionSema.
+	waitForRestoreComplete(t, primary.VttabletProcess, 180*time.Second)
 	// Vitess expects that the user has set the database into ReadWrite mode before calling
 	// TabletExternallyReparented
 	err = localCluster.VtctldClientProcess.ExecuteCommand(
@@ -347,6 +352,28 @@ func restore(t *testing.T, tablet *cluster.Vttablet, tabletType string, waitForS
 	require.NoError(t, err)
 }
 
+// waitForRestoreComplete waits for a vttablet's background restore to finish.
+// The tablet type transitions replica -> restore -> replica during a restore.
+// This function polls until it observes "restore" and then sees "replica" again.
+// If the restore completes faster than the polling interval and "restore" is
+// never observed, the function returns since the restore is already done.
+func waitForRestoreComplete(t *testing.T, vttablet *cluster.VttabletProcess, timeout time.Duration) {
+	t.Helper()
+	sawRestore := false
+	assert.Eventually(t, func() bool {
+		tabletType := vttablet.GetTabletType()
+		if tabletType == "restore" {
+			sawRestore = true
+		}
+		return sawRestore && tabletType == "replica"
+	}, timeout, 300*time.Millisecond)
+	if sawRestore {
+		require.Equal(t, "replica", vttablet.GetTabletType(), "timed out waiting for tablet restore to complete")
+	}
+	// If we never observed "restore" type, the restore likely completed
+	// before we started polling. Nothing to wait for.
+}
+
 func resetTabletDirectory(t *testing.T, tablet cluster.Vttablet, initMysql bool) {
 	extraArgs := []string{"--db-credentials-file", dbCredentialFile}
 	tablet.MysqlctlProcess.ExtraArgs = extraArgs