Wrap fatal TX errors in a new vterrors code

changelog/22.0/22.0.0/summary.md

@@ -24,6 +24,7 @@
   - **[KeyRanges in `--clusters_to_watch` in VTOrc](#key-range-vtorc)**
   - **[Support for Filtering Query logs on Error](#query-logs)**
   - **[Semi-sync monitor in vttablet](#semi-sync-monitor)**
+  - **[Wrapped fatal transaction errors](#new-errors-fatal-tx)**
 - **[Minor Changes](#minor-changes)**
   - **[VTTablet Flags](#flags-vttablet)**
   - **[VTTablet ACL enforcement and reloading](#reloading-vttablet-acl)**
@@ -262,6 +263,22 @@ To address this, the new component continuously monitors the semi-sync status. I
 
 The monitoring interval can be adjusted using the `--semi-sync-monitor-interval` flag, which defaults to 10 seconds.
 
+---
+
+### <a id="new-errors-fatal-tx"/>Wrapped fatal transaction errors</a>
+
+
+When a query fails while being in a transaction, due to the transaction no longer being valid (e.g. PRS, rollout, primary down, etc), the original error is now wrapped around a `VT15001` error.
+
+For non-transactional queries that produce a `VT15001`, VTGate will try to rollback and clear the transaction.
+Any new queries on the same connection will fail with a `VT09032` error, until a `ROLLBACK` is received
+to acknowledge that the transaction was automatically rolled back and cleared by VTGate.
+
+`VT09032` is returned to clients to avoid applications blindly sending queries to VTGate thinking they are still in a transaction.
+
+This change was introduced by [#17669](https://github.com/vitessio/vitess/pull/17669).
+
+
 ---
 
 ## <a id="minor-changes"/>Minor Changes</a>

go/test/endtoend/reparent/newfeaturetest/reparent_test.go

@@ -182,7 +182,7 @@ func TestERSWithWriteInPromoteReplica(t *testing.T) {
 }
 
 func TestBufferingWithMultipleDisruptions(t *testing.T) {
-	clusterInstance := utils.SetupShardedReparentCluster(t, policy.DurabilitySemiSync)
+	clusterInstance := utils.SetupShardedReparentCluster(t, policy.DurabilitySemiSync, nil)
 	defer utils.TeardownCluster(clusterInstance)
 
 	// Stop all VTOrc instances, so that they don't interfere with the test.

go/test/endtoend/reparent/newfeaturetest/reparent_with_open_tx_test.go

@@ -0,0 +1,224 @@
+/*
+Copyright 2025 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package newfeaturetest
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"vitess.io/vitess/go/vt/vterrors"
+
+	"vitess.io/vitess/go/mysql"
+	"vitess.io/vitess/go/test/endtoend/cluster"
+	"vitess.io/vitess/go/test/endtoend/reparent/utils"
+	"vitess.io/vitess/go/vt/vtctl/reparentutil/policy"
+)
+
+var primary int
+
+// This test ensures that we get a VT15001 when doing a commit while the primary is down.
+func testCommitError(t *testing.T, conn *mysql.Conn, clusterInstance *cluster.LocalProcessCluster, tablets []*cluster.Vttablet) {
+	tabletStopped := make(chan bool)
+	commitDone := make(chan bool)
+	idx := 1
+	createTxAndInsertRows(conn, t, &idx)
+
+	go func() {
+		<-tabletStopped
+		_, err := conn.ExecuteFetch("commit", 0, false)
+		require.ErrorContains(t, err, "VT15001")
+		commitDone <- true
+	}()
+
+	reparent(t, clusterInstance, tablets, tabletStopped, commitDone)
+
+	_, err := conn.ExecuteFetch("delete from vt_insert_test", 0, false)
+	require.NoError(t, err)
+}
+
+// This test ensures that we are getting a VT15001 when executing a query on an open transaction
+// while the primary is down. Subsequent queries should fail with a VT09032 until a ROLLBACK
+// or SHOW WARNINGS is issued.
+func testExecuteError(t *testing.T, conn *mysql.Conn, clusterInstance *cluster.LocalProcessCluster, tablets []*cluster.Vttablet) {
+	tabletStopped := make(chan bool)
+	executeDone := make(chan bool)
+	idx := 1
+	createTxAndInsertRows(conn, t, &idx)
+
+	go func() {
+		idx += 5
+		<-tabletStopped
+		_, err := conn.ExecuteFetch(utils.GetInsertMultipleValuesQuery(idx, idx+1, idx+2, idx+3), 0, false)
+		require.ErrorContains(t, err, "VT15001")
+
+		// Subsequent queries after a VT15001 should start returning a VT09032 error until we issue a ROLLBACK
+		_, err = conn.ExecuteFetch("select * from vt_insert_test", 1, false)
+		require.ErrorContains(t, err, "VT09032")
+
+		_, err = conn.ExecuteFetch("rollback", 0, false)
+		require.NoError(t, err)
+		executeDone <- true
+	}()
+
+	reparent(t, clusterInstance, tablets, tabletStopped, executeDone)
+
+	// if the unhealthy shard is the first one where we commited, let's assert that the table is empty on all the shards
+	r, err := conn.ExecuteFetch("select * from vt_insert_test", 1, false)
+	require.NoError(t, err)
+	require.Len(t, r.Rows, 0)
+}
+
+func testExecuteErrorWhileTabletIsNotServing(t *testing.T, conn *mysql.Conn, clusterInstance *cluster.LocalProcessCluster, tablets []*cluster.Vttablet) {
+	tabletNotServing := make(chan bool)
+	executeDone := make(chan bool)
+	idx := 1
+	createTxAndInsertRows(conn, t, &idx)
+
+	go func() {
+		idx += 5
+		<-tabletNotServing
+		_, err := conn.ExecuteFetch(utils.GetInsertMultipleValuesQuery(idx, idx+1, idx+2, idx+3), 0, false)
+		require.ErrorContains(t, err, "VT15001")
+		require.ErrorContains(t, err, vterrors.WrongTablet)
+
+		// Subsequent queries after a VT15001 should start returning a VT09032 error until we issue a ROLLBACK
+		_, err = conn.ExecuteFetch("select * from vt_insert_test", 1, false)
+		require.ErrorContains(t, err, "VT09032")
+
+		_, err = conn.ExecuteFetch("rollback", 0, false)
+		require.NoError(t, err)
+		executeDone <- true
+	}()
+
+	makeTabletNotServing(t, clusterInstance, tablets, tabletNotServing, executeDone)
+
+	// if the unhealthy shard is the first one where we commited, let's assert that the table is empty on all the shards
+	r, err := conn.ExecuteFetch("select * from vt_insert_test", 1, false)
+	require.NoError(t, err)
+	require.Len(t, r.Rows, 0)
+}
+
+func createTxAndInsertRows(conn *mysql.Conn, t *testing.T, idx *int) {
+	_, err := conn.ExecuteFetch("begin", 0, false)
+	require.NoError(t, err)
+
+	for i := 0; i < 25; i++ {
+		*idx += 5
+		_, err = conn.ExecuteFetch(utils.GetInsertMultipleValuesQuery(*idx, *idx+1, *idx+2, *idx+3), 0, false)
+		require.NoError(t, err)
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
+func reparent(t *testing.T, clusterInstance *cluster.LocalProcessCluster, tablets []*cluster.Vttablet, tabletStopped, actionDone chan bool) {
+	// Reparent to the other replica
+	utils.ShardName = "40-80"
+	defer func() {
+		utils.ShardName = "0"
+	}()
+
+	prsTo := primary - 1
+	if primary == 0 {
+		prsTo = primary + 1
+	}
+	output, err := utils.Prs(t, clusterInstance, tablets[prsTo])
+	require.NoError(t, err, "error in PlannedReparentShard output - %s", output)
+
+	// We now restart the vttablet that became a replica.
+	utils.StopTablet(t, tablets[primary], false)
+	tabletStopped <- true
+
+	// Wait for the action triggering the VT15001 to be done before moving on
+	<-actionDone
+
+	tablets[primary].VttabletProcess.ServingStatus = "SERVING"
+	err = tablets[primary].VttabletProcess.Setup()
+	require.NoError(t, err)
+	primary = prsTo
+}
+
+func makeTabletNotServing(t *testing.T, clusterInstance *cluster.LocalProcessCluster, tablets []*cluster.Vttablet, stateChanged, actionDone chan bool) {
+	// Reparent to the other replica
+	utils.ShardName = "40-80"
+	defer func() {
+		utils.ShardName = "0"
+	}()
+
+	prsTo := primary - 1
+	if primary == 0 {
+		prsTo = primary + 1
+	}
+	output, err := utils.Prs(t, clusterInstance, tablets[prsTo])
+	require.NoError(t, err, "error in PlannedReparentShard output - %s", output)
+
+	// We now restart the vttablet that became a replica.
+	utils.StopTablet(t, tablets[primary], false)
+	tablets[primary].VttabletProcess.ServingStatus = "NOT_SERVING"
+	err = tablets[primary].VttabletProcess.Setup()
+	require.NoError(t, err)
+	stateChanged <- true
+
+	// Wait for the action triggering the VT15001 to be done before moving on
+	<-actionDone
+
+	utils.StopTablet(t, tablets[primary], false)
+	tablets[primary].VttabletProcess.ServingStatus = "SERVING"
+	err = tablets[primary].VttabletProcess.Setup()
+	require.NoError(t, err)
+	primary = prsTo
+}
+
+func TestErrorsInTransaction(t *testing.T) {
+	clusterInstance := utils.SetupShardedReparentCluster(t, policy.DurabilitySemiSync, []string{
+		"--queryserver-config-transaction-timeout", "5m",
+		"--queryserver-config-query-timeout", "5m",
+	})
+
+	defer utils.TeardownCluster(clusterInstance)
+
+	keyspace := clusterInstance.Keyspaces[0]
+	vtParams := clusterInstance.GetVTParams(keyspace.Name)
+	tablets := clusterInstance.Keyspaces[0].Shards[1].Vttablets
+
+	primary = 0
+
+	// Start by reparenting all the shards to the first tablet.
+	// Confirm that the replication is setup correctly in the beginning.
+	// tablets[0] is the primary tablet in the beginning.
+	utils.ConfirmReplication(t, tablets[primary], []*cluster.Vttablet{tablets[1], tablets[2]})
+
+	conn, err := mysql.Connect(context.Background(), &vtParams)
+	require.NoError(t, err)
+
+	_, err = conn.ExecuteFetch("delete from vt_insert_test", 0, false)
+	require.NoError(t, err)
+
+	t.Run("commit while reparenting", func(t *testing.T) {
+		testCommitError(t, conn, clusterInstance, tablets)
+	})
+
+	t.Run("execute DML while tablet is NOT_SERVING", func(t *testing.T) {
+		testExecuteErrorWhileTabletIsNotServing(t, conn, clusterInstance, tablets)
+	})
+
+	t.Run("execute DML while reparenting", func(t *testing.T) {
+		testExecuteError(t, conn, clusterInstance, tablets)
+	})
+}

go/test/endtoend/reparent/utils/utils.go

@@ -43,13 +43,14 @@ import (
 )
 
 var (
-	KeyspaceName = "ks"
-	dbName       = "vt_" + KeyspaceName
-	username     = "vt_dba"
-	Hostname     = "localhost"
-	insertVal    = 1
-	insertSQL    = "insert into vt_insert_test(id, msg) values (%d, 'test %d')"
-	sqlSchema    = `
+	KeyspaceName            = "ks"
+	dbName                  = "vt_" + KeyspaceName
+	username                = "vt_dba"
+	Hostname                = "localhost"
+	insertVal               = 1
+	insertSQL               = "insert into vt_insert_test(id, msg) values (%d, 'test %d')"
+	insertSQLMultipleValues = "insert into vt_insert_test(id, msg) values (%d, 'test %d'), (%d, 'test %d'), (%d, 'test %d'), (%d, 'test %d')"
+	sqlSchema               = `
 	create table vt_insert_test (
 	id bigint,
 	msg varchar(64),
@@ -76,7 +77,7 @@ func SetupRangeBasedCluster(ctx context.Context, t *testing.T) *cluster.LocalPro
 }
 
 // SetupShardedReparentCluster is used to setup a sharded cluster for testing
-func SetupShardedReparentCluster(t *testing.T, durability string) *cluster.LocalProcessCluster {
+func SetupShardedReparentCluster(t *testing.T, durability string, extraVttabletFlags []string) *cluster.LocalProcessCluster {
 	clusterInstance := cluster.NewCluster(cell1, Hostname)
 	// Start topo server
 	err := clusterInstance.StartTopo()
@@ -88,6 +89,11 @@ func SetupShardedReparentCluster(t *testing.T, durability string) *cluster.Local
 		"--health_check_interval", "1s",
 		"--track_schema_versions=true",
 		"--queryserver_enable_online_ddl=false")
+
+	if len(extraVttabletFlags) > 0 {
+		clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, extraVttabletFlags...)
+	}
+
 	clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs,
 		"--enable_buffer",
 		// Long timeout in case failover is slow.
@@ -117,6 +123,11 @@ func GetInsertQuery(idx int) string {
 	return fmt.Sprintf(insertSQL, idx, idx)
 }
 
+// GetInsertMultipleValuesQuery returns a built insert query to insert multiple rows at once.
+func GetInsertMultipleValuesQuery(idx1, idx2, idx3, idx4 int) string {
+	return fmt.Sprintf(insertSQLMultipleValues, idx1, idx1, idx2, idx2, idx3, idx3, idx4, idx4)
+}
+
 // GetSelectionQuery returns a built selection query read the data.
 func GetSelectionQuery() string {
 	return `select * from vt_insert_test`

go/test/endtoend/tabletgateway/vtgate_test.go

@@ -261,7 +261,7 @@ func TestReplicaTransactions(t *testing.T) {
 	_ = replicaTablet.VttabletProcess.TearDown()
 	// Healthcheck interval on tablet is set to 1s, so sleep for 2s
 	time.Sleep(2 * time.Second)
-	utils.AssertContainsError(t, readConn, fetchAllCustomers, "is either down or nonexistent")
+	utils.AssertContainsMultipleErrors(t, readConn, fetchAllCustomers, "VT15001", "is either down or nonexistent")
 
 	// bring up the tablet again
 	// trying to use the same session/transaction should fail as the vtgate has
@@ -271,7 +271,8 @@ func TestReplicaTransactions(t *testing.T) {
 	require.NoError(t, err)
 	serving := replicaTablet.VttabletProcess.WaitForStatus("SERVING", 60*time.Second)
 	assert.Equal(t, serving, true, "Tablet did not become ready within a reasonable time")
-	utils.AssertContainsError(t, readConn, fetchAllCustomers, "not found")
+	utils.AssertContainsError(t, readConn, fetchAllCustomers, "VT09032")
+	utils.Exec(t, readConn, "rollback")
 
 	// create a new connection, should be able to query again
 	readConn, err = mysql.Connect(ctx, &vtParams)

go/test/endtoend/utils/utils.go

@@ -118,6 +118,17 @@ func AssertContainsError(t *testing.T, conn *mysql.Conn, query, expected string)
 	assert.ErrorContains(t, err, expected, "actual error: %s", err.Error())
 }
 
+// AssertContainsMultipleErrors acts the same way as AssertContainsError, but it will assert that
+// multiple sub-strings are present in the error
+func AssertContainsMultipleErrors(t *testing.T, conn *mysql.Conn, query string, expected ...string) {
+	t.Helper()
+	_, err := ExecAllowError(t, conn, query)
+	require.Error(t, err)
+	for _, s := range expected {
+		assert.ErrorContains(t, err, s, "actual error: %s", err.Error())
+	}
+}
+
 // AssertMatchesNoOrder executes the given query and makes sure it matches the given `expected` string.
 // The order applied to the results or expectation is ignored. They are both re-sorted.
 func AssertMatchesNoOrder(t *testing.T, conn *mysql.Conn, query, expected string) {

go/vt/discovery/healthcheck.go

@@ -889,7 +889,6 @@ func (hc *HealthCheckImpl) TabletConnection(ctx context.Context, alias *topodata
 	thc := hc.healthByAlias[tabletAliasString(topoproto.TabletAliasString(alias))]
 	hc.mu.Unlock()
 	if thc == nil || thc.Conn == nil {
-		// TODO: test that throws this error
 		return nil, vterrors.Errorf(vtrpc.Code_NOT_FOUND, "tablet: %v is either down or nonexistent", alias)
 	}
 	return thc.Connection(ctx), nil