VReplication: Improve permission check logic on external tablets on SwitchTraffic

go/flags/endtoend/vtbench.txt

@@ -65,10 +65,10 @@ Flags:
       --host string                                                 VTGate host(s) in the form 'host1,host2,...'
       --keep-logs duration                                          keep logs for this long (using ctime) (zero to keep forever)
       --keep-logs-by-mtime duration                                 keep logs for this long (using mtime) (zero to keep forever)
-      --log_backtrace_at traceLocations                             when logging hits line file:N, emit a stack trace
-      --log_dir string                                              If non-empty, write log files in this directory
       --log-err-stacks                                              log stack traces for errors
       --log-rotate-max-size uint                                    size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800)
+      --log_backtrace_at traceLocations                             when logging hits line file:N, emit a stack trace
+      --log_dir string                                              If non-empty, write log files in this directory
       --logtostderr                                                 log to standard error instead of files
       --mysql-server-version string                                 MySQL server version to advertise. (default "8.0.40-Vitess")
       --port int                                                    VTGate port

go/flags/endtoend/vtcombo.txt

@@ -406,8 +406,8 @@ Flags:
   -v, --version                                                          print binary version
       --vmodule vModuleFlag                                              comma-separated list of pattern=N settings for file-filtered logging
       --vreplication-copy-phase-duration duration                        Duration for each copy phase loop (before running the next catchup: default 1h) (default 1h0m0s)
-      --vreplication-copy-phase-max-innodb-history-list-length int       The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 1000000)
-      --vreplication-copy-phase-max-mysql-replication-lag int            The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 43200)
+      --vreplication-copy-phase-max-innodb-history-list-length int       The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet (default 10000000)
+      --vreplication-copy-phase-max-mysql-replication-lag int            The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet (default 43200)
       --vreplication-enable-http-log                                     Enable the /debug/vrlog HTTP endpoint, which will produce a log of the events replicated on primary tablets in the target keyspace by all VReplication workflows that are in the running/replicating phase.
       --vreplication-experimental-flags int                              (Bitmask) of experimental features in vreplication to enable (default 7)
       --vreplication-heartbeat-update-interval int                       Frequency (in seconds, default 1, max 60) at which the time_updated column of a vreplication stream when idling (default 1)

go/flags/endtoend/vttablet.txt

@@ -408,8 +408,8 @@ Flags:
   -v, --version                                                          print binary version
       --vmodule vModuleFlag                                              comma-separated list of pattern=N settings for file-filtered logging
       --vreplication-copy-phase-duration duration                        Duration for each copy phase loop (before running the next catchup: default 1h) (default 1h0m0s)
-      --vreplication-copy-phase-max-innodb-history-list-length int       The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 1000000)
-      --vreplication-copy-phase-max-mysql-replication-lag int            The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet. (default 43200)
+      --vreplication-copy-phase-max-innodb-history-list-length int       The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet (default 10000000)
+      --vreplication-copy-phase-max-mysql-replication-lag int            The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet (default 43200)
       --vreplication-enable-http-log                                     Enable the /debug/vrlog HTTP endpoint, which will produce a log of the events replicated on primary tablets in the target keyspace by all VReplication workflows that are in the running/replicating phase.
       --vreplication-experimental-flags int                              (Bitmask) of experimental features in vreplication to enable (default 7)
       --vreplication-heartbeat-update-interval int                       Frequency (in seconds, default 1, max 60) at which the time_updated column of a vreplication stream when idling (default 1)

go/test/endtoend/vreplication/vreplication_test.go

@@ -1762,6 +1762,7 @@ func testSwitchTrafficPermissionChecks(t *testing.T, workflowType, sourceKeyspac
 	applyPrivileges := func(query string) {
 		for _, shard := range sourceShards {
 			primary := vc.getPrimaryTablet(t, sourceKeyspace, shard)
+			log.Infof("Running permission query on %s: %s", primary.Name, query)
 			_, err := primary.QueryTablet(query, primary.Keyspace, false)
 			require.NoError(t, err)
 		}
@@ -1788,13 +1789,25 @@ func testSwitchTrafficPermissionChecks(t *testing.T, workflowType, sourceKeyspac
 				sidecarDBIdentifier))
 			runDryRunCmd(false)
 		})
-
 		t.Run("test without global or db level privileges", func(t *testing.T) {
 			applyPrivileges(fmt.Sprintf("revoke select,insert,update,delete on %s.* from vt_filtered@localhost",
 				sidecarDBIdentifier))
 			runDryRunCmd(true)
 		})
 
+		partialSidecarDBName := sidecarDBName[:len(sidecarDBName)-2] + "%"
+		partialSidecarDBIdentifier := sqlparser.String(sqlparser.NewIdentifierCS(partialSidecarDBName))
+		t.Run("test with db level privileges to partial sidecardb name", func(t *testing.T) {
+			applyPrivileges(fmt.Sprintf("grant select,insert,update,delete on %s.* to vt_filtered@localhost",
+				partialSidecarDBIdentifier))
+			runDryRunCmd(false)
+		})
+		t.Run("test after revoking partial sidecardb privs", func(t *testing.T) {
+			applyPrivileges(fmt.Sprintf("revoke select,insert,update,delete on %s.* from vt_filtered@localhost",
+				partialSidecarDBIdentifier))
+			runDryRunCmd(true)
+		})
+
 		t.Run("test with table level privileges", func(t *testing.T) {
 			applyPrivileges(fmt.Sprintf("grant select,insert,update,delete on %s.vreplication to vt_filtered@localhost",
 				sidecarDBIdentifier))

go/vt/vtctl/workflow/server.go

@@ -34,7 +34,6 @@ import (
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/encoding/prototext"
 
-	"vitess.io/vitess/go/constants/sidecar"
 	"vitess.io/vitess/go/protoutil"
 	"vitess.io/vitess/go/ptr"
 	"vitess.io/vitess/go/sets"
@@ -3653,8 +3652,7 @@ func (s *Server) validateShardsHaveVReplicationPermissions(ctx context.Context,
 			}
 			if !res.GetOk() {
 				return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION,
-					"user %s does not have the required set of permissions (select,insert,update,delete) on the %s.vreplication table on tablet %s",
-					res.GetUser(), sidecar.GetIdentifier(), topoproto.TabletAliasString(tablet.Alias))
+					"failed to validate required vreplication metadata permissions: %s", res.GetError())
 			}
 			return nil
 		})

go/vt/vtctl/workflow/server_test.go

@@ -2474,8 +2474,9 @@ func TestValidateShardsHaveVReplicationPermissions(t *testing.T) {
 			name: "no permissions",
 			response: &validateVReplicationPermissionsResponse{
 				res: &tabletmanagerdatapb.ValidateVReplicationPermissionsResponse{
-					User: "vt_test_user",
-					Ok:   false,
+					User:  "vt_test_user",
+					Ok:    false,
+					Error: "vt_test_user does not have the required set of permissions",
 				},
 			},
 			expectedErrContains: "vt_test_user does not have the required set of permissions",

go/vt/vttablet/tabletmanager/rpc_vreplication.go

@@ -75,23 +75,26 @@ const (
 	sqlGetVReplicationCopyStatus = "select distinct vrepl_id from %s.copy_state where vrepl_id = %d"
 	// Validate the minimum set of permissions needed to manage vreplication metadata.
 	// This is a simple check for a matching user rather than any specific user@host
-	// combination.
+	// combination. Also checks for wildcards. Note the, seemingly reverse check, `%a LIKE d.db`,
+	// which is required since %a replaces the actual sidecar db name and
+	// d.db is where a (potential) wildcard match is specified in a privilege grant.
 	sqlValidateVReplicationPermissions = `
 select count(*)>0 as good from mysql.user as u
   left join mysql.db as d on (u.user = d.user)
   left join mysql.tables_priv as t on (u.user = t.user)
 where u.user = %a
   and (
     (u.select_priv = 'y' and u.insert_priv = 'y' and u.update_priv = 'y' and u.delete_priv = 'y') /* user has global privs */
-    or (d.db = %a and d.select_priv = 'y' and d.insert_priv = 'y' and d.update_priv = 'y' and d.delete_priv = 'y') /* user has db privs */
-    or (t.db = %a and t.table_name = 'vreplication' /* user has table privs */
+    or (%a LIKE d.db escape '\\' and d.select_priv = 'y' and d.insert_priv = 'y' and d.update_priv = 'y' and d.delete_priv = 'y') /* user has db privs */
+    or (%a LIKE t.db escape '\\' and t.table_name = 'vreplication'
       and find_in_set('select', t.table_priv)
       and find_in_set('insert', t.table_priv)
       and find_in_set('update', t.table_priv)
       and find_in_set('delete', t.table_priv)
     )
   )
 limit 1
+
 `
 	sqlGetMaxSequenceVal   = "select max(%a) as maxval from %a.%a"
 	sqlInitSequenceTable   = "insert into %a.%a (id, next_id, cache) values (0, %d, 1000) on duplicate key update next_id = if(next_id < %d, %d, next_id)"
@@ -848,6 +851,7 @@ func (tm *TabletManager) ValidateVReplicationPermissions(ctx context.Context, re
 	if err != nil {
 		return nil, err
 	}
+	log.Infof("Validating VReplication permissions on %s using query %s", tm.tabletAlias, query)
 	conn, err := tm.MysqlDaemon.GetAllPrivsConnection(ctx)
 	if err != nil {
 		return nil, err
@@ -866,9 +870,16 @@ func (tm *TabletManager) ValidateVReplicationPermissions(ctx context.Context, re
 		return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected result for query %s: expected boolean-like value, got: %q",
 			query, qr.Rows[0][0].ToString())
 	}
+	var errorString string
+	if !val {
+		errorString = fmt.Sprintf("user %s does not have the required set of permissions (select,insert,update,delete) on the %s.vreplication table on tablet %s",
+			tm.DBConfigs.Filtered.User, sidecar.GetName(), topoproto.TabletAliasString(tm.tabletAlias))
+		log.Errorf("validateVReplicationPermissions returning error: %s. Permission query run was %s", errorString, query)
+	}
 	return &tabletmanagerdatapb.ValidateVReplicationPermissionsResponse{
-		User: tm.DBConfigs.Filtered.User,
-		Ok:   val,
+		User:  tm.DBConfigs.Filtered.User,
+		Ok:    val,
+		Error: errorString,
 	}, nil
 }
 

go/vt/vttablet/tabletserver/tabletenv/config.go

@@ -211,8 +211,8 @@ func registerTabletEnvFlags(fs *pflag.FlagSet) {
 	utils.SetFlagBoolVar(fs, &currentConfig.EnableOnlineDDL, "queryserver-enable-online-ddl", true, "Enable online DDL.")
 	fs.BoolVar(&currentConfig.SanitizeLogMessages, "sanitize_log_messages", false, "Remove potentially sensitive information in tablet INFO, WARNING, and ERROR log messages such as query parameters.")
 
-	utils.SetFlagInt64Var(fs, &currentConfig.RowStreamer.MaxInnoDBTrxHistLen, "vreplication-copy-phase-max-innodb-history-list-length", 1000000, "The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet.")
-	utils.SetFlagInt64Var(fs, &currentConfig.RowStreamer.MaxMySQLReplLagSecs, "vreplication-copy-phase-max-mysql-replication-lag", 43200, "The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet.")
+	utils.SetFlagInt64Var(fs, &currentConfig.RowStreamer.MaxInnoDBTrxHistLen, "vreplication-copy-phase-max-innodb-history-list-length", 10000000, "The maximum InnoDB transaction history that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet")
+	utils.SetFlagInt64Var(fs, &currentConfig.RowStreamer.MaxMySQLReplLagSecs, "vreplication-copy-phase-max-mysql-replication-lag", 43200, "The maximum MySQL replication lag (in seconds) that can exist on a vstreamer (source) before starting another round of copying rows. This helps to limit the impact on the source tablet")
 
 	fs.BoolVar(&currentConfig.EnableViews, "queryserver-enable-views", false, "Enable views support in vttablet.")
 
@@ -1112,7 +1112,7 @@ var defaultConfig = TabletConfig{
 	EnableTableGC:            true,
 
 	RowStreamer: RowStreamerConfig{
-		MaxInnoDBTrxHistLen: 1000000,
+		MaxInnoDBTrxHistLen: 10000000,
 		MaxMySQLReplLagSecs: 43200,
 	},
 

go/vt/vttablet/tabletserver/tabletenv/config_test.go

@@ -161,7 +161,7 @@ replicationTracker:
   heartbeatIntervalSeconds: 250ms
   mode: disable
 rowStreamer:
-  maxInnoDBTrxHistLen: 1000000
+  maxInnoDBTrxHistLen: 10000000
   maxMySQLReplLagSecs: 43200
 schemaChangeReloadTimeout: 30s
 schemaReloadIntervalSeconds: 30m0s

proto/tabletmanagerdata.proto

@@ -711,6 +711,8 @@ message ValidateVReplicationPermissionsResponse {
   // Does the user have the minimum privileges needed to manage
   // vreplication metadata.
   bool ok = 2;
+  // The error has to be passed from the tablet since the sidecar name is tablet-specific
+  string error = 3;
 }
 
 message VDiffRequest {