Initialize Tablet with super_read_only mode

.github/workflows/upgrade_downgrade_test_backups_e2e.yml

@@ -161,8 +161,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Run test with VTTablet at version N-1 and VTBackup at version N
@@ -181,9 +183,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet
+        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtbackup $PWD/bin/vtbackup
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtbackup --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml

@@ -164,8 +164,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Run test with VTTablet at version N+1 and VTBackup at version N
@@ -184,9 +186,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet
-        cp /tmp/vitess-build-current/bin/vtbackup $PWD/bin/vtbackup
-        cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
+        cp /tmp/vitess-build-other/bin/vtbackup $PWD/bin/vtbackup
+        cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vtbackup --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -234,8 +234,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again, they will automatically start restoring the last backup.
@@ -290,8 +292,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again and restoring the previous backup.

.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml

@@ -237,8 +237,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again, they will automatically start restoring the last backup.
@@ -293,8 +295,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again and restoring the next backup.

.github/workflows/upgrade_downgrade_test_query_serving_queries.yml

@@ -209,9 +209,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml

@@ -212,9 +212,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_query_serving_schema.yml

@@ -209,9 +209,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml

@@ -212,9 +212,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml

@@ -182,8 +182,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtctl --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml

@@ -179,8 +179,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtctl --version
         vttablet --version
 

config/init_db.sql

@@ -11,6 +11,12 @@
 ###############################################################################
 # Equivalent of mysql_secure_installation
 ###############################################################################
+# We need to ensure that super_read_only is disabled so that we can execute
+# these commands. Note that disabling it does NOT disable read_only.
+# We save the current value so that we only re-enable it at the end if it was
+# enabled before.
+SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');
+SET GLOBAL super_read_only='OFF';
 
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
@@ -77,3 +83,9 @@ FLUSH PRIVILEGES;
 
 RESET SLAVE ALL;
 RESET MASTER;
+
+# custom sql is used to add custom scripts like creating users/passwords. We use it in our tests
+# add custom sql here
+
+# We need to set super_read_only back to what it was before
+SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'OFF');

config/init_testserver_db.sql

@@ -0,0 +1,91 @@
+# This file is for testing purpose only.
+# This file is executed immediately after mysql_install_db, to initialize a fresh data directory.
+# It is equivalent of init_db.sql. Given init_db.sql is for mysql which has super_read_only
+# related stuff therefore for testing purpose we avoid setting `super_read_only` during initialization.
+
+###############################################################################
+# WARNING: Any change to init_db.sql should gets reflected in this file as well.
+###############################################################################
+
+###############################################################################
+# WARNING: This sql is *NOT* safe for production use,
+#          as it contains default well-known users and passwords.
+#          Care should be taken to change these users and passwords
+#          for production.
+###############################################################################
+
+###############################################################################
+# Equivalent of mysql_secure_installation
+###############################################################################
+# We need to ensure that read_only is disabled so that we can execute
+# these commands.
+SET GLOBAL read_only='OFF';
+
+# Changes during the init db should not make it to the binlog.
+# They could potentially create errant transactions on replicas.
+SET sql_log_bin = 0;
+# Remove anonymous users.
+DELETE FROM mysql.user WHERE User = '';
+
+# Disable remote root access (only allow UNIX socket).
+DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+
+# Remove test database.
+DROP DATABASE IF EXISTS test;
+
+###############################################################################
+# Vitess defaults
+###############################################################################
+
+# Admin user with all privileges.
+CREATE USER 'vt_dba'@'localhost';
+GRANT ALL ON *.* TO 'vt_dba'@'localhost';
+GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost';
+
+# User for app traffic, with global read-write access.
+CREATE USER 'vt_app'@'localhost';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+    REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION CLIENT, CREATE VIEW,
+    SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_app'@'localhost';
+
+# User for app debug traffic, with global read access.
+CREATE USER 'vt_appdebug'@'localhost';
+GRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost';
+
+# User for administrative operations that need to be executed as non-SUPER.
+# Same permissions as vt_app here.
+CREATE USER 'vt_allprivs'@'localhost';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+    REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+    SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_allprivs'@'localhost';
+
+# User for slave replication connections.
+CREATE USER 'vt_repl'@'%';
+GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';
+
+# User for Vitess VReplication (base vstreamers and vplayer).
+CREATE USER 'vt_filtered'@'localhost';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
+    REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
+  LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
+    SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
+  ON *.* TO 'vt_filtered'@'localhost';
+
+# User for general MySQL monitoring.
+CREATE USER 'vt_monitoring'@'localhost';
+GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
+      ON *.* TO 'vt_monitoring'@'localhost';
+GRANT SELECT, UPDATE, DELETE, DROP
+      ON performance_schema.* TO 'vt_monitoring'@'localhost';
+
+FLUSH PRIVILEGES;
+
+RESET SLAVE ALL;
+RESET MASTER;
+
+# custom sql is used to add custom scripts like creating users/passwords. We use it in our tests
+# add custom sql here

config/mycnf/mysql57.cnf

@@ -32,3 +32,7 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 rpl_semi_sync_master_timeout = 1000000000000000000
 rpl_semi_sync_master_wait_no_slave = 1
 
+# In order to protect against any errand GTIDs we will start the mysql instance
+# in super-read-only mode.
+super-read-only
+

config/mycnf/mysql80.cnf

@@ -28,3 +28,7 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 loose_rpl_semi_sync_master_timeout = 1000000000000000000
 loose_rpl_semi_sync_master_wait_no_slave = 1
 
+# In order to protect against any errand GTIDs we will start the mysql instance
+# in super-read-only mode.
+super-read-only
+

config/mycnf/test-suite.cnf

@@ -23,3 +23,4 @@ sql_mode = STRICT_TRANS_TABLES
 
 # set a short heartbeat interval in order to detect failures quickly
 slave_net_timeout = 4
+super-read-only = false

doc/releasenotes/17_0_0_summary.md

@@ -0,0 +1,17 @@
+## Summary
+
+- [New command line flags and behavior](#new-command-line-flags-and-behavior)
+
+## Known Issues
+
+## Major Changes
+
+### New command line flags and behavior
+
+#### VTTablet: Initializing all replica DB with super_read_only
+In order to prevent SUPER privileged users like `root` or `vt_dba` to produce errant GTIDs anywhere anytime, all the replica DBs are initialized with the Mysql 
+global variable `super_read_only` value set to `ON`. During re-parenting, we set `super_read_only` to `OFF` for the promoted primary tablet. This will allow the 
+primary to accept writes. All replica except the primary will still have their global variable `super_read_only` set to `ON`. This will make sure that apart from
+the replication no other component or offline system can mutate replica DB resulting in errant GTIDs that are then lying in wait to cause later failures.
+
+Reference PR for this change is [PR #12206](https://github.com/vitessio/vitess/pull/12206)

go/cmd/vtbackup/vtbackup.go

@@ -294,6 +294,7 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 	}
 	// In initial_backup mode, just take a backup of this empty database.
 	if initialBackup {
+		log.Infof("Inside initialBackup creating initial binlog entry.")
 		// Take a backup of this empty DB without restoring anything.
 		// First, initialize it the way InitShardPrimary would, so this backup
 		// produces a result that can be used to skip InitShardPrimary entirely.
@@ -302,6 +303,19 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		if err := mysqld.ResetReplication(ctx); err != nil {
 			return fmt.Errorf("can't reset replication: %v", err)
 		}
+		// We need to switch off super-read-only before we create database.
+		resetFunc, err := mysqld.SetSuperReadOnly(false)
+		if err != nil {
+			return fmt.Errorf("can't turn-off super-read-only during backup: %v", err)
+		}
+		if resetFunc != nil {
+			defer func() {
+				err := resetFunc()
+				if err != nil {
+					log.Info("not able to set super_read_only to its original value during backup")
+				}
+			}()
+		}
 		cmd := mysqlctl.GenerateInitialBinlogEntry()
 		if err := mysqld.ExecuteSuperQueryList(ctx, []string{cmd}); err != nil {
 			return err
@@ -390,6 +404,8 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		return err
 	}
 
+	log.Infof("takeBackup: primary position is: %s", primaryPos.String())
+
 	// Remember the time when we fetched the primary position, not when we caught
 	// up to it, so the timestamp on our backup is honest (assuming we make it
 	// to the goal position).

go/flags/endtoend/vttablet.txt

@@ -345,7 +345,7 @@ Usage of vttablet:
       --tx_throttler_config string                                       The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2\nmax_replication_lag_sec: 10\ninitial_rate: 100\nmax_increase: 1\nemergency_decrease: 0.5\nmin_duration_between_increases_sec: 40\nmax_duration_between_increases_sec: 62\nmin_duration_between_decreases_sec: 20\nspread_backlog_across_sec: 20\nage_bad_rate_after_sec: 180\nbad_rate_increase: 0.1\nmax_rate_approach_threshold: 0.9\n")
       --tx_throttler_healthcheck_cells strings                           A comma-separated list of cells. Only tabletservers running in these cells will be monitored for replication lag by the transaction throttler.
       --unhealthy_threshold duration                                     replication lag after which a replica is considered unhealthy (default 2h0m0s)
-      --use_super_read_only                                              Set super_read_only flag when performing planned failover.
+      --use_super_read_only                                              Set super_read_only flag when performing planned failover. (default true)
       --v Level                                                          log level for V logs
   -v, --version                                                          print binary version
       --vmodule moduleSpec                                               comma-separated list of pattern=N settings for file-filtered logging

go/mysql/collations/integration/helpers_test.go

@@ -137,7 +137,7 @@ func verifyWeightString(t *testing.T, local collations.Collation, remote *remote
 }
 
 func exec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
-	res, err := conn.ExecuteFetch(query, -1, true)
+	res, err := conn.ExecuteFetchWithSuperReadOnlyHandling(query, -1, true)
 	require.NoError(t, err, "failed to execute %q: %v", query, err)
 
 	return res