Initialize Tablet with super_read_only mode

.github/workflows/upgrade_downgrade_test_backups_e2e.yml

@@ -161,8 +161,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Run test with VTTablet at version N-1 and VTBackup at version N
@@ -181,9 +183,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet
+        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtbackup $PWD/bin/vtbackup
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtbackup --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml

@@ -164,8 +164,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Run test with VTTablet at version N+1 and VTBackup at version N
@@ -184,9 +186,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet
-        cp /tmp/vitess-build-current/bin/vtbackup $PWD/bin/vtbackup
-        cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
+        cp /tmp/vitess-build-other/bin/vtbackup $PWD/bin/vtbackup
+        cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vtbackup --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -234,8 +234,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again, they will automatically start restoring the last backup.
@@ -290,8 +292,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again and restoring the previous backup.

.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml

@@ -237,8 +237,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again, they will automatically start restoring the last backup.
@@ -293,8 +295,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again and restoring the next backup.

.github/workflows/upgrade_downgrade_test_query_serving_queries.yml

@@ -209,9 +209,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml

@@ -212,9 +212,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_query_serving_schema.yml

@@ -209,9 +209,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml

@@ -212,9 +212,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml

@@ -182,8 +182,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtctl --version
         vttablet --version
 

.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml

@@ -179,8 +179,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtctl --version
         vttablet --version
 

config/init_db.sql

@@ -1,5 +1,4 @@
-# This file is executed immediately after mysql_install_db,
-# to initialize a fresh data directory.
+# This file is executed immediately after initializing a fresh data directory.
 
 ###############################################################################
 # WARNING: This sql is *NOT* safe for production use,
@@ -11,6 +10,12 @@
 ###############################################################################
 # Equivalent of mysql_secure_installation
 ###############################################################################
+# We need to ensure that super_read_only is disabled so that we can execute
+# these commands. Note that disabling it does NOT disable read_only.
+# We save the current value so that we only re-enable it at the end if it was
+# enabled before.
+SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');
+SET GLOBAL super_read_only='OFF';
 
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
@@ -77,3 +82,9 @@ FLUSH PRIVILEGES;
 
 RESET SLAVE ALL;
 RESET MASTER;
+
+# custom sql is used to add custom scripts like creating users/passwords. We use it in our tests
+# {{custom_sql}}
+
+# We need to set super_read_only back to what it was before
+SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'ON');

config/mycnf/default.cnf

@@ -14,9 +14,6 @@ port = {{.MysqlPort}}
 secure-file-priv = {{.SecureFilePriv}}
 {{end}}
 
-# all db instances should start in read-only mode - once the db is started and
-# fully functional, we'll push it into read-write mode
-read-only
 server-id = {{.ServerID}}
 
 # all db instances should skip starting replication threads - that way we can do any

config/mycnf/mariadb10.cnf

@@ -39,3 +39,5 @@ slave_net_timeout = 60
 character_set_server = utf8
 collation_server = utf8_general_ci
 
+# All MariaDB instances should start in read-only mode
+read-only

config/mycnf/mysql57.cnf

@@ -32,3 +32,7 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 rpl_semi_sync_master_timeout = 1000000000000000000
 rpl_semi_sync_master_wait_no_slave = 1
 
+# In order to protect against any errand GTIDs we will start the mysql instance
+# in super-read-only mode.
+super-read-only
+

config/mycnf/mysql80.cnf

@@ -28,3 +28,7 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 loose_rpl_semi_sync_master_timeout = 1000000000000000000
 loose_rpl_semi_sync_master_wait_no_slave = 1
 
+# In order to protect against any errand GTIDs we will start the mysql instance
+# in super-read-only mode.
+super-read-only
+

config/mycnf/test-suite.cnf

@@ -23,3 +23,6 @@ sql_mode = STRICT_TRANS_TABLES
 
 # set a short heartbeat interval in order to detect failures quickly
 slave_net_timeout = 4
+# Disabling `super-read-only`. `test-suite` is mainly used for `vttestserver`. Since `vttestserver` uses a single MySQL for primary and replicas,
+# so it is not possible to run it with `super-read-only`. Therefore, we are disabling it.
+super-read-only = false

doc/releasenotes/17_0_0_summary.md

@@ -11,6 +11,9 @@
     - [Detailed backup and restore stats](#detailed-backup-and-restore-stats)
   - **[Deprecations and Deletions](#deprecations-and-deletions)**
     - [Deprecated Stats](#deprecated-stats)
+  - **[VTTablet](#vttablet)**
+    - [VTTablet: Initializing all replicas with super_read_only](#vttablet-initialization)
+    - [Deprecated Flags](#deprecated-flags)
 
 ## <a id="major-changes"/> Major Changes
 
@@ -197,4 +200,15 @@ These stats are deprecated in v17.
 | Deprecated stat | Supported alternatives |
 |-|-|
 | `backup_duration_seconds` | `BackupDurationNanoseconds` |
-| `restore_duration_seconds` | `RestoreDurationNanoseconds` |
+| `restore_duration_seconds` | `RestoreDurationNanoseconds` |
+### <a id="vttablet"/> VTTablet
+#### <a id="vttablet-initialization"/> Initializing all replicas with super_read_only
+In order to prevent SUPER privileged users like `root` or `vt_dba` from producing errant GTIDs on replicas, all the replica MySQL servers are initialized with the MySQL
+global variable `super_read_only` value set to `ON`. During failovers, we set `super_read_only` to `OFF` for the promoted primary tablet. This will allow the
+primary to accept writes. All of the shard's tablets, except the current primary, will still have their global variable `super_read_only` set to `ON`. This will make sure that apart from
+MySQL replication no other component, offline system or operator can write directly to a replica.
+
+Reference PR for this change is [PR #12206](https://github.com/vitessio/vitess/pull/12206)
+
+#### <a id="deprecated-flags"/> Deprecated Flags
+The flag `use_super_read_only` is deprecated and will be removed in a later release.

go/cmd/vtbackup/vtbackup.go

@@ -312,6 +312,19 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		if err := mysqld.ResetReplication(ctx); err != nil {
 			return fmt.Errorf("can't reset replication: %v", err)
 		}
+		// We need to switch off super_read_only before we create the database.
+		resetFunc, err := mysqld.SetSuperReadOnly(false)
+		if err != nil {
+			return fmt.Errorf("failed to disable super_read_only during backup: %v", err)
+		}
+		if resetFunc != nil {
+			defer func() {
+				err := resetFunc()
+				if err != nil {
+					log.Error("Failed to set super_read_only back to its original value during backup")
+				}
+			}()
+		}
 		cmd := mysqlctl.GenerateInitialBinlogEntry()
 		if err := mysqld.ExecuteSuperQueryList(ctx, []string{cmd}); err != nil {
 			return err
@@ -404,6 +417,8 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		return err
 	}
 
+	log.Infof("takeBackup: primary position is: %s", primaryPos.String())
+
 	// Remember the time when we fetched the primary position, not when we caught
 	// up to it, so the timestamp on our backup is honest (assuming we make it
 	// to the goal position).

go/flags/endtoend/vttablet.txt

@@ -348,7 +348,6 @@ Usage of vttablet:
       --tx_throttler_config string                                       The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2\nmax_replication_lag_sec: 10\ninitial_rate: 100\nmax_increase: 1\nemergency_decrease: 0.5\nmin_duration_between_increases_sec: 40\nmax_duration_between_increases_sec: 62\nmin_duration_between_decreases_sec: 20\nspread_backlog_across_sec: 20\nage_bad_rate_after_sec: 180\nbad_rate_increase: 0.1\nmax_rate_approach_threshold: 0.9\n")
       --tx_throttler_healthcheck_cells strings                           A comma-separated list of cells. Only tabletservers running in these cells will be monitored for replication lag by the transaction throttler.
       --unhealthy_threshold duration                                     replication lag after which a replica is considered unhealthy (default 2h0m0s)
-      --use_super_read_only                                              Set super_read_only flag when performing planned failover.
       --v Level                                                          log level for V logs
   -v, --version                                                          print binary version
       --vmodule moduleSpec                                               comma-separated list of pattern=N settings for file-filtered logging

go/mysql/fakesqldb/server.go

@@ -783,3 +783,40 @@ func (db *DB) MockQueriesForTable(table string, result *sqltypes.Result) {
 		cols...,
 	))
 }
+
+// GetRejectedQueryResult checks if we should reject the query.
+func (db *DB) GetRejectedQueryResult(key string) error {
+	if err, ok := db.rejectedData[key]; ok {
+		return err
+	}
+
+	return nil
+}
+
+// GetQueryResult checks for explicit queries add through AddQuery().
+func (db *DB) GetQueryResult(key string) *ExpectedResult {
+	result, ok := db.data[key]
+	if ok {
+		return result
+	}
+	return nil
+}
+
+// GetQueryPatternResult checks if a query matches any pattern previously added using AddQueryPattern().
+func (db *DB) GetQueryPatternResult(key string) (func(string), ExpectedResult, bool, error) {
+	for _, pat := range db.patternData {
+		if pat.expr.MatchString(key) {
+			userCallback, ok := db.queryPatternUserCallback[pat.expr]
+			if ok {
+				if pat.err != "" {
+					return userCallback, ExpectedResult{pat.result, nil}, true, fmt.Errorf(pat.err)
+				}
+				return userCallback, ExpectedResult{pat.result, nil}, true, nil
+			}
+
+			return nil, ExpectedResult{nil, nil}, false, nil
+		}
+	}
+
+	return nil, ExpectedResult{nil, nil}, false, nil
+}

go/test/endtoend/backup/pitr/backup_mysqlctld_pitr_test.go

@@ -138,9 +138,9 @@ func TestIncrementalBackupMysqlctld(t *testing.T) {
 			if tc.writeBeforeBackup {
 				backup.InsertRowOnPrimary(t, "")
 			}
-			// we wait for 1 second because backups ar ewritten to a directory named after the current timestamp,
-			// in 1 second resolution. We want to aoid two backups that have the same pathname. Realistically this
-			// is only ever a problem in this endtoend test, not in production.
+			// we wait for 1 second because backups are written to a directory named after the current timestamp,
+			// in 1 second resolution. We want to avoid two backups that have the same pathname. Realistically this
+			// is only ever a problem in this end-to-end test, not in production.
 			// Also, we gie the replica a chance to catch up.
 			time.Sleep(1100 * time.Millisecond)
 			waitForReplica(t)