Remove built-in decompression flag

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -120,13 +120,13 @@ jobs:
         #sudo apt-get update
         #sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
         ####
-        wget -c https://cdn.mysql.com/archives/mysql-8.0/mysql-common_8.0.29-1ubuntu20.04_amd64.deb \
-                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-core_8.0.29-1ubuntu20.04_amd64.deb \
-                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-plugins_8.0.29-1ubuntu20.04_amd64.deb \
-                https://cdn.mysql.com/archives/mysql-8.0/mysql-client_8.0.29-1ubuntu20.04_amd64.deb \
-                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server-core_8.0.29-1ubuntu20.04_amd64.deb \
-                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server_8.0.29-1ubuntu20.04_amd64.deb \
-                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client_8.0.29-1ubuntu20.04_amd64.deb
+        wget -c https://cdn.mysql.com/archives/mysql-8.0/mysql-common_8.0.28-1ubuntu20.04_amd64.deb \
+                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-core_8.0.28-1ubuntu20.04_amd64.deb \
+                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-plugins_8.0.28-1ubuntu20.04_amd64.deb \
+                https://cdn.mysql.com/archives/mysql-8.0/mysql-client_8.0.28-1ubuntu20.04_amd64.deb \
+                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server-core_8.0.28-1ubuntu20.04_amd64.deb \
+                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server_8.0.28-1ubuntu20.04_amd64.deb \
+                https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client_8.0.28-1ubuntu20.04_amd64.deb
         sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y ./mysql-*.deb
 
         # Install everything else we need, and configure

doc/releasenotes/15_0_0_summary.md

@@ -95,32 +95,46 @@ Please see the VDiff2 [documentation](https://vitess.io/docs/15.0/reference/vrep
 The new flag `--table-schema-only` skips column introspection. `GetSchema` only returns general schema analysis, and specifically it includes the `CREATE TABLE|VIEW` statement in the `schema` field.
 
 #### Support for additional compressors and decompressors during backup & restore
-Backup/Restore now allow you many more options for compression and decompression instead of relying on the default compressor(pgzip).
+Backup/Restore now allow you many more options for compression and decompression instead of relying on the default compressor(`pgzip`).
 There are some built-in compressors which you can use out-of-the-box. Users will need to evaluate which option works best for their
 use-case. Here are the flags that control this feature
 
-- --builtin-compressor
-- --builtin-decompressor
+- --compression-engine-name
 - --external-compressor
 - --external-decompressor
 - --external-compressor-extension
 - --compression-level
 
-builtin compressor as of today supports the following options
-- pgzip
+`--compression-engine-name` specifies the engine used for compression. It can have one of the following values
+
+- pgzip (Default)
 - pargzip
 - lz4
 - zstd
+- external
 
-If you want to use any of the builtin compressors, simply set one of the above values for `--builtin-compressor`. You don't need to set
-the `--builtin-decompressor` flag in this case as we infer it automatically from the MANIFEST file. The default value for
-`--builtin-decompressor` is  `auto`.
+where 'external' is set only when using a custom command or tool other than the ones that are already provided. 
+If you want to use any of the built-in compressors, simply set one of the above values for `--compression-engine-name`. The value
+specified in `--compression-engine-name` is saved in the backup MANIFEST, which is later read by the restore process to decide which
+engine to use for decompression. Default value for engine is 'pgzip'.
 
 If you would like to use a custom command or external tool for compression/decompression then you need to provide the full command with
 arguments to the `--external-compressor` and `--external-decompressor` flags. `--external-compressor-extension` flag also needs to be provided
-so that compressed files are created with the correct extension. There is no need to override `--builtin-compressor` and `--builtin-decompressor`
-when using an external compressor/decompressor. Please note that if you want the current behavior then you don't need to change anything
-in these flags. You can read more about backup & restore [here] (https://vitess.io/docs/15.0/user-guides/operating-vitess/backup-and-restore/).
+so that compressed files are created with the correct extension. If the external command is not using any of the built-in compression engines
+(i-e pgzip, pargzip, lz4 or zstd) then you need to set `--compression-engine-name` to value 'external'.
+
+Please note that if you want the current production behavior then you don't need to change any of these flags.
+You can read more about backup & restore [here] (https://vitess.io/docs/15.0/user-guides/operating-vitess/backup-and-restore/).
+
+If you decided to switch from an external compressor to one of the built-in supported compressors (i-e pgzip, pargzip, lz4 or zstd) at any point
+in the future, you will need to do it in two steps.
+
+- step #1, set `--external-compressor` and `--external-compressor-extension` flag values to empty and change `--compression-engine-name` to desired value.
+- Step #2, after at least one cycle of backup with new configuration, you can set `--external-decompressor` flag value to empty.
+
+The reason you cannot change all the values together is because the restore process will then have no way to find out which external decompressor
+should be used to process the previous backup. Please make sure you have thought out all possible scenarios for restore before transitioning from one
+compression engine to another.
 
 ### Online DDL changes
 

go/flags/endtoend/vtctld.txt

@@ -16,13 +16,12 @@ Usage of vtctld:
       --backup_storage_implementation string                             Which backup storage implementation to use for creating and restoring backups.
       --backup_storage_number_blocks int                                 if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, at once, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression (default 2)
       --binlog_player_protocol string                                    the protocol to download binlogs from a vttablet (default "grpc")
-      --builtin-compressor string                                        builtin compressor engine to use (default "pgzip")
-      --builtin-decompressor string                                      builtin decompressor engine to use (default "auto")
       --builtinbackup_mysqld_timeout duration                            how long to wait for mysqld to shutdown at the start of the backup (default 10m0s)
       --builtinbackup_progress duration                                  how often to send progress updates when backing up large files (default 5s)
       --catch-sigpipe                                                    catch and ignore SIGPIPE on stdout and stderr if specified
       --cell string                                                      cell to use
       --ceph_backup_storage_config string                                Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json")
+      --compression-engine-name string                                   compressor engine used for compression. (default "pgzip")
       --compression-level int                                            what level to pass to the compressor (default 1)
       --consolidator-stream-query-size int                               Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152)
       --consolidator-stream-total-size int                               Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728)

go/flags/endtoend/vtexplain.txt

@@ -9,11 +9,10 @@ Usage of vtexplain:
       --backup_storage_number_blocks int                                 if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, at once, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression (default 2)
       --batch-interval duration                                          Interval between logical time slots. (default 10ms)
       --binlog_player_protocol string                                    the protocol to download binlogs from a vttablet (default "grpc")
-      --builtin-compressor string                                        builtin compressor engine to use (default "pgzip")
-      --builtin-decompressor string                                      builtin decompressor engine to use (default "auto")
       --builtinbackup_mysqld_timeout duration                            how long to wait for mysqld to shutdown at the start of the backup (default 10m0s)
       --builtinbackup_progress duration                                  how often to send progress updates when backing up large files (default 5s)
       --catch-sigpipe                                                    catch and ignore SIGPIPE on stdout and stderr if specified
+      --compression-engine-name string                                   compressor engine used for compression. (default "pgzip")
       --compression-level int                                            what level to pass to the compressor (default 1)
       --consolidator-stream-query-size int                               Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152)
       --consolidator-stream-total-size int                               Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728)

go/flags/endtoend/vttablet.txt

@@ -27,12 +27,11 @@ Usage of vttablet:
       --binlog_ssl_key string                                            PITR restore parameter: Filename containing mTLS client private key for use in binlog server authentication.
       --binlog_ssl_server_name string                                    PITR restore parameter: TLS server name (common name) to verify against for the binlog server we are connecting to (If not set: use the hostname or IP supplied in -binlog_host).
       --binlog_user string                                               PITR restore parameter: username of binlog server.
-      --builtin-compressor string                                        builtin compressor engine to use (default "pgzip")
-      --builtin-decompressor string                                      builtin decompressor engine to use (default "auto")
       --builtinbackup_mysqld_timeout duration                            how long to wait for mysqld to shutdown at the start of the backup (default 10m0s)
       --builtinbackup_progress duration                                  how often to send progress updates when backing up large files (default 5s)
       --catch-sigpipe                                                    catch and ignore SIGPIPE on stdout and stderr if specified
       --ceph_backup_storage_config string                                Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json")
+      --compression-engine-name string                                   compressor engine used for compression. (default "pgzip")
       --compression-level int                                            what level to pass to the compressor (default 1)
       --consolidator-stream-query-size int                               Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152)
       --consolidator-stream-total-size int                               Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728)

go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go

@@ -32,15 +32,14 @@ func TestBackupMysqlctld(t *testing.T) {
 func TestBackupMysqlctldWithlz4Compression(t *testing.T) {
 	defer setDefaultCompressionFlag()
 	cDetails := &backup.CompressionDetails{
-		BuiltinCompressor: "lz4",
+		CompressorEngineName: "lz4",
 	}
 
 	backup.TestBackup(t, backup.Mysqlctld, "xbstream", 0, cDetails, []string{"TestReplicaBackup", "TestPrimaryBackup"})
 }
 
 func setDefaultCompressionFlag() {
-	*mysqlctl.BuiltinCompressor = "pgzip"
-	*mysqlctl.BuiltinDecompressor = "auto"
+	*mysqlctl.CompressionEngineName = "pgzip"
 	*mysqlctl.ExternalCompressorCmd = ""
 	*mysqlctl.ExternalCompressorExt = ""
 	*mysqlctl.ExternalDecompressorCmd = ""

go/test/endtoend/backup/vtbackup/backup_only_test.go

@@ -24,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	"vitess.io/vitess/go/vt/mysqlctl"
+
 	"github.com/stretchr/testify/require"
 
 	"vitess.io/vitess/go/test/endtoend/cluster"
@@ -135,6 +137,11 @@ func firstBackupTest(t *testing.T, tabletType string) {
 	require.Nil(t, err)
 	cluster.VerifyRowsInTablet(t, replica1, keyspaceName, 2)
 
+	// eventhough we change the value of compression it won't effect
+	// decompression since it gets its value from MANIFEST file, created
+	// as part of backup.
+	*mysqlctl.CompressionEngineName = "lz4"
+	defer func() { *mysqlctl.CompressionEngineName = "pgzip" }()
 	// now bring up the other replica, letting it restore from backup.
 	err = localCluster.VtctlclientProcess.InitTablet(replica2, cell, keyspaceName, hostname, shardName)
 	require.Nil(t, err)
@@ -160,7 +167,6 @@ func firstBackupTest(t *testing.T, tabletType string) {
 
 	removeBackups(t)
 	verifyBackupCount(t, shardKsName, 0)
-
 }
 
 func vtBackup(t *testing.T, initialBackup bool, restartBeforeBackup bool) {

go/test/endtoend/backup/vtctlbackup/backup_test.go

@@ -30,6 +30,7 @@ func TestBackupMain(t *testing.T) {
 func TestBackupMainWithZstdCompression(t *testing.T) {
 	defer setDefaultCompressionFlag()
 	cDetails := &CompressionDetails{
+		CompressorEngineName:    "zstd",
 		ExternalCompressorCmd:   "zstd",
 		ExternalCompressorExt:   ".zst",
 		ExternalDecompressorCmd: "zstd -d",
@@ -39,8 +40,7 @@ func TestBackupMainWithZstdCompression(t *testing.T) {
 }
 
 func setDefaultCompressionFlag() {
-	*mysqlctl.BuiltinCompressor = "pgzip"
-	*mysqlctl.BuiltinDecompressor = "auto"
+	*mysqlctl.CompressionEngineName = "pgzip"
 	*mysqlctl.ExternalCompressorCmd = ""
 	*mysqlctl.ExternalCompressorExt = ""
 	*mysqlctl.ExternalDecompressorCmd = ""