diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index dc116b4ce6d..44cb2a08462 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -3,8 +3,9 @@ Usage of vtbackup: --alsologtostderr log to standard error as well as files --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. + --azblob_backup_buffer_size int The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600) --azblob_backup_container_name string Azure Blob Container Name. - --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) + --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased -- a multiple of azblob_backup_buffer_size). (default 1) --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). --backup_engine_implementation string Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin") --backup_storage_block_size int if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000) diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index e653f32d8fb..75183e86843 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -3,8 +3,9 @@ Usage of vtctld: --alsologtostderr log to standard error as well as files --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. + --azblob_backup_buffer_size int The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600) --azblob_backup_container_name string Azure Blob Container Name. - --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) + --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased -- a multiple of azblob_backup_buffer_size). (default 1) --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). --backup_engine_implementation string Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin") --backup_storage_block_size int if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index ba53b392123..9b42ebb5644 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -4,8 +4,9 @@ Usage of vttablet: --app_pool_size int Size of the connection pool for app connections (default 40) --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. + --azblob_backup_buffer_size int The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service. (default 104857600) --azblob_backup_container_name string Azure Blob Container Name. - --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) + --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased -- a multiple of azblob_backup_buffer_size). (default 1) --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). --backup_engine_implementation string Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin") --backup_storage_block_size int if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000) diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index 2f382bdbfc6..7058745d6c6 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -73,6 +73,14 @@ var ( }, ) + azBlobBufferSize = viperutil.Configure( + configKey("buffer_size"), + viperutil.Options[int]{ + Default: 100 << (10 * 2), // 100 MiB + FlagName: "azblob_buffer_size", + }, + ) + azBlobParallelism = viperutil.Configure( configKey("parallelism"), viperutil.Options[int]{ @@ -91,7 +99,8 @@ func registerFlags(fs *pflag.FlagSet) { fs.String("azblob_backup_account_key_file", accountKeyFile.Default(), "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") fs.String("azblob_backup_container_name", containerName.Default(), "Azure Blob Container Name.") fs.String("azblob_backup_storage_root", storageRoot.Default(), "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').") - fs.Int("azblob_backup_parallelism", azBlobParallelism.Default(), "Azure Blob operation parallelism (requires extra memory when increased).") + fs.Int("azblob_backup_buffer_size", azBlobBufferSize.Default(), "The memory buffer size to use in bytes, per file or stripe, when streaming to Azure Blob Service.") + fs.Int("azblob_backup_parallelism", azBlobParallelism.Default(), "Azure Blob operation parallelism (requires extra memory when increased -- a multiple of azblob_backup_buffer_size).") viperutil.BindFlags(fs, accountName, accountKeyFile, containerName, storageRoot, azBlobParallelism) } @@ -248,7 +257,7 @@ func (bh *AZBlobBackupHandle) AddFile(ctx context.Context, filename string, file go func() { defer bh.waitGroup.Done() _, err := azblob.UploadStreamToBlockBlob(bh.ctx, reader, blockBlobURL, azblob.UploadStreamToBlockBlobOptions{ - BufferSize: azblob.BlockBlobMaxStageBlockBytes, + BufferSize: azBlobBufferSize.Get(), MaxBuffers: azBlobParallelism.Get(), }) if err != nil {