diff --git a/backuper/storage_providers/google_cloud_storage.py b/backuper/storage_providers/google_cloud_storage.py index 1bcde6d..f31baed 100644 --- a/backuper/storage_providers/google_cloud_storage.py +++ b/backuper/storage_providers/google_cloud_storage.py @@ -17,6 +17,7 @@ class GoogleCloudStorage(base_provider.BaseBackupProvider): NAME = config.BackupProviderEnum.GOOGLE_CLOUD_STORAGE MAX_UPLOAD_RETRY = 5 + CHUNK_SIZE = 25 * 1024 * 1024 # 25MB def __init__(self) -> None: service_account_bytes = base64.b64decode(config.GOOGLE_SERVICE_ACCOUNT_BASE64) @@ -37,11 +38,13 @@ def _post_save(self, backup_file: Path) -> str: log.info("start uploading %s to %s", zip_backup_file, backup_dest_in_bucket) - blob = self.bucket.blob(backup_dest_in_bucket) + blob = self.bucket.blob( + backup_dest_in_bucket, chunk_size=GoogleCloudStorage.CHUNK_SIZE + ) retry = 0 while retry < self.MAX_UPLOAD_RETRY: try: - blob.upload_from_filename(zip_backup_file) + blob.upload_from_filename(zip_backup_file, timeout=120) break except Exception as err: log.error( diff --git a/tests/test_storage_provider_gcs.py b/tests/test_storage_provider_gcs.py index 78e7c81..0ac7aa8 100644 --- a/tests/test_storage_provider_gcs.py +++ b/tests/test_storage_provider_gcs.py @@ -82,9 +82,12 @@ def test_gcs_post_save_with_google_bucket_upload_path( == "test123/fake_env_name/fake_backup.zip" ) assert fake_backup_file_zip_path.exists() - bucket_mock.blob.assert_called_once_with("test123/fake_env_name/fake_backup.zip") + bucket_mock.blob.assert_called_once_with( + "test123/fake_env_name/fake_backup.zip", + chunk_size=GoogleCloudStorage.CHUNK_SIZE, + ) single_blob_mock.upload_from_filename.assert_called_once_with( - fake_backup_file_zip_path + fake_backup_file_zip_path, timeout=120 )