Merge pull request #55 from rafsaf/gcs-chunk-size

Make GCS chunk size lower and timeout 120s to support worse network s…
rafsaf · Jun 18, 2023 · 9535c27 · 9535c27
2 parents e1e5548 + 1d276cc
commit 9535c27
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 4 deletions.
diff --git a/backuper/storage_providers/google_cloud_storage.py b/backuper/storage_providers/google_cloud_storage.py
@@ -17,6 +17,7 @@ class GoogleCloudStorage(base_provider.BaseBackupProvider):
 
     NAME = config.BackupProviderEnum.GOOGLE_CLOUD_STORAGE
     MAX_UPLOAD_RETRY = 5
+    CHUNK_SIZE = 25 * 1024 * 1024  # 25MB
 
     def __init__(self) -> None:
         service_account_bytes = base64.b64decode(config.GOOGLE_SERVICE_ACCOUNT_BASE64)
@@ -37,11 +38,13 @@ def _post_save(self, backup_file: Path) -> str:
 
         log.info("start uploading %s to %s", zip_backup_file, backup_dest_in_bucket)
 
-        blob = self.bucket.blob(backup_dest_in_bucket)
+        blob = self.bucket.blob(
+            backup_dest_in_bucket, chunk_size=GoogleCloudStorage.CHUNK_SIZE
+        )
         retry = 0
         while retry < self.MAX_UPLOAD_RETRY:
             try:
-                blob.upload_from_filename(zip_backup_file)
+                blob.upload_from_filename(zip_backup_file, timeout=120)
                 break
             except Exception as err:
                 log.error(

diff --git a/tests/test_storage_provider_gcs.py b/tests/test_storage_provider_gcs.py
@@ -82,9 +82,12 @@ def test_gcs_post_save_with_google_bucket_upload_path(
         == "test123/fake_env_name/fake_backup.zip"
     )
     assert fake_backup_file_zip_path.exists()
-    bucket_mock.blob.assert_called_once_with("test123/fake_env_name/fake_backup.zip")
+    bucket_mock.blob.assert_called_once_with(
+        "test123/fake_env_name/fake_backup.zip",
+        chunk_size=GoogleCloudStorage.CHUNK_SIZE,
+    )
     single_blob_mock.upload_from_filename.assert_called_once_with(
-        fake_backup_file_zip_path
+        fake_backup_file_zip_path, timeout=120
     )