From 5e86c2411f778e0f0f56f3ba5663d75adbed1b2c Mon Sep 17 00:00:00 2001
From: rafsaf <rafal.safin12@gmail.com>
Date: Wed, 16 Aug 2023 18:27:55 +0200
Subject: [PATCH] finish UploadProviderAWS  working implementation

---
 backuper/config.py                        |  1 +
 backuper/main.py                          |  2 +-
 backuper/models/upload_provider_models.py |  9 +++
 backuper/upload_providers/__init__.py     |  8 +-
 backuper/upload_providers/aws_s3.py       | 93 +++++++++++++++++++++++
 5 files changed, 111 insertions(+), 2 deletions(-)
 create mode 100644 backuper/upload_providers/aws_s3.py

diff --git a/backuper/config.py b/backuper/config.py
index 3471753..25d0d68 100644
--- a/backuper/config.py
+++ b/backuper/config.py
@@ -18,6 +18,7 @@
 class UploadProviderEnum(StrEnum):
     LOCAL_FILES_DEBUG = "debug"
     GOOGLE_CLOUD_STORAGE = "gcs"
+    AWS_S3 = "aws"
 
 
 class BackupTargetEnum(StrEnum):
diff --git a/backuper/main.py b/backuper/main.py
index 40577f9..37d24fd 100644
--- a/backuper/main.py
+++ b/backuper/main.py
@@ -9,7 +9,7 @@
 from typing import NoReturn
 
 from backuper import config, core
-from backuper.backup_targets.base_target import BaseBackupTarget
+from backuper.backup_targets import BaseBackupTarget
 from backuper.notifications import PROGRAM_STEP, NotificationsContext
 from backuper.upload_providers import BaseUploadProvider
 
diff --git a/backuper/models/upload_provider_models.py b/backuper/models/upload_provider_models.py
index a0c7b19..d069593 100644
--- a/backuper/models/upload_provider_models.py
+++ b/backuper/models/upload_provider_models.py
@@ -24,3 +24,12 @@ class GCSProviderModel(ProviderModel):
     def process_service_account_base64(cls, service_account_base64: str) -> str:
         base64.b64decode(service_account_base64)
         return service_account_base64
+
+
+class AWSProviderModel(ProviderModel):
+    bucket_name: str
+    bucket_upload_path: str
+    key_id: str
+    key_secret: str
+    region: str
+    max_bandwidth: int | None = None
diff --git a/backuper/upload_providers/__init__.py b/backuper/upload_providers/__init__.py
index ad9dc29..cf669d2 100644
--- a/backuper/upload_providers/__init__.py
+++ b/backuper/upload_providers/__init__.py
@@ -1,5 +1,11 @@
 from .base_provider import BaseUploadProvider
 from .google_cloud_storage import UploadProviderGCS
 from .debug import UploadProviderLocalDebug
+from .aws_s3 import UploadProviderAWS
 
-__all__ = ["BaseUploadProvider", "UploadProviderGCS", "UploadProviderLocalDebug"]
+__all__ = [
+    "BaseUploadProvider",
+    "UploadProviderAWS",
+    "UploadProviderGCS",
+    "UploadProviderLocalDebug",
+]
diff --git a/backuper/upload_providers/aws_s3.py b/backuper/upload_providers/aws_s3.py
new file mode 100644
index 0000000..2040b5b
--- /dev/null
+++ b/backuper/upload_providers/aws_s3.py
@@ -0,0 +1,93 @@
+import logging
+from pathlib import Path
+
+import boto3
+from typing import Any, TypedDict
+from backuper import config, core
+from backuper.upload_providers.base_provider import BaseUploadProvider
+from boto3.s3.transfer import TransferConfig
+
+log = logging.getLogger(__name__)
+
+
+class DeleteItemDict(TypedDict):
+    Key: str
+
+
+class UploadProviderAWS(
+    BaseUploadProvider,
+    name=config.UploadProviderEnum.AWS_S3,
+):
+    """AWS S3 bucket for storing backups"""
+
+    def __init__(
+        self,
+        bucket_name: str,
+        bucket_upload_path: str,
+        key_id: str,
+        key_secret: str,
+        region: str,
+        max_bandwidth: int | None,
+        **kwargs: str,
+    ) -> None:
+        self.bucket_upload_path = bucket_upload_path
+        self.max_bandwidth = max_bandwidth
+
+        s3: Any = boto3.resource(
+            "s3",
+            region_name=region,
+            aws_access_key_id=key_id,
+            aws_secret_access_key=key_secret,
+        )
+
+        self.bucket = s3.Bucket(bucket_name)
+        self.transfer_config = TransferConfig(max_bandwidth=self.max_bandwidth)
+
+    def _post_save(self, backup_file: Path) -> str:
+        zip_backup_file = core.run_create_zip_archive(backup_file=backup_file)
+
+        backup_dest_in_bucket = "{}/{}/{}".format(
+            self.bucket_upload_path,
+            zip_backup_file.parent.name,
+            zip_backup_file.name,
+        )
+
+        log.info("start uploading %s to %s", zip_backup_file, backup_dest_in_bucket)
+
+        self.bucket.upload_file(
+            Filename=zip_backup_file,
+            Key=backup_dest_in_bucket,
+            Config=self.transfer_config,
+        )
+
+        log.info("uploaded %s to %s", zip_backup_file, backup_dest_in_bucket)
+        return backup_dest_in_bucket
+
+    def _clean(self, backup_file: Path, max_backups: int) -> None:
+        for backup_path in backup_file.parent.iterdir():
+            core.remove_path(backup_path)
+            log.info("removed %s from local disk", backup_path)
+
+        backup_list_cloud: list[str] = []
+        prefix = f"{self.bucket_upload_path}/{backup_file.parent.name}/"
+        for bucket_obj in self.bucket.objects.filter(Delimiter="/", Prefix=prefix):
+            backup_list_cloud.append(bucket_obj.key)
+
+        # remove oldest
+        backup_list_cloud.sort(reverse=True)
+        items_to_delete: list[DeleteItemDict] = []
+
+        while len(backup_list_cloud) > max_backups:
+            backup_to_remove = backup_list_cloud.pop()
+            items_to_delete.append({"Key": backup_to_remove})
+            log.info("backup %s will be deleted from aws s3 bucket", backup_to_remove)
+
+        if items_to_delete:
+            delete_response = self.bucket.delete_objects(
+                Delete={"Objects": items_to_delete, "Quiet": False}
+            )
+            if "Errors" in delete_response and delete_response["Errors"]:
+                raise RuntimeError(
+                    "Fail to delete backups from aws s3: %s", delete_response["Errors"]
+                )
+            log.info("backups were successfully deleted from aws s3 bucket")