Merge pull request #92 from rafsaf/3.0-release

3.0 release, improve gcs provider, rename it to upload provider, fix bug in cleanup after backup, refactor notifications, improve overall docs

README.md

@@ -20,6 +20,7 @@ A tool for performing scheduled database backups and transferring encrypted data
 ## Supported upload providers
 
 - Google Cloud Storage bucket
+- Debug (local)
 
 ## Notifications
 
@@ -58,7 +59,7 @@ services:
 
 ```
 
-(NOTE this will use provider [debug](https://backuper.rafsaf.pl/providers/debug/) that store backups locally in the container.
+(NOTE this will use provider [debug](https://backuper.rafsaf.pl/providers/debug/) that store backups locally in the container).
 
 
 <br>

backuper/backup_targets/base_target.py

@@ -14,11 +14,12 @@
 class BaseBackupTarget(ABC):
     NAME: config.BackupTargetEnum
 
-    def __init__(self, cron_rule: str, env_name: str) -> None:
+    def __init__(self, cron_rule: str, env_name: str, max_backups: int) -> None:
         self.cron_rule: str = cron_rule
-        self.env_name = env_name
-        self.last_backup_time = datetime.utcnow()
-        self.next_backup_time = self._get_next_backup_time()
+        self.env_name: str = env_name
+        self.max_backups: int = max_backups
+        self.last_backup_time: datetime = datetime.utcnow()
+        self.next_backup_time: datetime = self._get_next_backup_time()
         log.info(
             "first calculated backup of target `%s` will be: %s",
             env_name,

backuper/backup_targets/file.py

@@ -13,11 +13,14 @@ def __init__(
         abs_path: Path,
         cron_rule: str,
         env_name: str,
+        max_backups: int,
         **kwargs: str | int,
     ) -> None:
         self.cron_rule: str = cron_rule
         self.file: Path = abs_path
-        super().__init__(cron_rule=cron_rule, env_name=env_name)
+        super().__init__(
+            cron_rule=cron_rule, env_name=env_name, max_backups=max_backups
+        )
 
     def _backup(self) -> Path:
         out_file = core.get_new_backup_path(self.env_name, self.file.name)

backuper/backup_targets/folder.py

@@ -13,11 +13,14 @@ def __init__(
         abs_path: Path,
         cron_rule: str,
         env_name: str,
+        max_backups: int,
         **kwargs: str | int,
     ) -> None:
         self.cron_rule: str = cron_rule
         self.folder: Path = abs_path
-        super().__init__(cron_rule=cron_rule, env_name=env_name)
+        super().__init__(
+            cron_rule=cron_rule, env_name=env_name, max_backups=max_backups
+        )
 
     def _backup(self) -> Path:
         out_file = core.get_new_backup_path(self.env_name, self.folder.name)

backuper/backup_targets/mariadb.py

@@ -27,9 +27,12 @@ def __init__(
         db: str,
         cron_rule: str,
         env_name: str,
+        max_backups: int,
         **kwargs: str | int,
     ) -> None:
-        super().__init__(cron_rule=cron_rule, env_name=env_name)
+        super().__init__(
+            cron_rule=cron_rule, env_name=env_name, max_backups=max_backups
+        )
         self.cron_rule: str = cron_rule
         self.user: str = user
         self.db: str = db

backuper/backup_targets/mysql.py

@@ -27,9 +27,12 @@ def __init__(
         db: str,
         cron_rule: str,
         env_name: str,
+        max_backups: int,
         **kwargs: str | int,
     ) -> None:
-        super().__init__(cron_rule=cron_rule, env_name=env_name)
+        super().__init__(
+            cron_rule=cron_rule, env_name=env_name, max_backups=max_backups
+        )
         self.cron_rule: str = cron_rule
         self.user: str = user
         self.db: str = db

backuper/backup_targets/postgresql.py

@@ -29,9 +29,12 @@ def __init__(
         db: str,
         cron_rule: str,
         env_name: str,
+        max_backups: int,
         **kwargs: str | int,
     ) -> None:
-        super().__init__(cron_rule=cron_rule, env_name=env_name)
+        super().__init__(
+            cron_rule=cron_rule, env_name=env_name, max_backups=max_backups
+        )
         self.cron_rule: str = cron_rule
         self.user: str = user
         self.db: str = db

backuper/config.py

@@ -15,7 +15,7 @@
     pass
 
 
-class BackupProviderEnum(StrEnum):
+class UploadProviderEnum(StrEnum):
     LOCAL_FILES_DEBUG = "debug"
     GOOGLE_CLOUD_STORAGE = "gcs"
 
@@ -40,13 +40,13 @@ class BackupTargetEnum(StrEnum):
 ZIP_ARCHIVE_PASSWORD = os.environ.get("ZIP_ARCHIVE_PASSWORD", "")
 SUBPROCESS_TIMEOUT_SECS: int = int(os.environ.get("SUBPROCESS_TIMEOUT_SECS", 60 * 60))
 SIGTERM_TIMEOUT_SECS: float = float(os.environ.get("SIGTERM_TIMEOUT_SECS", 30))
-FAIL_NOTIFICATION_MAX_MSG_LEN: int = int(
-    os.environ.get("FAIL_NOTIFICATION_MAX_MSG_LEN", 1000)
-)
 ZIP_ARCHIVE_LEVEL: int = int(os.environ.get("ZIP_ARCHIVE_LEVEL", 3))
 BACKUP_MAX_NUMBER: int = int(os.environ.get("BACKUP_MAX_NUMBER", 7))
 DISCORD_SUCCESS_WEBHOOK_URL: str = os.environ.get("DISCORD_SUCCESS_WEBHOOK_URL", "")
 DISCORD_FAIL_WEBHOOK_URL: str = os.environ.get("DISCORD_FAIL_WEBHOOK_URL", "")
+DISCORD_NOTIFICATION_MAX_MSG_LEN: int = int(
+    os.environ.get("DISCORD_NOTIFICATION_MAX_MSG_LEN", 1500)
+)
 
 
 def logging_config(log_level: str) -> None:

backuper/core.py

@@ -12,8 +12,8 @@
 from pydantic import BaseModel
 
 from backuper import config
-from backuper.models.provider_models import ProviderModel
-from backuper.models.target_models import TargetModel
+from backuper.models.backup_target_models import TargetModel
+from backuper.models.upload_provider_models import ProviderModel
 
 log = logging.getLogger(__name__)
 
@@ -38,7 +38,7 @@ def run_subprocess(shell_args: str) -> str:
         log.error("run_subprocess failed with status %s", p.returncode)
         log.error("run_subprocess stdout: %s", p.stdout)
         log.error("run_subprocess stderr: %s", p.stderr)
-        raise CoreSubprocessError()
+        raise CoreSubprocessError(p.stderr)
 
     log.debug("run_subprocess finished with status %s", p.returncode)
     log.debug("run_subprocess stdout: %s", p.stdout)
@@ -141,9 +141,9 @@ def create_target_models() -> list[TargetModel]:
 
 
 def create_provider_model() -> ProviderModel:
-    target_map: dict[config.BackupProviderEnum, type[ProviderModel]] = {}
+    target_map: dict[config.UploadProviderEnum, type[ProviderModel]] = {}
     for target_model in ProviderModel.__subclasses__():
-        name = config.BackupProviderEnum(
+        name = config.UploadProviderEnum(
             target_model.__name__.lower().removesuffix("providermodel")
         )
         target_map[name] = target_model

backuper/main.py

@@ -11,7 +11,7 @@
 from backuper import config, core
 from backuper.backup_targets.base_target import BaseBackupTarget
 from backuper.notifications import PROGRAM_STEP, NotificationsContext
-from backuper.providers import BaseBackupProvider
+from backuper.upload_providers import BaseUploadProvider
 
 exit_event = threading.Event()
 log = logging.getLogger(__name__)
@@ -23,9 +23,9 @@ def quit(sig: int, frame: FrameType | None) -> None:
 
 
 @NotificationsContext(step_name=PROGRAM_STEP.SETUP_PROVIDER)
-def backup_provider() -> BaseBackupProvider:
-    backup_provider_map: dict[config.BackupProviderEnum, type[BaseBackupProvider]] = {}
-    for backup_provider in BaseBackupProvider.__subclasses__():
+def backup_provider() -> BaseUploadProvider:
+    backup_provider_map: dict[config.UploadProviderEnum, type[BaseUploadProvider]] = {}
+    for backup_provider in BaseUploadProvider.__subclasses__():
         backup_provider_map[backup_provider.NAME] = backup_provider  # type: ignore
 
     provider_model = core.create_provider_model()
@@ -115,7 +115,7 @@ def shutdown() -> NoReturn:
         sys.exit(1)
 
 
-def run_backup(target: BaseBackupTarget, provider: BaseBackupProvider) -> None:
+def run_backup(target: BaseBackupTarget, provider: BaseUploadProvider) -> None:
     log.info("start making backup of target: `%s`", target.env_name)
     with NotificationsContext(
         step_name=PROGRAM_STEP.BACKUP_CREATE, env_name=target.env_name
@@ -129,10 +129,16 @@ def run_backup(target: BaseBackupTarget, provider: BaseBackupProvider) -> None:
     with NotificationsContext(
         step_name=PROGRAM_STEP.UPLOAD,
         env_name=target.env_name,
-        send_on_success=True,
     ):
         provider.post_save(backup_file=backup_file)
 
+    with NotificationsContext(
+        step_name=PROGRAM_STEP.CLEANUP,
+        env_name=target.env_name,
+        send_on_success=True,
+    ):
+        provider.safe_clean(backup_file=backup_file, max_backups=target.max_backups)
+
     log.info(
         "backup and upload finished, next backup of target `%s` is: %s",
         target.env_name,

backuper/models/target_models.py → backuper/models/backup_target_models.py

@@ -18,7 +18,6 @@ class TargetModel(BaseModel):
     env_name: str
     cron_rule: str
     max_backups: int = config.BACKUP_MAX_NUMBER
-    archive_level: int = config.ZIP_ARCHIVE_LEVEL
 
     @field_validator("cron_rule")
     def cron_rule_is_valid(cls, cron_rule: str) -> str:

backuper/models/provider_models.py → backuper/models/upload_provider_models.py

@@ -6,7 +6,7 @@
 
 
 class ProviderModel(BaseModel):
-    name: config.BackupProviderEnum
+    name: config.UploadProviderEnum
 
 
 class DebugProviderModel(ProviderModel):
@@ -15,8 +15,10 @@ class DebugProviderModel(ProviderModel):
 
 class GCSProviderModel(ProviderModel):
     bucket_name: str
-    bucket_upload_path: str | None = None
+    bucket_upload_path: str
     service_account_base64: str
+    chunk_size_mb: int = 100
+    chunk_timeout_secs: int = 60
 
     @field_validator("service_account_base64")
     def process_service_account_base64(cls, service_account_base64: str) -> str:

backuper/notifications.py

@@ -16,16 +16,27 @@
 
 
 class PROGRAM_STEP(StrEnum):
-    SETUP_PROVIDER = "backup provider setup"
+    SETUP_PROVIDER = "upload provider setup"
     SETUP_TARGETS = "backup targets setup"
     BACKUP_CREATE = "backup create"
     UPLOAD = "upload to provider"
+    CLEANUP = "cleanup old backups"
 
 
 def _formated_now() -> str:
     return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S,%f UTC")
 
 
+def _limit_message(message: str, limit: int) -> str:
+    limit = max(100, limit)
+    if len(message) <= limit:
+        return message
+
+    truncate_text = f"...\n\n(truncated to {limit} chars)"
+    truncate_message = message[: limit - len(truncate_text)]
+    return truncate_message + truncate_text
+
+
 class NotificationsContext(ContextDecorator):
     def __init__(
         self,
@@ -44,29 +55,31 @@ def _success_message(self) -> str:
         )
         return message_to_send
 
-    def _fail_message(self, traceback: str) -> str:
+    def _fail_message(
+        self,
+        exc_type: type[BaseException],
+        exc_val: BaseException,
+        exc_traceback: TracebackType,
+    ) -> str:
         now = _formated_now()
-        msg = f"[FAIL] {now}\nSTEP: {self.step_name}\n"
+        msg = f"[FAIL] {now}\nStep: {self.step_name}\n"
         if self.env_name:
-            msg += f"TARGET: {self.env_name}\n"
-
-        traceback_length = len(traceback)
-        limit = config.FAIL_NOTIFICATION_MAX_MSG_LEN
-        if traceback_length < limit:
-            reason = traceback
-        else:
-            reason = f"{traceback[:limit]}...({traceback_length - limit} more chars)"
-        msg += f"REASON:\n```\n{reason}\n```"
+            msg += f"Target: {self.env_name}\n"
+        msg += f"Exception Type: {exc_type}\n"
+        msg += f"Exception Value: {exc_val}\n"
+
+        tb = "".join(traceback.format_exception(exc_type, exc_val, exc_traceback))
+        msg += f"\n{tb}\n"
         return msg
 
-    def _send_discord(self, message: str, webhook_url: str) -> None:
+    def _send_discord(self, message: str, webhook_url: str, limit_chars: int) -> None:
         if not webhook_url:
             log.debug("skip sending discord notification, no webhook url")
             return None
         try:
             discord_resp = requests.post(
                 webhook_url,
-                json={"content": message},
+                json={"content": _limit_message(message=message, limit=limit_chars)},
                 headers={"Content-Type": "application/json"},
                 timeout=5,
             )
@@ -84,17 +97,21 @@ def __exit__(
         exc_val: BaseException | None,
         exc_traceback: TracebackType | None,
     ) -> None:
-        if exc_type and exc_val:
-            tb = "".join(traceback.format_exception(None, exc_val, exc_traceback))
+        if exc_type and exc_val and exc_traceback:
             log.error("step %s failed, sending notifications", self.step_name)
-            fail_message = self._fail_message(traceback=tb)
+            fail_message = self._fail_message(
+                exc_type=exc_type, exc_val=exc_val, exc_traceback=exc_traceback
+            )
             log.debug("fail message: %s", fail_message)
             self._send_discord(
-                message=fail_message, webhook_url=config.DISCORD_FAIL_WEBHOOK_URL
+                message=fail_message,
+                webhook_url=config.DISCORD_FAIL_WEBHOOK_URL,
+                limit_chars=config.DISCORD_NOTIFICATION_MAX_MSG_LEN,
             )
         elif self.send_on_success:
             sucess_message = self._success_message()
             self._send_discord(
                 message=sucess_message,
                 webhook_url=config.DISCORD_SUCCESS_WEBHOOK_URL,
+                limit_chars=config.DISCORD_NOTIFICATION_MAX_MSG_LEN,
             )

backuper/upload_providers/__init__.py

@@ -0,0 +1,5 @@
+from .base_provider import BaseUploadProvider
+from .google_cloud_storage import UploadProviderGCS
+from .debug import UploadProviderLocalDebug
+
+__all__ = ["BaseUploadProvider", "UploadProviderGCS", "UploadProviderLocalDebug"]

backuper/providers/base_provider.py → backuper/upload_providers/base_provider.py

@@ -8,32 +8,33 @@
 log = logging.getLogger(__name__)
 
 
-class BaseBackupProvider(ABC):
-    NAME: config.BackupProviderEnum
+class BaseUploadProvider(ABC):
+    NAME: config.UploadProviderEnum
 
-    def __init_subclass__(cls, name: config.BackupProviderEnum) -> None:
+    def __init_subclass__(cls, name: config.UploadProviderEnum) -> None:
         cls.NAME = name
         super().__init_subclass__()
 
     @final
-    def post_save(self, backup_file: Path) -> str | None:
+    def post_save(self, backup_file: Path) -> str:
         try:
             return self._post_save(backup_file=backup_file)
         except Exception as err:
             log.error(err, exc_info=True)
             raise
 
     @final
-    def safe_clean(self, backup_file: Path) -> None:
+    def safe_clean(self, backup_file: Path, max_backups: int) -> None:
         try:
-            return self._clean(backup_file=backup_file)
-        except Exception as err:  # pragma: no cover
+            return self._clean(backup_file=backup_file, max_backups=max_backups)
+        except Exception as err:
             log.error(err, exc_info=True)
+            raise
 
     @abstractmethod
     def _post_save(self, backup_file: Path) -> str:  # pragma: no cover
         pass
 
     @abstractmethod
-    def _clean(self, backup_file: Path) -> None:  # pragma: no cover
+    def _clean(self, backup_file: Path, max_backups: int) -> None:  # pragma: no cover
         pass