Update debug provider

ogion/backup_targets/file.py

@@ -23,7 +23,7 @@ def backup(self) -> Path:
 
         out_file = core.get_new_backup_path(self.env_name, escaped_filename)
 
-        shell_create_file_symlink = f"ln -s {self.target_model.abs_path} {out_file}"
+        shell_create_file_symlink = f"cp {self.target_model.abs_path} {out_file}"
         log.debug("start ln in subprocess: %s", shell_create_file_symlink)
         core.run_subprocess(shell_create_file_symlink)
         log.debug("finished ln, output: %s", out_file)

ogion/config.py

@@ -17,9 +17,11 @@
 CONST_BACKUP_FOLDER_PATH: Path = CONST_BASE_DIR / "data"
 CONST_CONFIG_FOLDER_PATH: Path = CONST_BASE_DIR / "conf"
 CONST_DOWNLOADS_FOLDER_PATH: Path = CONST_BACKUP_FOLDER_PATH / "downloads"
+CONST_DEBUG_FOLDER_PATH: Path = CONST_BACKUP_FOLDER_PATH / "debug"
 CONST_BACKUP_FOLDER_PATH.mkdir(mode=0o700, parents=True, exist_ok=True)
 CONST_CONFIG_FOLDER_PATH.mkdir(mode=0o700, parents=True, exist_ok=True)
 CONST_DOWNLOADS_FOLDER_PATH.mkdir(mode=0o700, exist_ok=True)
+CONST_DEBUG_FOLDER_PATH.mkdir(mode=0o700, exist_ok=True)
 
 
 try:
@@ -49,6 +51,7 @@ class Settings(BaseSettings):
     LOG_LEVEL: _log_levels = "INFO"
     BACKUP_PROVIDER: str
     AGE_RECIPIENTS: str
+    DEBUG_AGE_SECRET_KEY: str = ""
     INSTANCE_NAME: str = socket.gethostname()
     CPU_ARCH: Literal["amd64", "arm64"] = Field(
         default="amd64", alias_priority=2, alias="OGION_CPU_ARCHITECTURE"

ogion/core.py

@@ -6,7 +6,6 @@
 import os
 import re
 import secrets
-import shutil
 import subprocess
 import tempfile
 from datetime import UTC, datetime, timedelta
@@ -55,10 +54,7 @@ def run_subprocess(shell_args: str) -> str:
 
 def remove_path(path: Path) -> None:
     if path.exists():
-        if path.is_file() or path.is_symlink():
-            path.unlink()
-        else:
-            shutil.rmtree(path=path)
+        path.unlink()
 
 
 def get_new_backup_path(env_name: str, name: str) -> Path:
@@ -73,13 +69,13 @@ def get_new_backup_path(env_name: str, name: str) -> Path:
     return base_dir_path / new_file
 
 
-def run_decrypt_age_archive(backup_file: Path, debug_secret: str | None = None) -> Path:
+def run_decrypt_age_archive(backup_file: Path) -> Path:
     log.info("start age decrypt archive in subprocess: %s", backup_file)
 
     out = Path(str(backup_file).removesuffix(".age"))
 
-    if debug_secret:
-        secret = debug_secret
+    if config.options.DEBUG_AGE_SECRET_KEY:
+        secret = config.options.DEBUG_AGE_SECRET_KEY
     else:  # pragma: no cover
         secret = input("please input age private key to decrypt\n")
 

ogion/main.py

@@ -268,7 +268,8 @@ def run_list_backup_files(target_name: str) -> NoReturn:
         for i in backups:
             print(i)
         sys.exit(0)
-    log.warning("target '%s' does not exist")
+    log.warning("target '%s' does not exist", target_name)
+    print(f"target '{target_name}' does not exist")
     sys.exit(1)
 
 
@@ -282,13 +283,15 @@ def run_restore_latest(target_name: str) -> NoReturn:
         backups = provider.all_target_backups(target.env_name.lower())
         if not backups:
             log.warning("no backups at all for '%s'", target_name)
+            print(f"no backups at all for '{target_name}'")
             sys.exit(2)
         latest_backup = backups[0]
         path_age = provider.download_backup(latest_backup)
         path = core.run_decrypt_age_archive(path_age)
         target.restore(str(path))
         sys.exit(0)
     log.warning("target '%s' does not exist")
+    print(f"target '{target_name}' does not exist")
     sys.exit(1)
 
 
@@ -302,17 +305,20 @@ def run_restore(backup_name: str, target_name: str) -> NoReturn:
         backups = provider.all_target_backups(target.env_name.lower())
         if not backups:
             log.warning("no backups at all for '%s'", target_name)
+            print(f"no backups at all for '{target_name}'")
             sys.exit(2)
         if backup_name not in backups:
             log.warning(
                 "backup '%s' not exist at all for '%s'", backup_name, target_name
             )
+            print(f"backup '{backup_name}' not exist at all for '{target_name}'")
             sys.exit(2)
         path_age = provider.download_backup(backup_name)
         path = core.run_decrypt_age_archive(path_age)
         target.restore(str(path))
         sys.exit(0)
     log.warning("target '%s' does not exist")
+    print(f"target '{target_name}' does not exist")
     sys.exit(1)
 
 
@@ -340,7 +346,7 @@ def run_main_loop() -> NoReturn:  # pragma: no cover
     shutdown()
 
 
-def main() -> NoReturn:
+def main() -> NoReturn:  # pragma: no cover
     log.info("parsing runtime arguments...")
 
     runtime_args = setup_runtime_arguments()
@@ -370,7 +376,7 @@ def main() -> NoReturn:
             log.warning("--target must be defined to use --restore-latest")
             sys.exit(3)
         run_restore(runtime_args.restore, runtime_args.target)
-    else:  # pragma: no cover
+    else:
         run_main_loop()
 
 

ogion/upload_providers/debug.py

@@ -24,26 +24,39 @@ def __init__(self, target_provider: DebugProviderModel) -> None:
     @override
     def post_save(self, backup_file: Path) -> str:
         age_file = core.run_create_age_archive(backup_file=backup_file)
-        return str(age_file)
+
+        out_path = config.CONST_DEBUG_FOLDER_PATH / age_file.parent.name / age_file.name
+        out_path.parent.mkdir(mode=0o700, exist_ok=True)
+
+        shell_copy_to_debug_dir = f"cp {age_file} {out_path}"
+        core.run_subprocess(shell_copy_to_debug_dir)
+
+        return str(out_path)
 
     @override
     def all_target_backups(self, env_name: str) -> list[str]:
         backups: list[str] = []
-        path = config.CONST_BACKUP_FOLDER_PATH / env_name
+        path = config.CONST_DEBUG_FOLDER_PATH / env_name
+        path.mkdir(mode=0o700, exist_ok=True)
         for backup_path in path.iterdir():
             backups.append(str(backup_path.absolute()))
         backups.sort(reverse=True)
         return backups
 
     @override
     def download_backup(self, path: str) -> Path:
-        return Path(path)
+        backup_file = config.CONST_DOWNLOADS_FOLDER_PATH / path
+        backup_file.parent.mkdir(parents=True, exist_ok=True)
+
+        return Path(backup_file)
 
     @override
     def clean(
         self, backup_file: Path, max_backups: int, min_retention_days: int
     ) -> None:
-        core.remove_path(backup_file)
+        for backup_path in backup_file.parent.iterdir():
+            core.remove_path(backup_path)
+            log.info("removed %s from local disk", backup_path)
 
         backups = self.all_target_backups(env_name=backup_file.parent.name)
 

ogion/upload_providers/s3.py

@@ -110,7 +110,7 @@ def clean(
             delete_response = self.client.remove_objects(
                 self.bucket, delete_object_list=items_to_delete
             )
-            if len([error for error in delete_response]):
+            if len([error for error in delete_response]):  # pragma: no cover
                 raise RuntimeError(
                     "Fail to delete backups from s3: %s", delete_response
                 )

pyproject.toml

@@ -45,7 +45,7 @@ python-dotenv = "^1.0.1"
 responses = "^0.25.0"
 
 [tool.pytest.ini_options]
-addopts = "-v --cov --cov-report term-missing --cov-fail-under 90 -n auto"
+addopts = "-v --cov --cov-report term-missing --cov-fail-under 100 -n auto"
 env = [
   "AGE_RECIPIENTS=",
   "BACKUP_MAX_NUMBER=1",
@@ -64,12 +64,12 @@ filterwarnings = [
 reportImplicitOverride = true
 
 [tool.ruff]
-target-version = "py312"
+target-version = "py313"
 
 [tool.ruff.lint]
 # pycodestyle, pyflakes, isort, pylint, pyupgrade
+ignore = ["PLR0913"]
 select = ["E", "F", "I", "PL", "UP", "W"]
-
 [tool.coverage.run]
 omit = ["ogion/tools/*"]
 source = ["ogion"]

tests/conftest.py

@@ -3,10 +3,13 @@
 
 import os
 import secrets
+import time
 from pathlib import Path
 from typing import TypeVar
 
+import google.cloud.storage as storage_client
 import pytest
+from google.auth.credentials import AnonymousCredentials
 from pydantic import SecretStr
 
 from ogion import config
@@ -16,6 +19,12 @@
     PostgreSQLTargetModel,
     SingleFileTargetModel,
 )
+from ogion.models.upload_provider_models import (
+    AzureProviderModel,
+    DebugProviderModel,
+    GCSProviderModel,
+    S3ProviderModel,
+)
 from ogion.tools.compose_db_models import ComposeDatabase
 from ogion.tools.compose_file_generator import (
     DB_NAME,
@@ -25,6 +34,11 @@
     db_compose_mysql_data,
     db_compose_postgresql_data,
 )
+from ogion.upload_providers.azure import UploadProviderAzure
+from ogion.upload_providers.base_provider import BaseUploadProvider
+from ogion.upload_providers.debug import UploadProviderLocalDebug
+from ogion.upload_providers.google_cloud_storage import UploadProviderGCS
+from ogion.upload_providers.s3 import UploadProviderS3
 
 TM = TypeVar("TM", MariaDBTargetModel, PostgreSQLTargetModel)
 
@@ -51,7 +65,10 @@ def _to_target_model(
 
 DOCKER_TESTS: bool = os.environ.get("DOCKER_TESTS", None) is not None
 CONST_TOKEN_URLSAFE = "mock"
-CONST_UNSAFE_AGE_KEY = (
+CONST_UNSAFE_AGE_PUBLIC_KEY = (
+    "age1q5g88krfjgty48thtctz22h5ja85grufdm0jly3wll6pr9f30qsszmxzm2"
+)
+CONST_UNSAFE_AGE_SECRET_KEY = (
     "AGE-SECRET-KEY-12L9ETSAZJXK2XLGQRU503VMJ59NGXASGXKAUH05KJ4TDC6UKTAJQGMSN3L"
 )
 DB_VERSION_BY_ENV_VAR: dict[str, str] = {}
@@ -77,6 +94,12 @@ def _to_target_model(
     cron_rule="* * * * *",
     abs_path=Path(__file__).absolute().parent / "const/testfolder",
 )
+ALL_TARGETS = (
+    ALL_POSTGRES_DBS_TARGETS
+    + ALL_MYSQL_DBS_TARGETS
+    + ALL_MARIADB_DBS_TARGETS
+    + [FILE_1, FOLDER_1]
+)
 
 
 @pytest.fixture(autouse=True)
@@ -90,6 +113,9 @@ def fixed_const_config_setup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
     download_folder_path = tmp_path / "pytest_download"
     monkeypatch.setattr(config, "CONST_DOWNLOADS_FOLDER_PATH", download_folder_path)
     download_folder_path.mkdir(mode=0o700, parents=True, exist_ok=True)
+    debug_folder_path = tmp_path / "pytest_data_debug"
+    monkeypatch.setattr(config, "CONST_DEBUG_FOLDER_PATH", debug_folder_path)
+    debug_folder_path.mkdir(mode=0o700, parents=True, exist_ok=True)
     options = config.Settings(
         LOG_LEVEL="DEBUG",
         BACKUP_PROVIDER="name=debug",
@@ -107,7 +133,8 @@ def fixed_const_config_setup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
         SMTP_FROM_ADDR="",
         SMTP_PASSWORD=SecretStr(""),
         SMTP_TO_ADDRS="",
-        AGE_RECIPIENTS="age1q5g88krfjgty48thtctz22h5ja85grufdm0jly3wll6pr9f30qsszmxzm2",
+        AGE_RECIPIENTS=CONST_UNSAFE_AGE_PUBLIC_KEY,
+        DEBUG_AGE_SECRET_KEY=CONST_UNSAFE_AGE_SECRET_KEY,
     )
     monkeypatch.setattr(config, "options", options)
 
@@ -118,3 +145,66 @@ def mock_token_urlsafe(nbytes: int) -> str:
         return CONST_TOKEN_URLSAFE
 
     monkeypatch.setattr(secrets, "token_urlsafe", mock_token_urlsafe)
+
+
+@pytest.fixture(params=["gcs", "s3", "azure", "debug"])
+def provider(request: pytest.FixtureRequest) -> BaseUploadProvider:
+    if request.param == "gcs":
+        bucket = storage_client.Client(
+            credentials=AnonymousCredentials()  # type: ignore[no-untyped-call]
+        ).create_bucket(str(time.time_ns()))
+        return UploadProviderGCS(
+            GCSProviderModel(
+                bucket_name=bucket.name or "",
+                bucket_upload_path="test",
+                service_account_base64=SecretStr("Z29vZ2xlX3NlcnZpY2VfYWNjb3VudAo="),
+                chunk_size_mb=100,
+                chunk_timeout_secs=100,
+            )
+        )
+
+    elif request.param == "s3":
+        bucket = str(time.time_ns())
+        provider_s3 = UploadProviderS3(
+            S3ProviderModel(
+                endpoint="localhost:9000",
+                bucket_name=bucket,
+                access_key="minioadmin",
+                secret_key=SecretStr("minioadmin"),
+                bucket_upload_path="test",
+                secure=False,
+            )
+        )
+        provider_s3.client.make_bucket(bucket)
+        return provider_s3
+
+    elif request.param == "azure":
+        provider_azure = UploadProviderAzure(
+            # https://github.com/Azure/Azurite?tab=readme-ov-file#connection-strings
+            AzureProviderModel(
+                container_name=str(time.time_ns()),
+                connect_string=SecretStr(
+                    "DefaultEndpointsProtocol=http;"
+                    "AccountName=devstoreaccount1;"
+                    "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
+                    "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+                    "QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;"
+                    "TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+                ),
+            )
+        )
+        provider_azure.container_client.create_container()
+        return provider_azure
+    elif request.param == "debug":
+        return UploadProviderLocalDebug(DebugProviderModel())
+    else:
+        raise ValueError("unknown")
+
+
+@pytest.fixture
+def provider_prefix(provider: BaseUploadProvider) -> str:
+    if provider.__class__ == UploadProviderAzure:
+        return ""
+    elif provider.__class__ == UploadProviderLocalDebug:
+        return f"{config.CONST_DEBUG_FOLDER_PATH}/"
+    return "test/"