6.1

.github/workflows/dev_build.yml

@@ -29,7 +29,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASS }}
 
       - name: Build and push ogion image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: docker/Dockerfile
           context: .

.github/workflows/release_build.yml

@@ -27,7 +27,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASS }}
 
       - name: Build and push ogion image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           file: docker/Dockerfile
           context: .

.github/workflows/tests.yml

@@ -32,4 +32,8 @@ jobs:
         run: make docker_dbs_setup_up
 
       - name: Build image and run tests in container
-        run: make tests_${{ matrix.arch }}
+        env:
+          OGION_ARCH: ${{ matrix.arch }}
+        run: |
+          touch .env
+          make unit_tests

.github/workflows/type_check.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Install dependencies and actiavte virtualenv
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
-          poetry install --no-interaction --no-root
+          poetry install --no-interaction
 
       - name: Run ${{ matrix.check }}
         run: |

.github/workflows/update_compose_dbs.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Install dependencies and actiavte virtualenv
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
-          poetry install --no-interaction --no-root
+          poetry install --no-interaction
 
       - name: Check update
         run: |

.pre-commit-config.yaml

@@ -1,16 +1,16 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-yaml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.2
+    rev: v0.5.1
     hooks:
       - id: ruff-format
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.2
+    rev: v0.5.1
     hooks:
       - id: ruff
         args: [--fix]

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
-  }
+  },
+  "python.analysis.extraPaths": ["./ogion/tools"]
 }

Makefile

@@ -1,25 +1,42 @@
 # to run tests with arm64 see https://docs.docker.com/build/building/multi-platform/
-docker_dbs_setup_up:
-	docker compose -f docker/docker-compose.dbs.yml up -d 
+export OGION_ARCH ?= amd64
 
-docker_dbs_setup_down:
-	docker compose -f docker/docker-compose.dbs.yml down
+ogion/bin/7zip/amd64/7zzs:
+	rm -rf ogion/bin/7zip/amd64
+	mkdir -p ogion/bin/7zip/amd64
+	cd ogion/bin/7zip/amd64 && \
+	wget --quiet "https://www.7-zip.org/a/7z2301-linux-x64.tar.xz" && \
+	tar -xf "7z2301-linux-x64.tar.xz" && \
+	rm -f "7z2301-linux-x64.tar.xz" && \
+	rm -rf MANUAL
 
-tests_amd64:
-	make docker_dbs_setup_up
-	docker compose -f docker/docker-compose.tests.yml run --rm --build ogion_tests_amd64
+ogion/bin/7zip/arm64/7zzs:
+	rm -rf ogion/bin/7zip/arm64
+	mkdir -p ogion/bin/7zip/arm64
+	cd ogion/bin/7zip/arm64 && \
+	wget --quiet "https://www.7-zip.org/a/7z2301-linux-arm64.tar.xz" && \
+	tar -xf "7z2301-linux-arm64.tar.xz" && \
+	rm -f "7z2301-linux-arm64.tar.xz" && \
+	rm -rf MANUAL
 
-tests_arm64:
-	make docker_dbs_setup_up
-	docker compose -f docker/docker-compose.tests.yml run --rm --build ogion_tests_arm64
+.PHONY: docker_dbs_setup_up
+docker_dbs_setup_up:
+	docker compose -f docker/docker-compose.dbs.yml up -d --remove-orphans
+
+.PHONY: docker_dbs_setup_down
+docker_dbs_setup_down:
+	docker compose -f docker/docker-compose.dbs.yml down --remove-orphans
 
-acceptance_tests_amd64:
-	make docker_dbs_setup_up
-	docker compose -f docker/docker-compose.acceptance.yml run --rm --build ogion_acceptance_test_amd64
+.PHONY: unit_tests
+unit_tests:
+	$(MAKE) docker_dbs_setup_up
+	docker compose -f docker/docker-compose.yml run --rm --build ogion_unit_tests 
 
-acceptance_tests_arm64:
-	make docker_dbs_setup_up
-	docker compose -f docker/docker-compose.acceptance.yml run --rm --build ogion_acceptance_test_arm64
+.PHONY: acceptance_tests
+acceptance_tests:
+	$(MAKE) docker_dbs_setup_up
+	docker compose -f docker/docker-compose.yml run --rm --build ogion_acceptance_tests
 
+.PHONY: update_compose_db_file
 update_compose_db_file:
 	poetry run python ogion/tools/compose_file_generator.py > docker/docker-compose.dbs.yml

docker/Dockerfile

@@ -8,7 +8,7 @@ ENV FOLDER_PATH="/var/lib/ogion"
 ENV LOG_FOLDER_PATH="/var/log/ogion"
 WORKDIR ${FOLDER_PATH}
 
-RUN apt-get -y update && apt-get -y install wget unzip gpg
+RUN apt-get -y update && apt-get -y install wget unzip gpg rsync
 COPY scripts scripts
 RUN ./scripts/install_mariadb_client.sh
 RUN ./scripts/install_postgresql_client.sh
@@ -42,6 +42,22 @@ RUN remove_arch=$(arch | sed s/aarch64/amd64/ | sed s/x86_64/arm64/) \
 # rm 7zip 7zz dynamic linked version binary
 RUN target_arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
     && rm -f ogion/bin/7zip/${target_arch}/7zz
+# rm to save ~65M
+RUN rm -f \
+    /usr/bin/mariadb-binlog \
+    /usr/bin/mariadb-slap \
+    /usr/bin/mariadb-admin \
+    /usr/bin/mariadb-check \
+    /usr/bin/mariadb-show \
+    /usr/bin/mariadb-import \
+    /usr/bin/perror \
+    /usr/bin/mariadb-conv \
+    /usr/bin/mariadb-plugin \
+    /usr/bin/mariadb-tzinfo-to-sql \
+    /usr/bin/resolve_stack_dump \
+    /usr/bin/my_print_defaults \
+    /usr/bin/mariadb-waitpid \
+    /usr/bin/replace
 
 COPY scripts/docker_entrypoint.sh /docker_entrypoint.sh
 RUN rm -rf scripts

docker/docker-compose.acceptance.yml → docker/docker-compose.yml

@@ -1,24 +1,21 @@
 services:
-  ogion_acceptance_test_amd64:
+  ogion_unit_tests:
     restart: "no"
     network_mode: "host"
-    platform: linux/amd64
-    build: 
+    platform: linux/${OGION_ARCH}
+    build:
       context: ./../
       dockerfile: docker/Dockerfile
-      target: build
-    command: python -m ogion.main --single
-    env_file:
-      - ./../.env
+      target: tests
 
-  ogion_acceptance_test_arm64:
+  ogion_acceptance_tests:
     restart: "no"
     network_mode: "host"
-    platform: linux/arm64
+    platform: linux/${OGION_ARCH:-amd64}
     build: 
       context: ./../
       dockerfile: docker/Dockerfile
       target: build
     command: python -m ogion.main --single
     env_file:
-      - ./../.env
+      - ./../.env

ogion/backup_targets/base_target.py

@@ -5,16 +5,14 @@
 from abc import ABC, abstractmethod
 from datetime import UTC, datetime
 from pathlib import Path
-from typing import TypeVar, final
+from typing import final
 
 from croniter import croniter
 
 from ogion.models.backup_target_models import TargetModel
 
 log = logging.getLogger(__name__)
 
-TM = TypeVar("TM", bound=TargetModel)
-
 
 class BaseBackupTarget(ABC):
     def __init__(self, target_model: TargetModel) -> None:
@@ -43,14 +41,6 @@ def max_backups(self) -> int:
     def min_retention_days(self) -> int:
         return self.target_model.min_retention_days
 
-    @final
-    def make_backup(self) -> Path:
-        try:
-            return self._backup()
-        except Exception as err:
-            log.error(err, exc_info=True)
-            raise
-
     @final
     def _get_next_backup_time(self) -> datetime:
         now = datetime.now(UTC)
@@ -70,6 +60,16 @@ def next_backup(self) -> bool:
             return True
         return False
 
+    @final
+    @property
+    def pretty_thread_name(self) -> str:
+        pretty_env_name = self.env_name.replace("_", "-")
+        return f"Thread-{pretty_env_name}"
+
+    @abstractmethod
+    def backup(self) -> Path:  # pragma: no cover
+        pass
+
     @abstractmethod
-    def _backup(self) -> Path:  # pragma: no cover
+    def restore(self, path: str) -> None:  # pragma: no cover
         pass

ogion/backup_targets/file.py

@@ -3,6 +3,7 @@
 
 import logging
 from pathlib import Path
+from typing import override
 
 from ogion import core
 from ogion.backup_targets.base_target import BaseBackupTarget
@@ -16,7 +17,8 @@ def __init__(self, target_model: SingleFileTargetModel) -> None:
         super().__init__(target_model)
         self.target_model: SingleFileTargetModel = target_model
 
-    def _backup(self) -> Path:
+    @override
+    def backup(self) -> Path:
         escaped_filename = core.safe_text_version(self.target_model.abs_path.name)
 
         out_file = core.get_new_backup_path(self.env_name, escaped_filename)
@@ -26,3 +28,10 @@ def _backup(self) -> Path:
         core.run_subprocess(shell_create_file_symlink)
         log.debug("finished ln, output: %s", out_file)
         return out_file
+
+    @override
+    def restore(self, path: str) -> None:
+        shell_cp_file = f"cp {path} {self.target_model.abs_path}"
+        log.debug("start cp in subprocess: %s", shell_cp_file)
+        core.run_subprocess(shell_cp_file)
+        log.debug("finished cp to %s", self.target_model.abs_path)

ogion/backup_targets/folder.py

@@ -3,6 +3,7 @@
 
 import logging
 from pathlib import Path
+from typing import override
 
 from ogion import core
 from ogion.backup_targets.base_target import BaseBackupTarget
@@ -16,7 +17,8 @@ def __init__(self, target_model: DirectoryTargetModel) -> None:
         super().__init__(target_model)
         self.target_model: DirectoryTargetModel = target_model
 
-    def _backup(self) -> Path:
+    @override
+    def backup(self) -> Path:
         escaped_foldername = core.safe_text_version(self.target_model.abs_path.name)
 
         out_file = core.get_new_backup_path(self.env_name, escaped_foldername)
@@ -26,3 +28,10 @@ def _backup(self) -> Path:
         core.run_subprocess(shell_create_dir_symlink)
         log.debug("finished ln, output: %s", out_file)
         return out_file
+
+    @override
+    def restore(self, path: str) -> None:
+        shell_cp_file = f"rsync -avh {path}/ {self.target_model.abs_path}"
+        log.debug("start cp in subprocess: %s", shell_cp_file)
+        core.run_subprocess(shell_cp_file)
+        log.debug("finished cp to %s", self.target_model.abs_path)

ogion/backup_targets/mariadb.py

@@ -6,6 +6,7 @@
 import re
 import shlex
 from pathlib import Path
+from typing import override
 
 from ogion import config, core
 from ogion.backup_targets.base_target import BaseBackupTarget
@@ -91,7 +92,8 @@ def _mariadb_connection(self) -> str:
         log.info("mariadb_connection calculated version: %s", version)
         return version
 
-    def _backup(self) -> Path:
+    @override
+    def backup(self) -> Path:
         escaped_dbname = core.safe_text_version(self.target_model.db)
         escaped_version = core.safe_text_version(self.db_version)
         name = f"{escaped_dbname}_{escaped_version}"
@@ -106,3 +108,12 @@ def _backup(self) -> Path:
         core.run_subprocess(shell_mariadb_dump_db)
         log.debug("finished mariadbdump, output: %s", out_file)
         return out_file
+
+    @override
+    def restore(self, path: str) -> None:
+        shell_mariadb_restore = (
+            f"mariadb --defaults-file={self.option_file} {self.db_name} < {path}"
+        )
+        log.debug("start restore in subprocess: %s", shell_mariadb_restore)
+        core.run_subprocess(shell_mariadb_restore)
+        log.debug("finished restore")