refactor scripts, 7zip path detection in app, added better check for …

…mariadb/psql clients installation

Dockerfile

@@ -5,6 +5,7 @@ ENV FOLDER_PATH="/var/lib/backuper"
 ENV LOG_FOLDER_PATH="/var/log/backuper"
 WORKDIR ${FOLDER_PATH}
 
+RUN apt-get update -y && apt-get install -y curl wget unzip gpg xz-utils
 RUN addgroup --gid 1001 --system $SERVICE_NAME && \
     adduser --gid 1001 --shell /bin/false --disabled-password --uid 1001 $SERVICE_NAME
 
@@ -14,9 +15,10 @@ ENV PATH="/venv/bin:$PATH"
 COPY scripts/docker_entrypoint.sh /docker_entrypoint.sh
 COPY scripts scripts
 COPY bin bin
-RUN /bin/bash scripts/install_apt_libs_and_7zip.sh
-RUN rm -rf scripts bin/7zip
-RUN apt-get update -y && apt-get install -y curl wget unzip
+RUN /bin/bash scripts/install_postgresql_client.sh
+RUN /bin/bash scripts/install_mariadb_mysql_client.sh
+RUN rm -rf scripts
+
 
 ENTRYPOINT ["/bin/bash", "/docker_entrypoint.sh"]
 

backuper/backup_targets/file.py

@@ -6,8 +6,6 @@
 
 log = logging.getLogger(__name__)
 
-config.BackupTargetEnum.FILE
-
 
 class File(BaseBackupTarget, target_model_name=config.BackupTargetEnum.FILE):
     def __init__(

backuper/backup_targets/mariadb.py

@@ -63,16 +63,28 @@ def escape(s: str) -> str:
         return path
 
     def _mariadb_connection(self) -> str:
-        log.debug("mariadb_connection start mariadb connection")
+        try:
+            log.debug("check mariadb installation")
+            mariadb_version = core.run_subprocess("mariadb -V")
+            log.debug("output: %s", mariadb_version)
+        except core.CoreSubprocessError as version_err:  # pragma: no cover
+            log.critical(
+                "mariadb client is not detected on your system (%s)\n"
+                "check out ready script: "
+                "https://github.com/rafsaf/backuper/blob/main/scripts/install_mariadb_mysql_client.sh",
+                version_err,
+            )
+            sys.exit(1)
+        log.debug("start mariadb connection")
         try:
             db = shlex.quote(self.db)
             result = core.run_subprocess(
                 f"mariadb --defaults-file={self.option_file} {db} "
                 f"--execute='SELECT version();'",
             )
-        except core.CoreSubprocessError as err:
-            log.error(err, exc_info=True)
-            log.error("mariadb_connection unable to connect to database, exiting")
+        except core.CoreSubprocessError as conn_err:
+            log.error(conn_err, exc_info=True)
+            log.error("unable to connect to database, exiting")
             sys.exit(1)
 
         version = None

backuper/backup_targets/mysql.py

@@ -63,8 +63,19 @@ def escape(s: str) -> str:
         return path
 
     def _mysql_connection(self) -> str:
-        log.debug("mysql_connection start mysql connection")
-
+        try:
+            log.debug("check mysql installation")
+            mysql_version = core.run_subprocess("mysql -V")
+            log.debug("output: %s", mysql_version)
+        except core.CoreSubprocessError as version_err:  # pragma: no cover
+            log.critical(
+                "mysql client is not detected on your system (%s)\n"
+                "check out ready script: "
+                "https://github.com/rafsaf/backuper/blob/main/scripts/install_mariadb_mysql_client.sh",
+                version_err,
+            )
+            sys.exit(1)
+        log.debug("start mysql connection")
         try:
             db = shlex.quote(self.db)
             result = core.run_subprocess(
@@ -73,7 +84,7 @@ def _mysql_connection(self) -> str:
             )
         except core.CoreSubprocessError as err:
             log.error(err, exc_info=True)
-            log.error("mysql_connection unable to connect to database, exiting")
+            log.error("unable to connect to database, exiting")
             sys.exit(1)
 
         version = None

backuper/backup_targets/postgresql.py

@@ -83,15 +83,26 @@ def _get_escaped_conn_uri(self) -> str:
         return escaped_uri
 
     def _postgres_connection(self) -> str:
-        log.debug("postgres_connection start postgres connection")
-
+        try:
+            log.debug("check psql installation")
+            psql_version = core.run_subprocess("psql -V")
+            log.debug("output: %s", psql_version)
+        except core.CoreSubprocessError as version_err:  # pragma: no cover
+            log.critical(
+                "psql postgres client is not detected on your system (%s)\n"
+                "check out ready script: "
+                "https://github.com/rafsaf/backuper/blob/main/scripts/install_postgresql_client.sh",
+                version_err,
+            )
+            sys.exit(1)
+        log.debug("start postgres connection")
         try:
             result = core.run_subprocess(
                 f"psql -d {self.escaped_conn_uri} -w --command 'SELECT version();'",
             )
         except core.CoreSubprocessError as err:
             log.error(err, exc_info=True)
-            log.error("postgres_connection unable to connect to database, exiting")
+            log.error("unable to connect to database, exiting")
             sys.exit(1)
 
         version = None

backuper/config.py

@@ -29,7 +29,7 @@ class BackupTargetEnum(StrEnum):
 
 
 CONST_ENV_NAME_REGEX = re.compile(r"^[A-Za-z_0-9]{1,}$")
-CONST_ZIP_BIN_7ZZ_PATH: Path = BASE_DIR / "bin/7zz"
+CONST_BIN_ZIP_PATH: Path = BASE_DIR / "bin/7zip"
 CONST_BACKUP_FOLDER_PATH: Path = BASE_DIR / "data"
 CONST_GOOGLE_SERVICE_ACCOUNT_PATH: Path = BASE_DIR / "google_auth.json"
 CONST_BACKUP_FOLDER_PATH.mkdir(mode=0o744, parents=True, exist_ok=True)
@@ -45,11 +45,6 @@ class BackupTargetEnum(StrEnum):
 DISCORD_SUCCESS_WEBHOOK_URL: str = os.environ.get("DISCORD_SUCCESS_WEBHOOK_URL", "")
 DISCORD_FAIL_WEBHOOK_URL: str = os.environ.get("DISCORD_FAIL_WEBHOOK_URL", "")
 
-if not CONST_ZIP_BIN_7ZZ_PATH.exists():  # pragma: no cover
-    raise RuntimeError(
-        f"`{CONST_ZIP_BIN_7ZZ_PATH}` binary does not exists, did you forget to create it?"
-    )
-
 
 def logging_config(log_level: str) -> None:
     conf = {

backuper/core.py

@@ -62,20 +62,19 @@ def get_new_backup_path(env_name: str, name: str, sql: bool = False) -> Path:
 
 
 def run_create_zip_archive(backup_file: Path) -> Path:
+    seven_zip_path = seven_zip_bin_path()
     out_file = Path(f"{backup_file}.zip")
     log.debug("run_create_zip_archive start creating in subprocess: %s", backup_file)
     zip_escaped_password = shlex.quote(config.ZIP_ARCHIVE_PASSWORD)
     shell_args_create = (
-        f"{config.CONST_ZIP_BIN_7ZZ_PATH} a -p{zip_escaped_password} "
+        f"{seven_zip_path} a -p{zip_escaped_password} "
         f"-mx={config.ZIP_ARCHIVE_LEVEL} {out_file} {backup_file}"
     )
     run_subprocess(shell_args_create)
     log.debug("run_create_zip_archive finished, output: %s", out_file)
 
     log.debug("run_create_zip_archive start integriy test in subprocess: %s", out_file)
-    shell_args_integriy = (
-        f"{config.CONST_ZIP_BIN_7ZZ_PATH} t -p{zip_escaped_password} {out_file}"
-    )
+    shell_args_integriy = f"{seven_zip_path} t -p{zip_escaped_password} {out_file}"
     integrity_check_result = run_subprocess(shell_args_integriy)
     if "Everything is Ok" not in integrity_check_result:  # pragma: no cover
         raise AssertionError("zip arichive integrity fatal error")
@@ -158,3 +157,14 @@ def create_provider_model() -> ProviderModel:
     )
     target_model_cls = target_map[base_provider.name]
     return _validate_model("backup_provider", config.BACKUP_PROVIDER, target_model_cls)
+
+
+def seven_zip_bin_path() -> Path:
+    shell_args_cpu_architecture = "dpkg --print-architecture"
+    cpu_arch = run_subprocess(shell_args_cpu_architecture).strip()
+    seven_zip = config.CONST_BIN_ZIP_PATH / f"{cpu_arch}/7zz"
+    if not seven_zip.exists():  # pragma: no cover
+        raise RuntimeError(
+            f"unsuported architecture {cpu_arch}, 7zip not found at {seven_zip}"
+        )
+    return seven_zip

scripts/download_7zip.sh

@@ -0,0 +1,33 @@
+#########################################################################
+# 7ZIP DOWNLOAD FOR ARM64 AND AMD64
+#
+# https://www.7-zip.org/download.html
+#########################################################################
+
+AMD64_DIR="$SCRIPT_DIR/../bin/7zip/amd64"
+AMD64_7ZZ="$AMD64_DIR/7zz"
+
+ARM64_DIR="$SCRIPT_DIR/../bin/7zip/arm64"
+ARM64_7ZZ="$ARM64_DIR/7zz"
+
+if [ -f "$AMD64_7ZZ" ]
+then
+  echo "$AMD64_7ZZ exists"
+else
+  mkdir -p $AMD64_DIR
+  cd $AMD64_DIR
+  wget --quiet "https://www.7-zip.org/a/7z2301-linux-x64.tar.xz"
+  tar -xf "7z2301-linux-x64.tar.xz"
+  rm -f "7z2301-linux-x64.tar.xz"
+fi
+
+if [ -f "$ARM64_7ZZ" ]
+then
+  echo "$ARM64_7ZZ exists"
+else
+  mkdir -p $ARM64_DIR
+  cd $ARM64_DIR
+  wget --quiet "https://www.7-zip.org/a/7z2301-linux-arm64.tar.xz"
+  tar -xf "7z2301-linux-arm64.tar.xz"
+  rm -f "7z2301-linux-arm64.tar.xz"
+fi

scripts/install_mariadb_mysql_client.sh

@@ -0,0 +1,10 @@
+#########################################################################
+# MARIADB CLIENT INSTALLATION
+#
+# https://mariadb.com/kb/en/mariadb-package-repository-setup-and-usage/
+#########################################################################
+
+echo "Installing mariadb-client with ready script"
+curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | bash
+apt-get -y update && apt-get -y install mariadb-client
+echo "mariadb-client installed"

scripts/install_postgresql_client.sh

@@ -0,0 +1,28 @@
+#########################################################################
+# POSTGRES CLIENT INSTALLATION
+#
+# https://www.postgresql.org/download/linux/debian/
+# Note apt-key is considered unsecure and signed-by used as a replacement
+#########################################################################
+
+CPU="$(dpkg --print-architecture)"
+DISTR="$(awk -F= '/^ID=/{print $2}' /etc/os-release)"
+DISTR_VERSION="$(awk -F= '/^VERSION_CODENAME=/{print $2}' /etc/os-release)"
+
+echo "Installing postgresql-client-15"
+
+mkdir -p /usr/share/keyrings/
+if [ -f "/usr/share/keyrings/www.postgresql.org.gpg" ]
+then
+  echo "/usr/share/keyrings/www.postgresql.org.gpg exists"
+else
+  wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/www.postgresql.org.gpg
+fi
+if [ -f "/etc/apt/sources.list.d/pgdg.list" ]
+then
+  echo "/etc/apt/sources.list.d/pgdg.list exists"
+else
+  echo "deb [signed-by=/usr/share/keyrings/www.postgresql.org.gpg arch=$CPU] http://apt.postgresql.org/pub/repos/apt $DISTR_VERSION-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+fi
+apt-get -y update && apt-get -y install postgresql-client-15
+echo "postgresql-client-15 installed"