Postgres backups

_scripts/Makefile.docker-compose

@@ -14,10 +14,10 @@ docker-compose-build: check-instance-project
 
 .PHONY: docker-compose-shell
 docker-compose-shell:
-	@echo '## Starting shell ...'
-	@echo '## Available arguments for `make shell`:'
-	@echo '## USERNAME: the user to run the shell as. (defaults to the docker image USER)'
-	@echo '## COMMAND: the interactive command to run. (defaults to /bin/bash || /bin/sh)'
-	@echo '## CD: the working directory to use for COMMAND. (defaults to the docker image WORKDIR)'
+#	@echo '## Starting shell ...'
+#	@echo '## Available arguments for `make shell`:'
+#	@echo '## USERNAME: the user to run the shell as. (defaults to the docker image USER)'
+#	@echo '## COMMAND: the interactive command to run. (defaults to /bin/bash || /bin/sh)'
+#	@echo '## CD: the working directory to use for COMMAND. (defaults to the docker image WORKDIR)'
 	@echo
 	@COMMAND=$${COMMAND:-"if [ -f /bin/bash ]; then /bin/bash; else /bin/sh; fi;"}; USERNAME_ARG=$$(test -z "$$USERNAME" && echo "" || echo "-u '$$USERNAME'"); CD_ARG=$$(test -z "$$CD" && echo "" || echo "-w '$$CD'"); make --no-print-directory docker-compose-lifecycle-cmd EXTRA_ARGS="exec -it $${USERNAME_ARG} $${CD_ARG} ${SERVICE} /bin/sh -c '$${COMMAND}'"

_scripts/Makefile.lifecycle

@@ -31,12 +31,20 @@ destroy: check-instance-project
 
 .PHONY: ps # Show containers status (docker compose ps)
 ps:
-	@echo "Showing containers for a single instance (use \`make status\` to see all instances.)"
+	@echo "Showing containers for a single instance (use \`make status\` to see all instances.)" >/dev/stderr
 	@make --no-print-directory docker-compose-lifecycle-cmd EXTRA_ARGS="ps -a"
 
+.PHONY: ensure-started
+ensure-started:
+	@if [[ "$$(make --no-print-directory docker-compose-lifecycle-cmd EXTRA_ARGS='ps -q' | wc -l)" == "0" ]]; then echo "Services are not started."; exit 1; else echo "Services are started."; fi
+
+.PHONY: ensure-stopped
+ensure-stopped:
+	@if [[ "$$(make --no-print-directory docker-compose-lifecycle-cmd EXTRA_ARGS='ps -q' | wc -l)" == "0" ]]; then echo "Services are stopped."; exit 0; else make --no-print-directory ps; echo "Services are not stopped."; exit 1; fi
+
 .PHONY: status # Show status of all instances
 status:
-	@(echo -e "NAME\tENV\tID\tIMAGE\tSTATE\tHEALTH\tPORTS" && docker ps --filter "label=com.docker.compose.project.working_dir=$${PWD}" -q | xargs -iXX docker inspect XX | jq '.[0]' | jq -r '(.Name[1:]) + "\t" + (.Config.Labels["com.docker.compose.project.environment_file"] | split("/";"")[-1]) + "\t" + .Id[:10] + "\t" + .Config.Image + "\t" + .State.Status + "\t" + (.State.Health.Status|tostring) + "\t" + (.NetworkSettings.Ports|tostring) ') | column -t
+	@(echo -e "NAME\tENV\tID\tIMAGE\tSTATE\tHEALTH\tPORTS" && docker ps --filter "label=com.docker.compose.project.working_dir=$${PWD}" -q | xargs -iXX docker inspect XX | jq '.[0]' | jq -r '(.Name[1:]) + "\t" + (.Config.Labels["com.docker.compose.project.environment_file"] | split("/";"")[-1]) + "\t" + .Id[:10] + "\t" + .Config.Image + "\t" + .State.Status + "\t" + (.State.Health.Status|tostring) + "\t" + (.NetworkSettings.Ports|tostring) ') | column -t  >/dev/stderr
 
 .PHONY: logs # Tail all containers logs (set SERVICE=name to filter for one)
 logs:

_scripts/docker_wait_for_healthcheck

@@ -0,0 +1 @@
+run_func

_scripts/funcs.sh

@@ -101,36 +101,54 @@ get_root_domain() {
 }
 
 docker_compose() {
-    local ENV_FILE=${ENV_FILE:-.env_$(${BIN}/docker_context)}
-    local PROJECT_NAME="$(basename \"$PWD\")"
+    check_var PROJECT_NAME ENV_FILE
+    echo "PROJECT_NAME=${PROJECT_NAME}"
+    echo "ENV_FILE=${ENV_FILE}"
     if [[ -n "${instance:-${INSTANCE}}" ]] && [[ "${ENV_FILE}" != ".env_${DOCKER_CONTEXT}_${instance:-${INSTANCE}}" ]]; then
         ENV_FILE="${ENV_FILE}_${instance:-${INSTANCE}}"
-        PROJECT_NAME="$(basename \"$PWD\")_${instance:-${INSTANCE}}"
+        PROJECT_NAME="$(basename $PWD)_${instance:-${INSTANCE}}"
     fi
-    set -ex
-    docker compose ${DOCKER_COMPOSE_FILE_ARGS:--f docker-compose.yaml} --env-file="${ENV_FILE}" --project-name="${PROJECT_NAME}" "$@"
+    (set -ex; docker compose ${DOCKER_COMPOSE_FILE_ARGS:--f docker-compose.yaml} --env-file="${ENV_FILE}" --project-name="${PROJECT_NAME}" "$@")
 }
 
 docker_run() {
     local ENV_FILE=${ENV_FILE:-.env_$(${BIN}/docker_context)}
-    local PROJECT_NAME="$(basename \"$PWD\")"
+    local PROJECT_NAME="$(basename ${PWD})"
     if [[ -n "${instance:-${INSTANCE}}" ]] && [[ "${ENV_FILE}" != ".env_${DOCKER_CONTEXT}_${instance:-${INSTANCE}}" ]]; then
         ENV_FILE="${ENV_FILE}_${instance:-${INSTANCE}}"
-        PROJECT_NAME="$(basename \"$PWD\")_${instance:-${INSTANCE}}"
+        PROJECT_NAME="$(basename ${PWD})_${instance:-${INSTANCE}}"
     fi
-    set -ex
-    docker run --rm --env-file=${ENV_FILE} "$@"
+    (set -ex; docker run --rm --env-file=${ENV_FILE} "$@")
 }
 
 docker_exec() {
     local ENV_FILE=${ENV_FILE:-.env_$(${BIN}/docker_context)}
-    local PROJECT_NAME="$(basename \"$PWD\")"
+    local PROJECT_NAME="$(basename ${PWD})"
     if [[ -n "${instance:-${INSTANCE}}" ]] && [[ "${ENV_FILE}" != ".env_${DOCKER_CONTEXT}_${instance:-${INSTANCE}}" ]]; then
         ENV_FILE="${ENV_FILE}_${instance:-${INSTANCE}}"
-        PROJECT_NAME="$(basename \"$PWD\")_${instance:-${INSTANCE}}"
+        PROJECT_NAME="$(basename ${PWD})_${instance:-${INSTANCE}}"
     fi
-    set -ex
-    docker exec --env-file=${ENV_FILE} "$@"
+    (set -ex; docker exec --env-file=${ENV_FILE} "$@")
+}
+
+docker_wait_for_healthcheck() {
+    local container_id=$1
+    check_var container_id
+    local attempts=150;
+    echo "## Waiting for container healthcheck: ${container_id}"
+    while [[ "${attempts}" -gt 0 ]]; do
+        if [[ "$(docker inspect -f {{.State.Health.Status}} $container_id)" == "healthy" ]]; then
+            echo "## HEALTHY - Container ${container_id} healthcheck passed."
+            return 0
+        fi
+        if [ $(( attempts % 10 )) -eq 9 ]; then
+            echo "## Still waiting for container ${container_id} to start ..."
+        fi
+        attempts=$((attempts-1))
+        sleep 2;
+    done
+    echo "## UNHEALTHY - Container ${container_id} still has not started yet." >/dev/stderr
+    exit 1
 }
 
 ytt() {

minio/docker-compose.yaml

@@ -12,6 +12,7 @@ services:
     environment:
       - MINIO_ROOT_USER
       - MINIO_ROOT_PASSWORD
+      - MINIO_DOMAIN=${MINIO_TRAEFIK_HOST}
     volumes:
       - data:/data
     command: server /data --console-address ":9001"

postgresql/.env-dist

@@ -1,9 +1,11 @@
-## POSTGRES_TRAEFIK_HOST can be any domain that resolves to your Docker server:
+## POSTGRES_HOST can be any domain that resolves to your Docker server:
 ## This name is used in the server certificate as part of the sslmode=verify-full option
-POSTGRES_TRAEFIK_HOST=postgres.example.com
+POSTGRES_HOST=postgres.example.com
+POSTGRES_INSTANCE=
 
 ## The docker tag for the postgres image:
-POSTGRES_VERSION=14
+POSTGRES_VERSION=16
+POSTGRES_PGRATIONAL_VERSION=v0.0.2
 
 ## The postgres image creates the POSTGRES_USER as a Super User and with POSTGRES_PASSWORD.
 ## We will rename these here as POSTGRES_ADMIN_USER and POSTGRES_ADMIN_PASSWORD
@@ -13,16 +15,53 @@ POSTGRES_ADMIN_USER=root
 ## Note: This password is required to be set by the postgres image, but its not
 ## being used in any of the authentication schemes currently defined in our
 ## customized pg_hba.conf, so this password doesn't really matter.
-POSTGRES_ADMIN_PASSWORD=change_me_but_it_doesnt_really_matter
+POSTGRES_ADMIN_PASSWORD=
 
 ## The name of the database to create on first startup:
+## (as well as the username to connect to it):
 POSTGRES_DB=tweedle
 
 ## The external TCP port mapped on the docker host (public access!)
+## Each instance running on the same host must have a unique port.
 POSTGRES_EXTERNAL_TCP_PORT=5432
 
 ## Allowed IP address source range (CIDR with /netmask) for postgres clients to connect:
 ## eg. To allow any host to conenct (still needs valid certificate), use: 0.0.0.0/0
 ## eg. To allow only a single ip to connect (example 192.168.1.1), use: 192.168.1.1/32
 POSTGRES_ALLOWED_IP_SOURCERANGE=0.0.0.0/0
 
+## Set this to true to turn on backups via pgbackrest:
+POSTGRES_PGBACKREST=false
+## pgbackrest backup storage repositories to enable:
+### Set each one to true you wish to enable (Setting them all false will disable backups)
+POSTGRES_PGBACKREST_LOCAL=false
+POSTGRES_PGBACKREST_LOCAL_RETENTION_FULL=1
+POSTGRES_PGBACKREST_LOCAL_RETENTION_DIFF=7
+POSTGRES_PGBACKREST_S3=false
+POSTGRES_PGBACKREST_S3_RETENTION_FULL=4
+POSTGRES_PGBACKREST_S3_RETENTION_DIFF=7
+POSTGRES_PGBACKREST_S3_ENDPOINT=s3.example.com
+POSTGRES_PGBACKREST_S3_REGION=default
+POSTGRES_PGBACKREST_S3_BUCKET=my-bucket
+POSTGRES_PGBACKREST_S3_KEY_ID=my-access-key-id
+POSTGRES_PGBACKREST_S3_KEY_SECRET=my-secret-key
+
+
+### Cron scheduled backups:
+### Use cronexpr format: https://github.com/gorhill/cronexpr
+## eg. weekly at 2 AM on Sunday morning: 0 0 2 * * 0 *
+## eg. daily at midnight: @daily
+## eg. every six hours stating at 3AM: 0 0 3/6 * * * *
+POSTGRES_PGBACKREST_LOCAL_SCHEDULE_FULL=0 0 0 * * 0 *
+POSTGRES_PGBACKREST_LOCAL_SCHEDULE_DIFF=0 0 3/6 * * * *
+POSTGRES_PGBACKREST_S3_SCHEDULE_FULL=0 0 0 * * 0 *
+POSTGRES_PGBACKREST_S3_SCHEDULE_DIFF=0 0 3/6 * * * *
+
+## Optional encryption passphrase for backups (leave it blank to disable encryption):
+## (Keep this safe, obviously, you will need this later in order to perform restoration!)
+POSTGRES_PGBACKREST_ENCRYPTION_PASSPHRASE=
+
+## Set POSTGRES_MAINTAINANCE_MODE=true to start up in maintainance mode.
+## In maintainance mode, postgres and tinycron will not be started, so
+## you may enter the shell and perform any maintainace tasks.
+POSTGRES_MAINTAINANCE_MODE=false