Switch to compose v2, fixes for mac os (#26)

* Test mac os deployment * Use more posix-compatible grep calls * Fix brew command * More macos fixups * Fix * Fix arch check * Use sudo for modifying hosts * Try downloading docker-compose from github * More fixes * Make all docker images not require root * Use docker-compose v2 * Fixes * Disable cansetup check for now * Bump debian * Try auto-fixing permissions * Update compose to 2.8.0 minimum * Fix * Don't fail on old docker-compose deletion * fixes * Move to torrc * Fixes for tor * Upgrade docker-compose to 2.9.0 * Use DataDirectoryGroupReadable
bitcart · Aug 2, 2022 · 3a8a87c · 3a8a87c
1 parent e49d605
commit 3a8a87c
Show file tree

Hide file tree

Showing 20 changed files with 116 additions and 88 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -64,13 +64,13 @@ workflows:
       - test:
           requires:
             - lint
-      - cansetup:
-          requires:
-            - test
+      # - cansetup:
+      #     requires:
+      #       - test
       - deploy:
           context: global
-          requires:
-            - cansetup
+          # requires:
+          #   - cansetup
           filters:
             branches:
               only: master
diff --git a/backup.sh b/backup.sh
@@ -71,7 +71,7 @@ load_env true
 
 cd "$BITCART_BASE_DIRECTORY"
 
-deployment_name=$(container_name)
+deployment_name=$(volume_name)
 volumes_dir=/var/lib/docker/volumes
 backup_dir="$volumes_dir/backup_datadir"
 timestamp=$(date "+%Y%m%d-%H%M%S")
@@ -95,7 +95,7 @@ else
     echo "Backing up files …"
     files=()
     for fname in "${BACKUP_VOLUMES[@]}"; do
-        fname=$(container_name $fname)
+        fname=$(volume_name $fname)
         if [ -d "$volumes_dir/$fname" ]; then
             files+=("$fname")
         fi

diff --git a/compose/backend.Dockerfile b/compose/backend.Dockerfile
@@ -8,26 +8,30 @@ RUN cd /app && \
     chmod +x bitcart-cli
 
 
-FROM python:3.9-slim-buster
+FROM python:3.9-slim-bullseye
 
 ARG TARGETPLATFORM
 ENV IN_DOCKER=1
+ENV GOSU_VERSION 1.14
 LABEL org.bitcartcc.image=backend
 
 COPY bitcart /app
 COPY scripts/docker-entrypoint.sh /usr/local/bin/
 COPY scripts/setup-pypi-mirror.sh /usr/local/bin/
 COPY --from=go-builder /app/bitcart-cli /usr/local/bin/
 WORKDIR /app
-RUN apt-get update && apt-get install -y --no-install-recommends iproute2 openssh-client build-essential python3-dev libffi-dev && \
+RUN apt-get update && apt-get install -y --no-install-recommends iproute2 openssh-client build-essential python3-dev libffi-dev ca-certificates wget && \
+    dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" && \
+    wget -qO /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" && \
+    chmod +x /usr/local/bin/gosu && \
     groupadd --gid 1000 electrum && \
     useradd --uid 1000 --gid electrum --shell /bin/bash --create-home electrum && \
     setup-pypi-mirror.sh && \
     pip install -r requirements/deterministic/web.txt && \
     pip install -r requirements/deterministic/production.txt && \
-    apt-get purge -y build-essential python3-dev libffi-dev && \
+    apt-get purge -y build-essential python3-dev libffi-dev wget && \
     apt-get autoremove -y && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /root/.cache/pip
-USER electrum
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
 CMD ["sh"]
diff --git a/compose/bch.Dockerfile b/compose/bch.Dockerfile
@@ -26,10 +26,9 @@ RUN adduser -D $ELECTRUM_USER && \
     apk add --no-cache libsecp256k1-dev
 
 COPY --from=compile-image --chown=electrum /root/.local $ELECTRUM_HOME/.local
-COPY --from=compile-image $ELECTRUM_HOME/site $ELECTRUM_HOME/site
+COPY --from=compile-image --chown=electrum $ELECTRUM_HOME/site $ELECTRUM_HOME/site
 
 USER $ELECTRUM_USER
 WORKDIR $ELECTRUM_HOME/site
-VOLUME /data
 
 CMD ["python","daemons/bch.py"]
diff --git a/compose/bnb.Dockerfile b/compose/bnb.Dockerfile
@@ -26,10 +26,9 @@ RUN adduser -D $ELECTRUM_USER && \
     apk add --no-cache libsecp256k1-dev
 
 COPY --from=compile-image --chown=electrum /root/.local $ELECTRUM_HOME/.local
-COPY --from=compile-image $ELECTRUM_HOME/site $ELECTRUM_HOME/site
+COPY --from=compile-image --chown=electrum $ELECTRUM_HOME/site $ELECTRUM_HOME/site
 
 USER $ELECTRUM_USER
 WORKDIR $ELECTRUM_HOME/site
-VOLUME /data
 
 CMD ["python","daemons/bnb.py"]
diff --git a/compose/bsty.Dockerfile b/compose/bsty.Dockerfile
@@ -26,10 +26,9 @@ RUN adduser -D $ELECTRUM_USER && \
     apk add --no-cache libsecp256k1-dev
 
 COPY --from=compile-image --chown=electrum /root/.local $ELECTRUM_HOME/.local
-COPY --from=compile-image $ELECTRUM_HOME/site $ELECTRUM_HOME/site
+COPY --from=compile-image --chown=electrum $ELECTRUM_HOME/site $ELECTRUM_HOME/site
 
 USER $ELECTRUM_USER
 WORKDIR $ELECTRUM_HOME/site
-VOLUME /data
 
 CMD ["python","daemons/bsty.py"]
diff --git a/compose/btc.Dockerfile b/compose/btc.Dockerfile
@@ -26,10 +26,9 @@ RUN adduser -D $ELECTRUM_USER && \
     apk add --no-cache libsecp256k1-dev
 
 COPY --from=compile-image --chown=electrum /root/.local $ELECTRUM_HOME/.local
-COPY --from=compile-image $ELECTRUM_HOME/site $ELECTRUM_HOME/site
+COPY --from=compile-image --chown=electrum $ELECTRUM_HOME/site $ELECTRUM_HOME/site
 
 USER $ELECTRUM_USER
 WORKDIR $ELECTRUM_HOME/site
-VOLUME /data
 
 CMD ["python","daemons/btc.py"]
diff --git a/compose/eth.Dockerfile b/compose/eth.Dockerfile
@@ -26,10 +26,9 @@ RUN adduser -D $ELECTRUM_USER && \
     apk add --no-cache libsecp256k1-dev
 
 COPY --from=compile-image --chown=electrum /root/.local $ELECTRUM_HOME/.local
-COPY --from=compile-image $ELECTRUM_HOME/site $ELECTRUM_HOME/site
+COPY --from=compile-image --chown=electrum $ELECTRUM_HOME/site $ELECTRUM_HOME/site
 
 USER $ELECTRUM_USER
 WORKDIR $ELECTRUM_HOME/site
-VOLUME /data
 
 CMD ["python","daemons/eth.py"]
diff --git a/compose/ltc.Dockerfile b/compose/ltc.Dockerfile
@@ -26,10 +26,9 @@ RUN adduser -D $ELECTRUM_USER && \
     apk add --no-cache libsecp256k1-dev
 
 COPY --from=compile-image --chown=electrum /root/.local $ELECTRUM_HOME/.local
-COPY --from=compile-image $ELECTRUM_HOME/site $ELECTRUM_HOME/site
+COPY --from=compile-image --chown=electrum $ELECTRUM_HOME/site $ELECTRUM_HOME/site
 
 USER $ELECTRUM_USER
 WORKDIR $ELECTRUM_HOME/site
-VOLUME /data
 
 CMD ["python","daemons/ltc.py"]
diff --git a/compose/sbch.Dockerfile b/compose/sbch.Dockerfile
@@ -26,10 +26,9 @@ RUN adduser -D $ELECTRUM_USER && \
     apk add --no-cache libsecp256k1-dev
 
 COPY --from=compile-image --chown=electrum /root/.local $ELECTRUM_HOME/.local
-COPY --from=compile-image $ELECTRUM_HOME/site $ELECTRUM_HOME/site
+COPY --from=compile-image --chown=electrum $ELECTRUM_HOME/site $ELECTRUM_HOME/site
 
 USER $ELECTRUM_USER
 WORKDIR $ELECTRUM_HOME/site
-VOLUME /data
 
 CMD ["python","daemons/sbch.py"]
diff --git a/compose/scripts/docker-entrypoint.sh b/compose/scripts/docker-entrypoint.sh
@@ -3,9 +3,6 @@ set -ex
 
 # BitcartCC is configuring current instance or updating it via SSH access
 
-# Make host.docker.internal work (on linux-based docker engines)
-echo "$(/sbin/ip route | awk '/default/ { print $3 }')  host.docker.internal" >>/etc/hosts
-
 if [ ! -z "$SSH_KEY_FILE" ] && ! [ -f "$SSH_KEY_FILE" ]; then
     echo "Creating BitcartCC SSH key File..."
     ssh-keygen -t rsa -f "$SSH_KEY_FILE" -q -P "" -m PEM -C bitcartcc >/dev/null
@@ -22,4 +19,14 @@ if [ ! -z "$SSH_KEY_FILE" ] && [ -f "$SSH_AUTHORIZED_KEYS" ] && ! grep -q "bitca
     cat "$SSH_KEY_FILE.pub" >>"$SSH_AUTHORIZED_KEYS"
 fi
 
-exec "$@"
+# Fix all permissions
+
+getent group tor || groupadd --gid 19001 tor && usermod -a -G tor electrum
+
+for volume in $BITCART_VOLUMES; do
+    if [ -d "$volume" ]; then
+        find "$volume" \! -user electrum -exec chown electrum '{}' +
+    fi
+done
+
+exec gosu electrum "$@"
diff --git a/compose/torrc.tmpl b/compose/torrc.tmpl
@@ -21,6 +21,7 @@ HiddenServiceDir /var/lib/tor/hidden_services/{{ $name }}
 							{{ end }}
 # Redirecting to {{ $containerOrReverseProxyName }}
 HiddenServicePort {{ $port }} {{ $ip }}:{{ $virtualPort }}
+HiddenServiceDirGroupReadable 1
 						{{ end }}
 					{{ end }}
 				{{ end }}

diff --git a/compose/xrg.Dockerfile b/compose/xrg.Dockerfile
@@ -26,10 +26,9 @@ RUN adduser -D $ELECTRUM_USER && \
     apk add --no-cache libsecp256k1-dev
 
 COPY --from=compile-image --chown=electrum /root/.local $ELECTRUM_HOME/.local
-COPY --from=compile-image $ELECTRUM_HOME/site $ELECTRUM_HOME/site
+COPY --from=compile-image --chown=electrum $ELECTRUM_HOME/site $ELECTRUM_HOME/site
 
 USER $ELECTRUM_USER
 WORKDIR $ELECTRUM_HOME/site
-VOLUME /data
 
 CMD ["python","daemons/xrg.py"]
diff --git a/contrib/upgrades/upgrade-to-0600.sh b/contrib/upgrades/upgrade-to-0600.sh
@@ -15,14 +15,14 @@ load_env
 COMPOSE_DIR="$(realpath "${SCRIPT_DIR}/../../compose")"
 
 volumes_dir=/var/lib/docker/volumes
-datadir="$volumes_dir/$(container_name bitcart_datadir)/_data"
-logdir="$volumes_dir/$(container_name bitcart_logs)/_data"
+datadir="$volumes_dir/$(volume_name bitcart_datadir)/_data"
+logdir="$volumes_dir/$(volume_name bitcart_logs)/_data"
 cp -rv --preserve $logdir/* $datadir/logs/
 for fname in $datadir/logs/bitcart-log.log*; do
     fname_new=$(convert_name "$fname")
     mv -v "$fname" "$fname_new"
 done
-docker volume rm $(container_name bitcart_logs)
+docker volume rm $(volume_name bitcart_logs)
 cp -rv --preserve $COMPOSE_DIR/images/* $datadir/images/
 rm -rf "$COMPOSE_DIR/images"
 rm -rf "$COMPOSE_DIR/conf"
diff --git a/generator/docker-components/backend.yml b/generator/docker-components/backend.yml
@@ -2,12 +2,12 @@ services:
   backend:
     restart: unless-stopped
     image: bitcartcc/bitcart:stable
-    entrypoint: /usr/local/bin/docker-entrypoint.sh
     command: bash -c "alembic upgrade head && gunicorn -c gunicorn.conf.py main:app"
     environment:
       LOG_FILE: bitcart.log
       BITCART_DATADIR: /datadir
       BITCART_BACKUPS_DIR: /backups
+      BITCART_VOLUMES: /datadir /backups
       DB_PORT: 5432
       DB_HOST: database
       VIRTUAL_NETWORK: nginx-proxy
@@ -34,7 +34,8 @@ services:
       SSH_KEY_FILE: ${BITCART_SSH_KEY_FILE}
       SSH_AUTHORIZED_KEYS: ${BITCART_SSH_AUTHORIZED_KEYS}
       BASH_PROFILE_SCRIPT: ${BASH_PROFILE_SCRIPT}
-    user: "${USER_UID}:${USER_GID}"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     expose:
       - "8000"
     volumes:

diff --git a/generator/docker-components/tor.yml b/generator/docker-components/tor.yml
@@ -44,6 +44,7 @@ services:
       TOR_ADDITIONAL_CONFIG: /usr/local/etc/tor/torrc-2
       TOR_EXTRA_ARGS: |
         CookieAuthentication 1
+        DataDirectoryGroupReadable 1
     expose:
       - "9050" # SOCKS
       - "9051" # Tor Control

diff --git a/generator/docker-components/worker.yml b/generator/docker-components/worker.yml
@@ -2,12 +2,12 @@ services:
   worker:
     restart: unless-stopped
     image: bitcartcc/bitcart:stable
-    entrypoint: /usr/local/bin/docker-entrypoint.sh
     command: python3 worker.py
     environment:
       LOG_FILE: bitcart.log
       BITCART_DATADIR: /datadir
       BITCART_BACKUPS_DIR: /backups
+      BITCART_VOLUMES: /datadir /backups
       BITCART_CRYPTOS: ${BITCART_CRYPTOS:-btc}
       BTC_NETWORK: ${BTC_NETWORK:-mainnet}
       BTC_LIGHTNING: ${BTC_LIGHTNING:-false}
@@ -25,7 +25,8 @@ services:
       SSH_KEY_FILE: ${BITCART_SSH_KEY_FILE}
       SSH_AUTHORIZED_KEYS: ${BITCART_SSH_AUTHORIZED_KEYS}
       BASH_PROFILE_SCRIPT: ${BASH_PROFILE_SCRIPT}
-    user: "${USER_UID}:${USER_GID}"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     expose:
       - "9020"
     volumes:

diff --git a/helpers.sh b/helpers.sh
@@ -37,15 +37,15 @@ EOF
 
 bitcart_start() {
     create_backup_volume
-    USER_UID=${UID} USER_GID=${GID} docker-compose -p "$NAME" -f compose/generated.yml up --build --remove-orphans -d $1
+    docker compose -p "$NAME" -f compose/generated.yml up --build --remove-orphans -d $1
 }
 
 bitcart_stop() {
-    USER_UID=${UID} USER_GID=${GID} docker-compose -p "$NAME" -f compose/generated.yml down
+    docker compose -p "$NAME" -f compose/generated.yml down
 }
 
 bitcart_pull() {
-    docker-compose -f compose/generated.yml pull
+    docker compose -f compose/generated.yml pull
 }
 
 bitcart_restart() {
@@ -97,13 +97,13 @@ try() {
 
 remove_host() {
     if [ -n "$(grep -w "$1$" /etc/hosts)" ]; then
-        try sed -ie "/[[:space:]]$1/d" /etc/hosts
+        try sudo sed -ie "/[[:space:]]$1/d" /etc/hosts
     fi
 }
 
 add_host() {
-    if [ -z "$(grep -P "[[:space:]]$2" /etc/hosts)" ]; then
-        try printf "%s\t%s\n" "$1" "$2" | sudo tee -a /etc/hosts >/dev/null
+    if [ -z "$(grep -E "[[:space:]]$2" /etc/hosts)" ]; then
+        try sudo printf "%s\t%s\n" "$1" "$2" | sudo tee -a /etc/hosts >/dev/null
     fi
 }
 
@@ -127,14 +127,19 @@ apply_local_modifications() {
 }
 
 container_name() {
+    deployment_name=${NAME:-compose}
+    echo "${deployment_name}-$1"
+}
+
+volume_name() {
     deployment_name=${NAME:-compose}
     echo "${deployment_name}_$1"
 }
 
 create_backup_volume() {
     backup_dir="/var/lib/docker/volumes/backup_datadir/_data"
     if [ ! -d "$backup_dir" ]; then
-        docker volume create backup_datadir
+        docker volume create backup_datadir >/dev/null 2>&1
     fi
 }
 
@@ -147,3 +152,28 @@ bitcart_restore_db() {
     bitcart_start database
     cat $1 | docker exec -i $(container_name "database_1") psql -U postgres
 }
+
+version() {
+    echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'
+}
+
+check_docker_compose() {
+    if [ ! -z "$(docker-compose --version 2>/dev/null | grep docker-compose)" ] || ! [[ $(docker compose version 2>/dev/null) ]] || [ $(version $(docker compose version --short)) -lt $(version "2.9.0") ]; then
+        install_docker_compose
+    fi
+}
+
+install_docker_compose() {
+    OS=$(uname -s)
+    ARCH=$(uname -m)
+    if [[ "$OS" == "Darwin" ]] && [[ "$ARCH" == "arm64" ]]; then
+        ARCH="aarch64"
+    fi
+    DOCKER_COMPOSE_DOWNLOAD="https://github.com/docker/compose/releases/latest/download/docker-compose-$OS-$ARCH"
+    echo "Trying to install docker-compose by downloading on $DOCKER_COMPOSE_DOWNLOAD ($(uname -m))"
+    sudo mkdir -p /usr/local/lib/docker/cli-plugins
+    sudo curl -L "$DOCKER_COMPOSE_DOWNLOAD" -o /usr/local/lib/docker/cli-plugins/docker-compose
+    sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose
+    # remove old docker-compose
+    sudo rm /usr/local/bin/docker-compose || true
+}