Unify docker images

Remove unnecessary dependencies Add docker build platforms debug docker image debug docker image resolve cffi issue resolve cargo issue refine docker image Separate docker pipeline Add new pipeline Add new pipeline Add pull option debug matrix fix typo fix typo test matrix test matrix test matrix test matrix test matrix test matrix test matrix test matrix test metrixs test metrix test metrix test metrix test metrix test metrix test metrix test metrix test metrix test metrix
daya0576 · Dec 8, 2024 · 71ba974 · 71ba974
1 parent 14cf517
commit 71ba974
Show file tree

Hide file tree

Showing 7 changed files with 171 additions and 126 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -1,49 +1,46 @@
+# https://docs.docker.com/build/ci/github-actions/multi-platform/
+
 name: Publish Release
 
 on:
   workflow_dispatch:
   push:
     tags:
       - v**
+    branches:
+      - debug_docker_image
 
-jobs:
-  pre-deploy-test:
-    name: Unit tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip 
-          pip install poetry
-          poetry install --no-interaction --no-ansi
-
-      - name: Test with pytest
-        run: poetry run pytest tests
+env:
+  REGISTRY_IMAGE: daya0576/beaverhabits
 
-  docker:
-    needs: pre-deploy-test
+jobs:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [linux/amd64, linux/amd64/v3, linux/arm64]
+        dockerfile: [Dockerfile]
+        # include:
+        #   - platform: linux/arm/v7
+        #     dockerfile: Dockerfile.nobuildkit.arm32
 
     steps:
       - name: Prepare
         id: prep
         run: |
-          DOCKER_IMAGE=${{ secrets.DOCKER_USERNAME  }}/${GITHUB_REPOSITORY#*/}
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            VERSION=${GITHUB_REF#refs/tags/v}
-            TAGS="${DOCKER_IMAGE}:${VERSION},${DOCKER_IMAGE}:latest"
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.REGISTRY_IMAGE }}
 
-          fi
-          echo $TAGS
-          echo ::set-output name=tags::${TAGS}
+      - name: Checkout code
+        uses: actions/checkout@v4
 
-      - uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -53,40 +50,74 @@ jobs:
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
+          username: ${{ vars.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build and push
+      - name: Build and push by digest
+        id: build
         uses: docker/build-push-action@v6
         with:
-          platforms: linux/amd64,linux/amd64/v3,linux/arm64
-          push: true
-          tags: ${{ steps.prep.outputs.tags }}
+          platforms: ${{ matrix.platform }}
+          file: ${{ matrix.dockerfile }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache,mode=max
+
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"          
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
 
-  verify:
-    needs: docker
+  merge:
     runs-on: ubuntu-latest
+    needs:
+      - build
     steps:
-      - name: Pull and Test Container
-        env:
-          DOCKER_IMAGE: daya0576/beaverhabits
-          VERSION: latest
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.REGISTRY_IMAGE }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)          
+
+      - name: Inspect image
         run: |
-          docker pull ${DOCKER_IMAGE}:${VERSION}
-          docker run -d --name test_container ${DOCKER_IMAGE}:${VERSION}
-          sleep 3
-          CONTAINER_OUTPUT=$(docker logs test_container)
-          # Check if the container is still running
-          CONTAINER_STATUS=$(docker inspect -f '{{.State.Running}}' test_container)
-          if [ "${CONTAINER_STATUS}" != "true" ]; then
-            echo "The container is not running!"
-            exit 1
-          fi
-          # Check if the "Error" string is present in the container output
-          if echo "${CONTAINER_OUTPUT}" | grep -i -q "Error"; then
-            echo "Error found in container output!"
-            echo "${CONTAINER_OUTPUT}"
-            exit 1
-          fi
-          docker stop test_container
-          docker rm test_container
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} 
diff --git a/Dockerfile b/Dockerfile
@@ -26,21 +26,22 @@ ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
 FROM python-base AS builder-base
 RUN apt-get update \
     && apt-get install --no-install-recommends -y \
-        curl \
-        build-essential
+        build-essential \
+        curl
 
 # install poetry - respects $POETRY_VERSION & $POETRY_HOME
-RUN --mount=type=cache,target=/root/.cache \
-    curl -sSL https://install.python-poetry.org | python -
-# install dependency
-RUN python -m pip install --upgrade pip cffi
-
+RUN curl -sSL https://install.python-poetry.org | python3 -
 # copy project requirement files here to ensure they will be cached.
 WORKDIR $PYSETUP_PATH
 COPY poetry.lock pyproject.toml ./
-
 # install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
-RUN poetry install --no-dev
+RUN poetry install --only main
+
+# [Experimental] Remove unused nicegui libs
+ENV NICEGUI_LIB_PATH="/opt/pysetup/.venv/lib/python3.12/site-packages/nicegui/elements/lib"
+RUN rm -rf "$NICEGUI_LIB_PATH/mermaid/"
+RUN rm -rf "$NICEGUI_LIB_PATH/plotly/"
+RUN rm -rf "$NICEGUI_LIB_PATH/vanilla-jsoneditor/"
 
 
 ################################

diff --git a/Dockerfile.nobuildkit.arm32 b/Dockerfile.nobuildkit.arm32
@@ -1,41 +1,70 @@
-FROM python:3.12-slim AS builder
-
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \
-    apt-get install -y --no-install-recommends \
-    build-essential libffi-dev libssl-dev curl
-#    && rm -rf /var/lib/apt/lists/* \
-#    && apt-get clean
+FROM python:3.12-slim AS python-base
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    \
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    \
+    POETRY_VERSION=1.8.4 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    POETRY_NO_INTERACTION=1 \
+    \
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
+
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+
+################################
+# BUILDER-BASE
+# Used to build deps + create our virtual environment
+################################
+FROM python-base AS builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        build-essential libffi-dev libssl-dev curl
+
+# Installing a cryptography dependency which requires Rust.
+# TODO: Enable only for arm/v7
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
 ENV PATH=/root/.cargo/bin:$PATH
 ENV PKG_CONFIG_PATH=/usr/lib/arm-linux-gnueabihf/pkgconfig
 ENV OPENSSL_LIB_DIR=/usr/lib/arm-linux-gnueabihf
 ENV OPENSSL_INCLUDE_DIR=/usr/include/arm-linux-gnueabihf/openssl
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+RUN pip install cffi
+RUN pip install poetry
+# copy project requirement files here to ensure they will be cached.
+WORKDIR $PYSETUP_PATH
+COPY poetry.lock pyproject.toml ./
+# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
+RUN poetry install --only main
 
-RUN python -m pip install --upgrade pip
-# NiceGUI 1.4.20 - Make libsass optional 
-# RUN python -m pip install --upgrade libsass
-RUN python -m pip install --upgrade cffi
-COPY requirements.txt ./
-RUN mkdir /wheels
-RUN pip wheel --no-cache-dir -r requirements.txt -w /wheels
+# [Experimental] Remove unused nicegui libs
+ENV NICEGUI_LIB_PATH="/opt/pysetup/.venv/lib/python3.12/site-packages/nicegui/elements/lib"
+RUN rm -rf "$NICEGUI_LIB_PATH/mermaid/"
+RUN rm -rf "$NICEGUI_LIB_PATH/plotly/"
+RUN rm -rf "$NICEGUI_LIB_PATH/vanilla-jsoneditor/"
 
-FROM python:3.12-slim AS release
-COPY --from=builder /wheels /wheels
 
-LABEL maintainer="Henry Zhu <[email protected]>"
+################################
+# PRODUCTION
+# Final image used for runtime
+################################
+FROM python-base AS production
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
 
 WORKDIR /app
-COPY requirements.txt ./
-RUN pip install --no-cache-dir --no-index --find-links /wheels -r requirements.txt
-
 COPY start.sh .
 COPY beaverhabits ./beaverhabits
 COPY statics ./statics
 
 RUN chmod -R g+w /app && \
     chown -R nobody /app
-
 USER nobody
 
 CMD ["sh", "start.sh", "prd"]
diff --git a/beaverhabits/main.py b/beaverhabits/main.py
@@ -1,9 +1,9 @@
 import asyncio
-from contextlib import asynccontextmanager
 import logging
+from contextlib import asynccontextmanager
 
-from fastapi import FastAPI
 import sentry_sdk
+from fastapi import FastAPI
 
 from beaverhabits.app import crud
 
@@ -40,7 +40,7 @@ def read_root():
     return {"Hello": "World"}
 
 
-@app.get("/user/count")
+@app.get("/api/v1/users/count")
 async def user_count():
     return {"count": await crud.get_user_count()}
 

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -7,7 +7,7 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.12"
-nicegui = {extras = ["highcharts"], version = "^2.7.0"}
+nicegui = "^2.7.0"
 pydantic-settings = "^2.2.1"
 fastapi-users = {extras = ["sqlalchemy"], version = "^13.0.0"}
 uvicorn = {extras = ["standard"], version = "^0.28.0"}