fix: resolve WAL archiving performance and memory issues

Taskfile.yml

@@ -381,34 +381,6 @@ tasks:
         build --dir . --file containers/Dockerfile.sidecar --platform linux/amd64 --platform linux/arm64
         publish --ref {{.SIDECAR_IMAGE_NAME}} --tags {{.IMAGE_VERSION}}
 
-  publish-barman-base:
-    desc: Build and publish a barman-cloud base container image
-    vars:
-      BARMAN_BASE_IMAGE_NAME: ghcr.io/{{.GITHUB_REPOSITORY}}-base{{if not (hasPrefix "refs/heads/main" .GITHUB_REF)}}-testing{{end}}
-      BARMAN_VERSION:
-        sh: grep "^barman" containers/sidecar-requirements.in | sed -E 's/.*==([^ ]+)/\1/'
-      BUILD_DATE:
-        sh: date +"%Y%m%d%H%M"
-    requires:
-      # We expect this to run in a GitHub workflow, so we put a few GitHub-specific vars here
-      # to prevent running this task locally by accident.
-      vars:
-        - CI
-        - GITHUB_REPOSITORY
-        - GITHUB_REF
-        - GITHUB_REF_NAME
-        - REGISTRY_USER
-        - REGISTRY_PASSWORD
-    env:
-      # renovate: datasource=git-refs depName=docker lookupName=https://github.com/purpleclay/daggerverse currentValue=main
-      DAGGER_DOCKER_SHA: ee12c1a4a2630e194ec20c5a9959183e3a78c192
-    cmds:
-      - >
-        dagger call -m github.com/purpleclay/daggerverse/docker@${DAGGER_DOCKER_SHA}
-        --registry ghcr.io --username $REGISTRY_USER --password env:REGISTRY_PASSWORD
-        build --dir . --file containers/Dockerfile.barmanbase --platform linux/amd64 --platform linux/arm64
-        publish --ref {{.BARMAN_BASE_IMAGE_NAME}} --tags "{{.BARMAN_VERSION}}-{{.BUILD_DATE}}"
-
   controller-gen:
     desc: Run controller-gen
     run: once

containers/Dockerfile.sidecar

@@ -33,22 +33,70 @@ COPY ../internal/ internal/
 RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/manager/main.go
 
-# Use plugin-barman-cloud-base to get the dependencies.
-# pip will build everything inside /usr, so we copy every file into a new
-# destination that will then be copied into the distroless container
-FROM ghcr.io/cloudnative-pg/plugin-barman-cloud-base:3.17.0-202601131704 AS pythonbuilder
-# Prepare a new /usr/ directory with the files we'll need in the final image
-RUN mkdir /new-usr/ && \
-    cp -r --parents /usr/local/lib/ /usr/lib/*-linux-gnu/ /usr/local/bin/ \
-    /new-usr/
-
-# Joint process
-# Now we put everything that was build from the origin into our
-# distroless container
-FROM gcr.io/distroless/python3-debian12:nonroot
+# Build Python virtualenv with all dependencies
+# Using virtualenv ensures bytecode is compiled with correct timestamps
+FROM debian:trixie-slim AS pythonbuilder
+WORKDIR /build
+
+# Install postgresql-common and setup pgdg repository first
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends postgresql-common && \
+    /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
+
+# Install build dependencies
+# After pgdg repo setup, this ensures we get updated versions from apt.postgresql.org
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        python3 \
+        python3-venv \
+        python3-dev \
+        build-essential \
+        libpq-dev \
+        liblz4-dev \
+        libsnappy-dev
+
+# Copy requirements
+COPY containers/sidecar-requirements.txt .
+
+# Create virtualenv and install dependencies
+# Compileall ensures all bytecode is freshly compiled with correct timestamps
+RUN python3 -m venv /venv && \
+    /venv/bin/pip install --upgrade pip setuptools wheel && \
+    /venv/bin/pip install --no-cache-dir -r sidecar-requirements.txt && \
+    /venv/bin/python -m compileall -q /venv
+
+# Download and extract runtime library packages and their dependencies
+# Using apt-cache to automatically resolve dependencies, filtering out packages
+# already present in the distroless base image.
+# Distroless package list from: https://github.com/GoogleContainerTools/distroless/blob/main/base/config.bzl
+# and https://github.com/GoogleContainerTools/distroless/blob/main/python3/config.bzl
+RUN mkdir -p /dependencies /build/downloads && \
+    cd /build/downloads && \
+    DISTROLESS_PACKAGES="libc6 libssl3t64 libzstd1 zlib1g libgcc-s1 libstdc++6 \
+        libbz2-1.0 libdb5.3t64 libexpat1 liblzma5 libsqlite3-0 libuuid1 \
+        libncursesw6 libtinfo6 libcom-err2 libcrypt1 libgssapi-krb5-2 \
+        libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 libnsl2 \
+        libreadline8t64 libtirpc3t64 libffi8 libpython3.13-minimal \
+        libpython3.13-stdlib python3.13-minimal python3.13-venv" && \
+    apt-cache depends --recurse --no-recommends --no-suggests \
+        --no-conflicts --no-breaks --no-replaces --no-enhances \
+        $DISTROLESS_PACKAGES 2>/dev/null | grep "^\w" | sort -u > /tmp/distroless.txt && \
+    apt-cache depends --recurse --no-recommends --no-suggests \
+        --no-conflicts --no-breaks --no-replaces --no-enhances \
+        libpq5 liblz4-1 libsnappy1v5 2>/dev/null | grep "^\w" | sort -u | \
+        grep -v -F -x -f /tmp/distroless.txt > /tmp/packages.txt && \
+    apt-get download $(cat /tmp/packages.txt) && \
+    for deb in *.deb; do \
+        dpkg -x "$deb" /dependencies; \
+    done
+
+# Final sidecar image
+# Using distroless base for minimal attack surface (no shell, no package manager)
+FROM gcr.io/distroless/python3-debian13:nonroot
 
 ENV SUMMARY="CloudNativePG Barman plugin" \
-    DESCRIPTION="Container image that provides the barman-cloud sidecar"
+    DESCRIPTION="Container image that provides the barman-cloud sidecar" \
+    PATH="/venv/bin:$PATH"
 
 LABEL summary="$SUMMARY" \
       description="$DESCRIPTION" \
@@ -60,7 +108,15 @@ LABEL summary="$SUMMARY" \
       version="" \
       release="1"
 
-COPY --from=pythonbuilder /new-usr/* /usr/
+# Copy virtualenv with pre-compiled bytecode
+COPY --from=pythonbuilder /venv /venv
+
+# Copy runtime libraries from extracted packages
+# All libraries are in /usr/lib/x86_64-linux-gnu
+COPY --from=pythonbuilder /dependencies/usr/lib /usr/lib
+
+# Copy Go manager binary
 COPY --from=gobuilder /workspace/manager /manager
+
 USER 26:26
 ENTRYPOINT ["/manager"]

renovate.json5

@@ -79,12 +79,6 @@
     enabled: false,
   },
   packageRules: [
-    {
-      matchPackageNames: [
-       'ghcr.io/cloudnative-pg/plugin-barman-cloud-base',
-      ],
-      versioning: 'loose',
-    },
     {
       matchDatasources: [
         'go',