feat(fips): build grpcio from source against system OpenSSL and drop its scanner waiver

.github/ci-coverage-allowlist.yml

@@ -111,3 +111,11 @@ dockerfiles:
       An example image under cookbook/ that is documentation rather than a shipped artifact
     paths:
       - cookbook/litellm-ollama-docker-image/Dockerfile
+  - reason: >-
+      The FIPS image builds only with a Chainguard apk token (--secret id=cgr-token) that no
+      pull request job holds. Its build-time bundled-crypto gate
+      (docker/scan_bundled_crypto.sh) is exercised by
+      tests/integration/configuration/test_fips_bundled_crypto_scan.py in the CircleCI
+      management shard rather than by a docker build job
+    paths:
+      - docker/Dockerfile.fips

docker/Dockerfile.fips

@@ -0,0 +1,207 @@
+# syntax=docker/dockerfile:1.7
+
+# FIPS 140-3 variant (workstream A of the FIPS roadmap). Builds on
+# chainguard-base-fips so python/node/openssl resolve to Chainguard's
+# FIPS-provider-linked packages from the image's pinned virtualapk repo.
+# Build requires a Chainguard token:
+#   CGR_TOKEN=$(chainctl auth token --audience apk.cgr.dev) \
+#   docker build --secret id=cgr-token,env=CGR_TOKEN -f docker/Dockerfile.fips .
+ARG LITELLM_BUILD_IMAGE=cgr.dev/berri.ai/chainguard-base-fips@sha256:4f23c1a31be6a0778c4851c54474c7630b2feda7d113c7b1b5e7b1def8c89b04
+
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/berri.ai/chainguard-base-fips@sha256:4f23c1a31be6a0778c4851c54474c7630b2feda7d113c7b1b5e7b1def8c89b04
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
+# UI builder only emits static JS assets; no binary from it reaches the final
+# image, so it does not need a FIPS base.
+ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
+# Checksum from https://www.pgbouncer.org/downloads/
+ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
+
+FROM $UV_IMAGE AS uvbin
+
+FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
+ARG PGBOUNCER_VERSION
+ARG PGBOUNCER_SHA256
+USER root
+# openssl-dev here is the FIPS-configured system OpenSSL, so pgbouncer links
+# libcrypto.so.3 and inherits the FIPS provider at runtime.
+RUN --mount=type=secret,id=cgr-token \
+    HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
+    apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
+WORKDIR /build
+RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
+    echo "${PGBOUNCER_SHA256}  pgbouncer.tar.gz" | sha256sum -c - && \
+    tar xzf pgbouncer.tar.gz --strip-components=1 && \
+    ./configure --prefix=/usr/local --with-openssl=/usr && \
+    make -j"$(nproc)" pgbouncer && \
+    install -m 0755 pgbouncer /usr/local/bin/pgbouncer
+
+# grpcio's PyPI wheel statically bundles BoringSSL, which is not the validated
+# module. Rebuild it from source against the FIPS system OpenSSL instead
+# (GRPC_PYTHON_BUILD_SYSTEM_OPENSSL), so cygrpc links libcrypto.so.3 like
+# every other consumer in the image. The version is read from uv.lock through
+# its own tiny stage so this long compile is cached until grpcio itself moves.
+FROM $LITELLM_BUILD_IMAGE AS grpcio-version
+USER root
+COPY uv.lock /uv.lock
+RUN awk '/^name = "grpcio"$/ { found = 1 } found && /^version = / { gsub(/"/, "", $3); print $3; exit }' /uv.lock \
+    > /grpcio-version && test -s /grpcio-version
+
+FROM $LITELLM_BUILD_IMAGE AS grpcio-builder
+USER root
+RUN --mount=type=secret,id=cgr-token \
+    HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
+    apk add --no-cache build-base python-3.13 python-3.13-dev openssl-dev zlib-dev binutils
+COPY --from=uvbin /uv /usr/local/bin/uv
+COPY --from=grpcio-version /grpcio-version /grpcio-version
+ENV UV_PYTHON_DOWNLOADS=0
+RUN uv venv --seed --python python3.13 /grpcio-venv && \
+    GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
+    GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 \
+    GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS="$(nproc)" \
+    /grpcio-venv/bin/python -m pip wheel --no-binary grpcio --no-deps --no-cache-dir \
+        --wheel-dir /wheels "grpcio==$(cat /grpcio-version)" && \
+    /grpcio-venv/bin/python -m zipfile -e /wheels/grpcio-*.whl /wheels/check && \
+    objdump -p /wheels/check/grpc/_cython/cygrpc*.so | grep -q 'NEEDED.*libcrypto\.so\.3' && \
+    ! grep -aq BoringSSL /wheels/check/grpc/_cython/cygrpc*.so && \
+    rm -rf /wheels/check
+
+FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder
+
+ENV NEXT_TELEMETRY_DISABLED=1 \
+    npm_config_fund=false \
+    npm_config_audit=false
+
+WORKDIR /ui
+
+COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./
+RUN --mount=type=cache,target=/root/.npm npm ci --prefer-offline
+
+COPY ui/litellm-dashboard/ ./
+RUN npm run build
+
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+WORKDIR /app
+USER root
+
+COPY --from=uvbin /uv /usr/local/bin/uv
+COPY --from=uvbin /uvx /usr/local/bin/uvx
+
+RUN --mount=type=secret,id=cgr-token \
+    HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
+    apk add --no-cache \
+    bash \
+    binutils \
+    gcc \
+    python-3.13 \
+    python-3.13-dev \
+    openssl \
+    openssl-dev \
+    nodejs \
+    npm \
+    libsndfile
+
+# UV_PYTHON_DOWNLOADS=0: uv must never substitute a standalone CPython with
+# its own bundled OpenSSL for the FIPS-linked system python.
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
+    PATH="/app/.venv/bin:${PATH}"
+
+COPY pyproject.toml uv.lock ./
+COPY enterprise/pyproject.toml enterprise/
+COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
+
+RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+    --extra proxy \
+    --extra proxy-runtime \
+    --extra extra_proxy \
+    --extra semantic-router \
+    --extra saml \
+    --extra bedrock-realtime \
+    --python python3.13
+
+COPY . .
+
+RUN rm -rf litellm/proxy/_experimental/out
+COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
+
+RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+RUN uv sync --frozen --no-default-groups --no-editable \
+    --extra proxy \
+    --extra proxy-runtime \
+    --extra extra_proxy \
+    --extra semantic-router \
+    --extra saml \
+    --extra bedrock-realtime \
+    --python python3.13
+
+COPY --from=grpcio-builder /wheels /wheels
+RUN uv pip install --python /app/.venv/bin/python --no-deps --reinstall --no-cache /wheels/grpcio-*.whl && \
+    strip --strip-unneeded /app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc*.so && \
+    rm -rf /wheels
+
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
+
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
+    sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
+
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+USER root
+
+# Unlike the non-FIPS images, the public Wolfi repo is deliberately NOT added
+# here: apk in this image may only resolve FIPS-linked packages from the
+# pinned Chainguard repo.
+RUN --mount=type=secret,id=cgr-token \
+    HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
+    apk add --no-cache bash openssl openssl-fips-test tzdata nodejs python-3.13 libsndfile libevent
+COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
+
+WORKDIR /app
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
+    PRISMA_OFFLINE_MODE=true
+
+COPY --from=builder /app/.venv /app/.venv
+COPY --from=builder /app/docker /app/docker
+COPY --from=builder /app/schema.prisma /app/schema.prisma
+COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
+COPY --from=builder /app/enterprise /app/enterprise
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+COPY --from=builder /opt/prisma /opt/prisma
+
+RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
+
+# Build-time FIPS assertions. binutils (objdump) is installed only in this
+# throwaway stage; the final image below is FROM runtime and never sees it.
+FROM runtime AS fips-check
+RUN --mount=type=secret,id=cgr-token \
+    HTTP_AUTH="basic:virtualapk.cgr.dev:user:$(cat /run/secrets/cgr-token)" \
+    apk add --no-cache binutils
+RUN openssl-fips-test
+RUN python -c "import grpc._cython.cygrpc" && \
+    objdump -p /app/.venv/lib/python3.13/site-packages/grpc/_cython/cygrpc*.so | grep -q 'NEEDED.*libcrypto\.so\.3'
+RUN chmod +x docker/scan_bundled_crypto.sh && \
+    docker/scan_bundled_crypto.sh /app/.venv /opt/prisma /usr/local/bin
+
+FROM runtime AS final
+
+# Forces the fips-check stage to build and records its evidence in the image.
+COPY --from=fips-check /fips-scan-report.txt /fips-scan-report.txt
+
+EXPOSE 4000/tcp
+
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
+CMD ["--port", "4000"]