fix(docker): resolve LiteLLM startup failure on self-hosted dedicated deployments

.github/workflows/image-scan.yml

@@ -4,11 +4,13 @@ on:
   pull_request:
     branches:
       - main
+      - carto/main
       - litellm_internal_staging
       - litellm_oss_branch
       - "litellm_**"
     paths:
       - docker/Dockerfile.non_root
+      - tests/proxy_migration_tests/test_offline_image_migration.py
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
       - .github/workflows/image-scan.yml
@@ -51,6 +53,23 @@ jobs:
       - name: Build runtime image
         run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} .
 
+      # The prisma bake must migrate a fresh DB with no egress as an arbitrary
+      # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem).
+      # `docker run` as the default uid with network hides a broken bake because
+      # the migration entrypoint exits 0 even when it applied nothing; asserting
+      # the schema was created is what catches it.
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify offline migration as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
       # Scans the whole shipped artifact: OS/apk plus every language package
       # baked into the image, including ones no lockfile declares (e.g. prisma's
       # vendored node engine) that osv-scan cannot see. osv-scan stays the fast

docker/Dockerfile.non_root

@@ -54,8 +54,6 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
-    PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x,linux-arm64-openssl-3.0.x" \
     XDG_CACHE_HOME=/app/.cache
 
 # Copy dependency metadata first for layer caching
@@ -107,7 +105,9 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --python python3; \
     fi
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -128,8 +128,6 @@ RUN for i in 1 2 3; do \
 # the rest of the builder's /app is source and build metadata that must not
 # ship (manifest-scanning tools attribute everything in it to this image).
 # entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path.
-# Prisma caches live under /app/.cache here (XDG_CACHE_HOME /
-# PRISMA_BINARY_CACHE_DIR) so the runtime prisma generate finds them.
 COPY --from=builder /app/.venv /app/.venv
 COPY --from=builder /app/docker /app/docker
 COPY --from=builder /app/schema.prisma /app/schema.prisma
@@ -138,21 +136,36 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
 # working directory on sys.path; litellm/proxy/hooks resolves
 # enterprise.enterprise_hooks from it)
 COPY --from=builder /app/enterprise /app/enterprise
-COPY --from=builder /app/.cache /app/.cache
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+# Prisma CLI + engines are baked under /opt/prisma, a fixed path every runtime
+# uid can read and that no cache volume mount shadows (unlike /app/.cache or
+# $HOME/.cache under readOnlyRootFilesystem + emptyDir or arbitrary-uid setups).
+# PRISMA_CLI_QUERY_ENGINE_TYPE=binary makes the CLI use the baked binary query
+# engine directly, so `prisma migrate deploy` on a fresh database needs no npm
+# and no network access; without it the CLI looks for the library engine, which
+# prisma stopped baking, and falls back to a download that fails offline or as a
+# non-writable uid (#33650, #24554).
+COPY --from=builder /opt/prisma /opt/prisma
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
 COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
 
+# XDG_CACHE_HOME is intentionally left unset so it falls back to $HOME/.cache
+# (/app/.cache, writable by the runtime uid). The prisma bake at the read-only
+# /opt/prisma is anchored by PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH, so
+# nothing needs XDG to point there; pointing it at the read-only bake would
+# deny any XDG-aware library that writes a cache at runtime.
 ENV PATH="/app/.venv/bin:${PATH}" \
-    PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
+    PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
     HOME=/app \
     LITELLM_NON_ROOT=true \
-    XDG_CACHE_HOME=/app/.cache \
     PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
     PRISMA_HIDE_UPDATE_MESSAGE=1 \
     PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
     PRISMA_OFFLINE_MODE=true
 
-RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
+RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/ui && \
     chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup "$PRISMA_PATH" && \
@@ -165,12 +178,14 @@ RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
     [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \
     chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
     [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \
-    chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets /app/.cache
+    chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
+    chmod -R a+rX /opt/prisma && \
+    test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1
 
 USER 65534
 
-RUN prisma generate --schema=./schema.prisma
-
 EXPOSE 4000/tcp
 
 ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]

tests/proxy_migration_tests/test_offline_image_migration.py

@@ -0,0 +1,143 @@
+"""Image-level regression net for the prisma bake in the shipped runtime image.
+
+Boots a built image's migration entrypoint the way an OpenShift / air-gapped
+deployment does (an internal-only network with no egress, an arbitrary non-root
+uid in GID 0) against a brand-new Postgres, and asserts the schema was created.
+
+This catches the whole failure class, not one symptom: a bake that only works
+under `docker run` as the default uid with network still passes every existing
+check, because the migration entrypoint exits 0 even when it applied nothing.
+Asserting the table count is what turns that silent success into a hard fail.
+
+Gated on LITELLM_IMAGE (the tag of the image to exercise) so it is skipped in
+the normal unit-test run and exercised only where an image has been built (the
+image-scan workflow). Requires a working docker CLI.
+"""
+
+import shutil
+import subprocess
+import uuid
+
+import os
+import pytest
+
+IMAGE = os.getenv("LITELLM_IMAGE")
+POSTGRES_IMAGE = os.getenv("LITELLM_TEST_POSTGRES_IMAGE", "postgres:16-alpine")
+MIN_TABLES = int(os.getenv("LITELLM_TEST_MIN_TABLES", "20"))
+NON_ROOT_UID = "12345:0"  # arbitrary uid in GID 0, as OpenShift restricted-v2 assigns
+
+pytestmark = [
+    pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"),
+    pytest.mark.skipif(shutil.which("docker") is None, reason="requires the docker CLI"),
+]
+
+
+def _docker(*args: str, check: bool = True) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        ["docker", *args], capture_output=True, text=True, check=check
+    )
+
+
+@pytest.fixture()
+def offline_postgres():
+    """A fresh Postgres reachable only over an internal-only (no egress) network.
+
+    Yields (network_name, postgres_host). Both are torn down afterwards.
+    """
+    run_id = f"offlinemig-{uuid.uuid4().hex[:8]}"
+    network = f"{run_id}-net"
+    pg = f"{run_id}-pg"
+
+    # Pull Postgres while egress still exists; the internal network below has none.
+    _docker("pull", "--quiet", POSTGRES_IMAGE)
+    # --internal => containers on this network cannot reach the internet, so a
+    # prisma engine download (binaries.prisma.sh / npm) fails instead of masking
+    # a non-self-contained bake.
+    _docker("network", "create", "--internal", network)
+    try:
+        _docker(
+            "run", "-d", "--name", pg, "--network", network,
+            "-e", "POSTGRES_PASSWORD=pw", "-e", "POSTGRES_DB=litellm",
+            POSTGRES_IMAGE,
+        )
+        _wait_until_ready(pg)
+        yield network, pg
+    finally:
+        _docker("rm", "-f", pg, check=False)
+        _docker("network", "rm", network, check=False)
+
+
+def _wait_until_ready(pg: str, attempts: int = 60) -> None:
+    for _ in range(attempts):
+        running = _docker(
+            "ps", "--filter", f"name={pg}", "--filter", "status=running",
+            "--format", "{{.Names}}", check=False,
+        ).stdout
+        if pg not in running:
+            logs = _docker("logs", pg, check=False).stdout + _docker("logs", pg, check=False).stderr
+            pytest.fail(f"postgres container is not running:\n{logs}")
+        ready = _docker(
+            "exec", pg, "pg_isready", "-U", "postgres", "-d", "litellm", check=False
+        )
+        if ready.returncode == 0:
+            return
+        subprocess.run(["sleep", "1"])
+    pytest.fail(f"postgres never became ready after {attempts}s")
+
+
+def _table_count(pg: str) -> int:
+    result = _docker(
+        "exec", pg, "psql", "-U", "postgres", "-d", "litellm", "-tAc",
+        "SELECT count(*) FROM information_schema.tables WHERE table_schema='public';",
+    )
+    return int(result.stdout.strip() or "0")
+
+
+def test_migration_offline_as_non_root_uid(offline_postgres):
+    """The migration entrypoint creates the full schema offline as an arbitrary uid.
+
+    Reproduces the OpenShift / air-gapped failure: on the pre-fix image the
+    migration exits 0 having created 0 tables (every DB endpoint then 500s on
+    missing columns); a self-contained bake creates the full schema.
+    """
+    network, pg = offline_postgres
+    assert IMAGE is not None
+
+    migrate = _docker(
+        "run", "--rm", "--network", network, "--user", NON_ROOT_UID,
+        "-e", f"DATABASE_URL=postgresql://postgres:pw@{pg}:5432/litellm",
+        "-e", "LITELLM_MASTER_KEY=sk-offline-migration-test",
+        "-e", "DISABLE_SCHEMA_UPDATE=false",
+        "-w", "/app", "--entrypoint", "python",
+        IMAGE, "litellm/proxy/prisma_migration.py",
+        check=False,
+    )
+    tables = _table_count(pg)
+
+    assert migrate.returncode == 0, (
+        f"migration entrypoint exited {migrate.returncode} offline as uid {NON_ROOT_UID}\n"
+        f"stdout:\n{migrate.stdout}\nstderr:\n{migrate.stderr}"
+    )
+    assert tables >= MIN_TABLES, (
+        f"only {tables} tables created (need >= {MIN_TABLES}) offline as uid {NON_ROOT_UID}. "
+        "The prisma bake is not self-contained: it needs a runtime download or a "
+        "writable HOME/cache, so OpenShift and air-gapped deployments start on an "
+        f"empty database.\nstdout:\n{migrate.stdout}\nstderr:\n{migrate.stderr}"
+    )
+
+
+def test_runtime_cache_env_not_read_only():
+    """No runtime cache env var may point at the world-read-only /opt/prisma bake.
+
+    /opt/prisma is baked `a+rX` (no write). Pointing XDG_CACHE_HOME (or any cache
+    var an XDG-aware library honours) there would deny writes for every uid, so
+    guard against a future edit reintroducing that.
+    """
+    assert IMAGE is not None
+    env = _docker("run", "--rm", "--entrypoint", "env", IMAGE).stdout
+    offenders = [
+        line for line in env.splitlines()
+        if line.startswith(("XDG_CACHE_HOME=", "XDG_DATA_HOME=", "HOME="))
+        and line.split("=", 1)[1].startswith("/opt/prisma")
+    ]
+    assert not offenders, f"cache/home env points at the read-only bake: {offenders}"