diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index 90ede5a653f..bda770845a3 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -4,11 +4,13 @@ on: pull_request: branches: - main + - carto/main - litellm_internal_staging - litellm_oss_branch - "litellm_**" paths: - docker/Dockerfile.non_root + - tests/proxy_migration_tests/test_offline_image_migration.py - uv.lock - ui/litellm-dashboard/package-lock.json - .github/workflows/image-scan.yml @@ -51,6 +53,23 @@ jobs: - name: Build runtime image run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} . + # The prisma bake must migrate a fresh DB with no egress as an arbitrary + # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem). + # `docker run` as the default uid with network hides a broken bake because + # the migration entrypoint exits 0 even when it applied nothing; asserting + # the schema was created is what catches it. + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Verify offline migration as a non-root uid + env: + LITELLM_IMAGE: litellm-image-scan:${{ github.sha }} + run: | + python -m pip install "pytest==9.0.3" + python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v + # Scans the whole shipped artifact: OS/apk plus every language package # baked into the image, including ones no lockfile declares (e.g. prisma's # vendored node engine) that osv-scan cannot see. osv-scan stays the fast diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 37f4d01778c..8e05f312ba0 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -54,8 +54,6 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ PATH="/app/.venv/bin:${PATH}" \ LITELLM_NON_ROOT=true \ - PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ - PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x,linux-arm64-openssl-3.0.x" \ XDG_CACHE_HOME=/app/.cache # Copy dependency metadata first for layer caching @@ -107,7 +105,9 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ --python python3; \ fi -RUN prisma generate --schema=./schema.prisma +RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + npm_config_cache=/root/.npm \ + prisma generate --schema=./schema.prisma RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh @@ -128,8 +128,6 @@ RUN for i in 1 2 3; do \ # the rest of the builder's /app is source and build metadata that must not # ship (manifest-scanning tools attribute everything in it to this image). # entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path. -# Prisma caches live under /app/.cache here (XDG_CACHE_HOME / -# PRISMA_BINARY_CACHE_DIR) so the runtime prisma generate finds them. COPY --from=builder /app/.venv /app/.venv COPY --from=builder /app/docker /app/docker COPY --from=builder /app/schema.prisma /app/schema.prisma @@ -138,21 +136,36 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr # working directory on sys.path; litellm/proxy/hooks resolves # enterprise.enterprise_hooks from it) COPY --from=builder /app/enterprise /app/enterprise -COPY --from=builder /app/.cache /app/.cache +COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras +# Prisma CLI + engines are baked under /opt/prisma, a fixed path every runtime +# uid can read and that no cache volume mount shadows (unlike /app/.cache or +# $HOME/.cache under readOnlyRootFilesystem + emptyDir or arbitrary-uid setups). +# PRISMA_CLI_QUERY_ENGINE_TYPE=binary makes the CLI use the baked binary query +# engine directly, so `prisma migrate deploy` on a fresh database needs no npm +# and no network access; without it the CLI looks for the library engine, which +# prisma stopped baking, and falls back to a download that fails offline or as a +# non-writable uid (#33650, #24554). +COPY --from=builder /opt/prisma /opt/prisma COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets +# XDG_CACHE_HOME is intentionally left unset so it falls back to $HOME/.cache +# (/app/.cache, writable by the runtime uid). The prisma bake at the read-only +# /opt/prisma is anchored by PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH, so +# nothing needs XDG to point there; pointing it at the read-only bake would +# deny any XDG-aware library that writes a cache at runtime. ENV PATH="/app/.venv/bin:${PATH}" \ - PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ + PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ + PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ + PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ HOME=/app \ LITELLM_NON_ROOT=true \ - XDG_CACHE_HOME=/app/.cache \ PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ PRISMA_HIDE_UPDATE_MESSAGE=1 \ PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \ PRISMA_OFFLINE_MODE=true -RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \ +RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/ui && \ chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \ PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ chown -R nobody:nogroup "$PRISMA_PATH" && \ @@ -165,12 +178,14 @@ RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \ chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \ - chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets /app/.cache + chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ + chmod -R a+rX /opt/prisma && \ + test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ + test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \ + ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1 USER 65534 -RUN prisma generate --schema=./schema.prisma - EXPOSE 4000/tcp ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] diff --git a/tests/proxy_migration_tests/test_offline_image_migration.py b/tests/proxy_migration_tests/test_offline_image_migration.py new file mode 100644 index 00000000000..8ed88278e69 --- /dev/null +++ b/tests/proxy_migration_tests/test_offline_image_migration.py @@ -0,0 +1,143 @@ +"""Image-level regression net for the prisma bake in the shipped runtime image. + +Boots a built image's migration entrypoint the way an OpenShift / air-gapped +deployment does (an internal-only network with no egress, an arbitrary non-root +uid in GID 0) against a brand-new Postgres, and asserts the schema was created. + +This catches the whole failure class, not one symptom: a bake that only works +under `docker run` as the default uid with network still passes every existing +check, because the migration entrypoint exits 0 even when it applied nothing. +Asserting the table count is what turns that silent success into a hard fail. + +Gated on LITELLM_IMAGE (the tag of the image to exercise) so it is skipped in +the normal unit-test run and exercised only where an image has been built (the +image-scan workflow). Requires a working docker CLI. +""" + +import shutil +import subprocess +import uuid + +import os +import pytest + +IMAGE = os.getenv("LITELLM_IMAGE") +POSTGRES_IMAGE = os.getenv("LITELLM_TEST_POSTGRES_IMAGE", "postgres:16-alpine") +MIN_TABLES = int(os.getenv("LITELLM_TEST_MIN_TABLES", "20")) +NON_ROOT_UID = "12345:0" # arbitrary uid in GID 0, as OpenShift restricted-v2 assigns + +pytestmark = [ + pytest.mark.skipif(IMAGE is None, reason="requires a built image (set LITELLM_IMAGE)"), + pytest.mark.skipif(shutil.which("docker") is None, reason="requires the docker CLI"), +] + + +def _docker(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ["docker", *args], capture_output=True, text=True, check=check + ) + + +@pytest.fixture() +def offline_postgres(): + """A fresh Postgres reachable only over an internal-only (no egress) network. + + Yields (network_name, postgres_host). Both are torn down afterwards. + """ + run_id = f"offlinemig-{uuid.uuid4().hex[:8]}" + network = f"{run_id}-net" + pg = f"{run_id}-pg" + + # Pull Postgres while egress still exists; the internal network below has none. + _docker("pull", "--quiet", POSTGRES_IMAGE) + # --internal => containers on this network cannot reach the internet, so a + # prisma engine download (binaries.prisma.sh / npm) fails instead of masking + # a non-self-contained bake. + _docker("network", "create", "--internal", network) + try: + _docker( + "run", "-d", "--name", pg, "--network", network, + "-e", "POSTGRES_PASSWORD=pw", "-e", "POSTGRES_DB=litellm", + POSTGRES_IMAGE, + ) + _wait_until_ready(pg) + yield network, pg + finally: + _docker("rm", "-f", pg, check=False) + _docker("network", "rm", network, check=False) + + +def _wait_until_ready(pg: str, attempts: int = 60) -> None: + for _ in range(attempts): + running = _docker( + "ps", "--filter", f"name={pg}", "--filter", "status=running", + "--format", "{{.Names}}", check=False, + ).stdout + if pg not in running: + logs = _docker("logs", pg, check=False).stdout + _docker("logs", pg, check=False).stderr + pytest.fail(f"postgres container is not running:\n{logs}") + ready = _docker( + "exec", pg, "pg_isready", "-U", "postgres", "-d", "litellm", check=False + ) + if ready.returncode == 0: + return + subprocess.run(["sleep", "1"]) + pytest.fail(f"postgres never became ready after {attempts}s") + + +def _table_count(pg: str) -> int: + result = _docker( + "exec", pg, "psql", "-U", "postgres", "-d", "litellm", "-tAc", + "SELECT count(*) FROM information_schema.tables WHERE table_schema='public';", + ) + return int(result.stdout.strip() or "0") + + +def test_migration_offline_as_non_root_uid(offline_postgres): + """The migration entrypoint creates the full schema offline as an arbitrary uid. + + Reproduces the OpenShift / air-gapped failure: on the pre-fix image the + migration exits 0 having created 0 tables (every DB endpoint then 500s on + missing columns); a self-contained bake creates the full schema. + """ + network, pg = offline_postgres + assert IMAGE is not None + + migrate = _docker( + "run", "--rm", "--network", network, "--user", NON_ROOT_UID, + "-e", f"DATABASE_URL=postgresql://postgres:pw@{pg}:5432/litellm", + "-e", "LITELLM_MASTER_KEY=sk-offline-migration-test", + "-e", "DISABLE_SCHEMA_UPDATE=false", + "-w", "/app", "--entrypoint", "python", + IMAGE, "litellm/proxy/prisma_migration.py", + check=False, + ) + tables = _table_count(pg) + + assert migrate.returncode == 0, ( + f"migration entrypoint exited {migrate.returncode} offline as uid {NON_ROOT_UID}\n" + f"stdout:\n{migrate.stdout}\nstderr:\n{migrate.stderr}" + ) + assert tables >= MIN_TABLES, ( + f"only {tables} tables created (need >= {MIN_TABLES}) offline as uid {NON_ROOT_UID}. " + "The prisma bake is not self-contained: it needs a runtime download or a " + "writable HOME/cache, so OpenShift and air-gapped deployments start on an " + f"empty database.\nstdout:\n{migrate.stdout}\nstderr:\n{migrate.stderr}" + ) + + +def test_runtime_cache_env_not_read_only(): + """No runtime cache env var may point at the world-read-only /opt/prisma bake. + + /opt/prisma is baked `a+rX` (no write). Pointing XDG_CACHE_HOME (or any cache + var an XDG-aware library honours) there would deny writes for every uid, so + guard against a future edit reintroducing that. + """ + assert IMAGE is not None + env = _docker("run", "--rm", "--entrypoint", "env", IMAGE).stdout + offenders = [ + line for line in env.splitlines() + if line.startswith(("XDG_CACHE_HOME=", "XDG_DATA_HOME=", "HOME=")) + and line.split("=", 1)[1].startswith("/opt/prisma") + ] + assert not offenders, f"cache/home env points at the read-only bake: {offenders}"