From 3c500f20f8ec1965eaf40dfd70fb92e37f0c6b27 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 13:07:23 -0700 Subject: [PATCH 1/8] feat(lens): coordinate worker versions and bundled installs --- .github/workflows/lens-worker.yml | 2 +- Dockerfile | 2 + backend/Dockerfile | 2 + deploy/lens/Dockerfile | 4 +- deploy/lens/README.md | 68 ++++++++++++++++- deploy/lens/compose.build.yaml | 2 + deploy/lens/compose.yaml | 2 +- deploy/lens/config.yaml | 7 ++ deploy/lens/stack.yaml | 71 ++++++++++++++++++ docker/Dockerfile.database | 2 + docker/Dockerfile.non_root | 2 + .../litellm/templates/backend/deployment.yaml | 2 + helm/litellm/templates/lens/deployment.yaml | 72 ++++++++++++++++++ helm/litellm/tests/lens_worker_tests.yaml | 73 +++++++++++++++++++ helm/litellm/values.yaml | 22 ++++++ litellm/proxy/_lazy_openapi_snapshot.json | 5 ++ litellm/proxy/lens/endpoints.py | 10 ++- litellm/proxy/lens/models.py | 1 + litellm/proxy/lens/release.py | 28 +++++++ litellm/proxy/lens/worker.py | 33 ++++++--- tests/proxy_behavior/lens/test_lifecycle.py | 4 +- tests/unit/proxy/lens/test_endpoints.py | 22 +++++- tests/unit/proxy/lens/test_release.py | 39 ++++++++++ tests/unit/proxy/lens/test_worker.py | 19 +++++ .../worker/WorkerDialog.integration.test.tsx | 7 +- .../lens/setup/worker/WorkerInstall.tsx | 27 ++++++- .../lens/setup/worker/workerCommand.ts | 9 +-- .../lens/setup/worker/workerSchema.test.ts | 11 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 + 29 files changed, 510 insertions(+), 41 deletions(-) create mode 100644 deploy/lens/config.yaml create mode 100644 deploy/lens/stack.yaml create mode 100644 helm/litellm/templates/lens/deployment.yaml create mode 100644 helm/litellm/tests/lens_worker_tests.yaml create mode 100644 litellm/proxy/lens/release.py create mode 100644 tests/unit/proxy/lens/test_release.py diff --git a/.github/workflows/lens-worker.yml b/.github/workflows/lens-worker.yml index 54ec2593ed88..cf47eb2edb26 100644 --- a/.github/workflows/lens-worker.yml +++ b/.github/workflows/lens-worker.yml @@ -34,7 +34,7 @@ jobs: with: persist-credentials: false - name: Build Lens worker - run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} . + run: docker build --build-arg LITELLM_RELEASE_TAG=sha-${{ github.sha }} -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} . - name: Verify standalone imports with a read-only filesystem run: | docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \ diff --git a/Dockerfile b/Dockerfile index 4dcecf3ea3d6..35dbaa4d41bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -116,6 +116,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ # Runtime stage FROM $LITELLM_RUNTIME_IMAGE AS runtime +ARG LITELLM_RELEASE_TAG="" +ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} USER root diff --git a/backend/Dockerfile b/backend/Dockerfile index 59f836b55f8f..dfff6e71a468 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -71,6 +71,8 @@ RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component # ---------- Runtime ---------- FROM $LITELLM_RUNTIME_IMAGE AS runtime +ARG LITELLM_RELEASE_TAG="" +ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} USER root diff --git a/deploy/lens/Dockerfile b/deploy/lens/Dockerfile index f684940e9a8a..b1142fe4d71c 100644 --- a/deploy/lens/Dockerfile +++ b/deploy/lens/Dockerfile @@ -1,7 +1,9 @@ FROM python:3.12-slim +ARG LITELLM_RELEASE_TAG="" +ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} WORKDIR /app RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7 -COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py /app/lens/ +COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/ COPY litellm/proxy/lens/prompts/ /app/lens/prompts/ USER 65532:65532 CMD ["python", "-m", "lens.worker"] diff --git a/deploy/lens/README.md b/deploy/lens/README.md index 4f78b7bfb59d..fb658f32059e 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -2,7 +2,58 @@ Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`) -## Start a worker +## Install the release stack + +Each stable, RC, and dev release containing Lens publishes the worker at the same version on GHCR and Docker Hub. Use the [LiteLLM releases page](https://github.com/BerriAI/litellm/releases) to select a version that includes the coordinated worker release + +For a new local installation, install Docker with Compose, download the two release files, and create a private environment file. Replace `X.Y.Z` with the release version, without `v` (RCs use `X.Y.Z-rc.N`) + +```bash +mkdir litellm-lens +cd litellm-lens +LENS_RELEASE=X.Y.Z +curl -fSLo compose.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LENS_RELEASE}/deploy/lens/stack.yaml" +curl -fSLo config.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LENS_RELEASE}/deploy/lens/config.yaml" +umask 077 +printf 'LITELLM_VERSION=%s\nLITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' \ + "$LENS_RELEASE" "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +docker compose up -d +``` + +Open `http://localhost:4000/ui/`, log in as `admin` with `LITELLM_MASTER_KEY` from `.env`, and add a model in the dashboard. In Lens, select **Connect worker**, choose that model and a monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?**, copy the worker token, and add `LENS_WORKER_TOKEN=` to `.env` + +```bash +docker compose --profile lens up -d +``` + +The stack starts LiteLLM, PostgreSQL, ClickHouse, and the worker from published images. The dashboard shows **Worker connected**. The worker has a limited token, no database credentials, and no provider keys. The stack exposes only the dashboard on localhost; use your normal ingress and managed databases for a public production deployment + +Keep `.env` private and preserve its salt key. Keep both named database volumes. To upgrade, wait for active investigations to finish, stop the worker, change only `LITELLM_VERSION`, then pull and recreate the stack: + +```bash +docker compose --profile lens stop lens-worker +# Update LITELLM_VERSION in .env to the new release +docker compose --profile lens pull +docker compose --profile lens up -d +``` + +This preserves your investigations, findings, model credentials, and worker token. Never use `down -v` during an upgrade. If moving from an existing installation, keep its databases and add the standalone worker instead of creating an empty replacement stack + +## Helm + +The componentized `helm/litellm` chart includes an optional Lens worker. Configure PostgreSQL and ClickHouse as usual, install the chart, then obtain a limited worker token from Lens setup. Store it in a Kubernetes Secret and enable the worker in your values: + +```yaml +lensWorker: + enabled: true + tokenSecret: + name: litellm-lens-worker + key: token +``` + +The worker image defaults to the chart's application version, and the chart connects it to the backend service. Keep these values and the Secret when upgrading the chart so the gateway and worker upgrade together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.tag`, and `lensWorker.url`. The dashboard uses the chart's worker image for standalone install commands too + +## Standalone worker Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL and agent tracing. Configure one ClickHouse URL for trace writes, bounded reads, and Lens queries: @@ -23,17 +74,17 @@ In **Lens > Investigations**, click **Connect worker**, choose an analysis model The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis -The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. CI also publishes immutable `:sha-` tags for successful worker builds on `main`. Keep the worker image compatible with your gateway version +The dashboard selects the worker image matching the running gateway release. Release images support Linux amd64 and arm64. CI also publishes `:sha-` development images; use those only with a gateway built from the same commit and release tag After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying -For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected: +For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL`, `LENS_WORKER_TOKEN`, and `LITELLM_VERSION` (without `v`) in a private environment file. To use another registry, set `LENS_WORKER_IMAGE` to the compatible image instead of setting a version: ```bash docker compose --env-file /path/to/lens.env -f compose.yaml up -d ``` -Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`. To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000 +To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000. For a local container build, set `LENS_WORKER_IMAGE=litellm-lens-worker:local` and `LITELLM_RELEASE_TAG` to the gateway's release tag, then use `docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build` The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options @@ -130,3 +181,12 @@ The Lens API now uses `/lens` instead of `/engine`, list responses use `lenses`, Stop workers and let active scans finish before upgrading. Deploy proxy instances together: older proxies cannot use the renamed database tables. The schema migration renames the three Lens tables and the run-history identifier column in place, preserving saved investigations, findings, history, worker credentials, and billing assignments. Existing migration files retain their original names and checksums Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy Lens table exists, preventing Prisma from dropping saved data. Apply `litellm-proxy-extras/litellm_proxy_extras/migrations/20261001100000_rename_lens/migration.sql` to the configured database schema before retrying. Deployments already using migration history can instead start without `--use_prisma_db_push` to apply the shipped migration normally. Fresh databases and databases already using the renamed tables can continue using database push + + +## Release compatibility + +Released gateway and worker images carry `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched. During a rolling upgrade, workers wait for a gateway from their release + +The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Worker-only Compose accepts `LITELLM_VERSION` (without `v`) or an explicit `LENS_WORKER_IMAGE`. Release workers are available as `ghcr.io/berriai/litellm-lens-worker:vX.Y.Z` and `docker.io/berriai/litellm-lens-worker:vX.Y.Z`, including matching RC/dev suffixes, on amd64 and arm64 + +For source development, use `make lens-dev`. Custom paired builds must pass the same `--build-arg LITELLM_RELEASE_TAG=` to the proxy/backend and worker. The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds diff --git a/deploy/lens/compose.build.yaml b/deploy/lens/compose.build.yaml index e4237d8de23f..52d59a84a796 100644 --- a/deploy/lens/compose.build.yaml +++ b/deploy/lens/compose.build.yaml @@ -3,4 +3,6 @@ services: build: context: ../.. dockerfile: deploy/lens/Dockerfile + args: + LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:?Set the release tag used by the gateway} image: litellm-lens-worker:local diff --git a/deploy/lens/compose.yaml b/deploy/lens/compose.yaml index fc9850fcb04d..799f0a4fb1e7 100644 --- a/deploy/lens/compose.yaml +++ b/deploy/lens/compose.yaml @@ -1,6 +1,6 @@ services: lens-worker: - image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b} + image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION:?Set LITELLM_VERSION to the gateway release, without the v prefix}} environment: LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container} LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI} diff --git a/deploy/lens/config.yaml b/deploy/lens/config.yaml new file mode 100644 index 000000000000..cb12a2b09192 --- /dev/null +++ b/deploy/lens/config.yaml @@ -0,0 +1,7 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + tracing: + store: + type: clickhouse + url: os.environ/CLICKHOUSE_URL + retention_days: 14 diff --git a/deploy/lens/stack.yaml b/deploy/lens/stack.yaml new file mode 100644 index 000000000000..0452b27fa64e --- /dev/null +++ b/deploy/lens/stack.yaml @@ -0,0 +1,71 @@ +name: litellm-lens + +services: + litellm: + image: ghcr.io/berriai/litellm:${LITELLM_VERSION:?Set LITELLM_VERSION to a published release, without the v prefix} + command: ["--config", "/app/lens-config.yaml", "--port", "4000"] + environment: + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?Set a strong master key} + LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades} + DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm + STORE_MODEL_IN_DB: "True" + CLICKHOUSE_URL: http://default:local-lens@clickhouse:8123 + LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION} + volumes: + - ./config.yaml:/app/lens-config.yaml:ro + ports: + - "127.0.0.1:${LITELLM_PORT:-4000}:4000" + depends_on: + db: + condition: service_healthy + clickhouse: + condition: service_healthy + restart: unless-stopped + + lens-worker: + profiles: [lens] + image: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION} + environment: + LITELLM_URL: http://litellm:4000 + LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-} + depends_on: [litellm] + restart: unless-stopped + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g} + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + + db: + image: postgres:16 + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"] + interval: 5s + timeout: 5s + retries: 20 + restart: unless-stopped + + clickhouse: + image: clickhouse/clickhouse-server:26.9.6.6 + environment: + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: local-lens + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + volumes: + - clickhouse_data:/var/lib/clickhouse + healthcheck: + test: ["CMD", "clickhouse-client", "--user", "default", "--password", "local-lens", "--query", "SELECT 1"] + interval: 5s + timeout: 5s + retries: 20 + restart: unless-stopped + +volumes: + postgres_data: + clickhouse_data: diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 61b6faae6911..3309fdd53411 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -113,6 +113,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh FROM $LITELLM_RUNTIME_IMAGE AS runtime +ARG LITELLM_RELEASE_TAG="" +ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} USER root diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index ca526e068344..bafd1af46d18 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -122,6 +122,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh FROM $LITELLM_RUNTIME_IMAGE AS runtime +ARG LITELLM_RELEASE_TAG="" +ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} WORKDIR /app USER root diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 3eb64e5528c2..d56c0e1f52d3 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -57,6 +57,8 @@ spec: containerPort: 4001 protocol: TCP env: + - name: LENS_WORKER_IMAGE + value: "{{ .Values.lensWorker.image.repository }}:{{ .Values.lensWorker.image.tag | default .Chart.AppVersion }}" {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }} {{- if .Values.gateway.config.create }} - name: CONFIG_FILE_PATH diff --git a/helm/litellm/templates/lens/deployment.yaml b/helm/litellm/templates/lens/deployment.yaml new file mode 100644 index 000000000000..89db8cb3646d --- /dev/null +++ b/helm/litellm/templates/lens/deployment.yaml @@ -0,0 +1,72 @@ +{{- if .Values.lensWorker.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "litellm.fullname" . }}-lens-worker + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: lens-worker +spec: + replicas: {{ .Values.lensWorker.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: lens-worker + template: + metadata: + labels: + {{- include "litellm.commonLabels" . | nindent 8 }} + app.kubernetes.io/component: lens-worker + spec: + automountServiceAccountToken: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: lens-worker + image: "{{ .Values.lensWorker.image.repository }}:{{ .Values.lensWorker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.lensWorker.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + env: + - name: LITELLM_URL + value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.backend.fullname" .) .Values.backend.service.port) | quote }} + - name: LENS_WORKER_TOKEN + valueFrom: + secretKeyRef: + name: {{ required "lensWorker.tokenSecret.name must reference a Lens worker token" .Values.lensWorker.tokenSecret.name | quote }} + key: {{ .Values.lensWorker.tokenSecret.key | quote }} + resources: + {{- toYaml .Values.lensWorker.resources | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: + medium: Memory + sizeLimit: {{ .Values.lensWorker.tmpSizeLimit }} + {{- with .Values.lensWorker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.lensWorker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.lensWorker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/litellm/tests/lens_worker_tests.yaml b/helm/litellm/tests/lens_worker_tests.yaml new file mode 100644 index 000000000000..232ad9e26b45 --- /dev/null +++ b/helm/litellm/tests/lens_worker_tests.yaml @@ -0,0 +1,73 @@ +suite: Lens worker release and credentials +templates: + - lens/deployment.yaml + - backend/deployment.yaml + - gateway/configmap.yaml +values: + - ./values/required.yaml +tests: + - it: keeps the worker opt in + template: lens/deployment.yaml + asserts: + - hasDocuments: + count: 0 + - it: requires a limited worker credential when enabled + template: lens/deployment.yaml + set: + lensWorker.enabled: true + asserts: + - failedTemplate: + errorMessage: lensWorker.tokenSecret.name must reference a Lens worker token + - it: uses the chart release and a secret without granting Kubernetes access + template: lens/deployment.yaml + chart: + appVersion: v1.2.3 + set: + lensWorker.enabled: true + lensWorker.tokenSecret.name: lens-credential + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm-lens-worker:v1.2.3 + - equal: + path: spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef + value: + name: lens-credential + key: token + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.template.spec.volumes[0].emptyDir + value: + medium: Memory + sizeLimit: 1Gi + - it: advertises the same private dev image to standalone installers + template: backend/deployment.yaml + set: + lensWorker.image.repository: registry.example/lens-worker + lensWorker.image.tag: branch-main-1234567 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_WORKER_IMAGE + value: registry.example/lens-worker:branch-main-1234567 + - it: supports an external gateway and a registry override + template: lens/deployment.yaml + set: + lensWorker.enabled: true + lensWorker.tokenSecret.name: lens-credential + lensWorker.url: https://gateway.example/proxy + lensWorker.image.repository: registry.example/lens-worker + lensWorker.image.tag: branch-main-1234567 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example/lens-worker:branch-main-1234567 + - equal: + path: spec.template.spec.containers[0].env[0].value + value: https://gateway.example/proxy diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 2c0c7151a32a..6964dfd2b6e9 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -629,3 +629,25 @@ ui: affinity: {} # Same shape as gateway.topologySpreadConstraints. topologySpreadConstraints: [] + +lensWorker: + enabled: false + replicaCount: 1 + image: + repository: ghcr.io/berriai/litellm-lens-worker + tag: "" + pullPolicy: IfNotPresent + tokenSecret: + name: "" + key: token + url: "" + tmpSizeLimit: 1Gi + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 2Gi + nodeSelector: {} + tolerations: [] + affinity: {} diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4cf6db6f2576..fca8f2562758 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -34885,6 +34885,10 @@ "WorkerCreated": { "additionalProperties": false, "properties": { + "image": { + "title": "Image", + "type": "string" + }, "token": { "title": "Token", "type": "string" @@ -34894,6 +34898,7 @@ } }, "required": [ + "image", "worker", "token" ], diff --git a/litellm/proxy/lens/endpoints.py b/litellm/proxy/lens/endpoints.py index a20258853c0a..d52fbcda5d63 100644 --- a/litellm/proxy/lens/endpoints.py +++ b/litellm/proxy/lens/endpoints.py @@ -40,6 +40,7 @@ Worker, WorkerCreated, ) +from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag, worker_image from litellm.proxy.lens.repository import LensRepository, WriterDatabase from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution from litellm.proxy.lens.state import ( @@ -369,7 +370,7 @@ async def register_worker(body: WorkerName, auth: Auth) -> WorkerCreated: last_seen=datetime(1970, 1, 1, tzinfo=timezone.utc), ) await repository().save_worker(worker, hashlib.sha256(token.encode()).hexdigest()) - return WorkerCreated(worker=worker, token=token) + return WorkerCreated(worker=worker, token=token, image=worker_image()) @router.put("/workers/{worker_id}/billing-key", response_model=Worker) @@ -401,9 +402,10 @@ async def revoke_worker(worker_id: str, auth: Auth) -> bool: @router.post("/worker/claim", response_model=Claim | None) -async def claim(worker: WorkerAuth, protocol_version: int = 1) -> Claim | None: - if protocol_version not in (2, 3): - raise HTTPException(409, "Upgrade the Lens worker using the current Connect worker command") +async def claim(worker: WorkerAuth, protocol_version: int = 1, worker_release: str = "") -> Claim | None: + expected: Final = release_tag() + if protocol_version != PROTOCOL_VERSION or (expected and worker_release != expected): + raise HTTPException(409, f"Upgrade the Lens worker to {worker_image()} and retry") if worker.analysis_key_id is None: raise HTTPException(409, "Assign an analysis key to this worker in Lens setup") now: Final = datetime.now(timezone.utc) diff --git a/litellm/proxy/lens/models.py b/litellm/proxy/lens/models.py index d967d18d0d14..84562f999cfe 100644 --- a/litellm/proxy/lens/models.py +++ b/litellm/proxy/lens/models.py @@ -236,6 +236,7 @@ class Worker(Record): class WorkerCreated(Record): + image: str worker: Worker token: str diff --git a/litellm/proxy/lens/release.py b/litellm/proxy/lens/release.py new file mode 100644 index 000000000000..7abc89351390 --- /dev/null +++ b/litellm/proxy/lens/release.py @@ -0,0 +1,28 @@ +import os +from importlib.metadata import PackageNotFoundError, version +from typing import Final + +PROTOCOL_VERSION: Final = 4 + + +def release_tag() -> str: + if tag := os.environ.get("LITELLM_RELEASE_TAG", ""): + return tag + try: + installed: Final = version("litellm") + except PackageNotFoundError: + return "" + + from packaging.version import Version + + parsed: Final = Version(installed) + suffix: Final = f"-dev.{parsed.dev}" if parsed.dev is not None else f"-rc.{parsed.pre[1]}" if parsed.pre else "" + return f"v{parsed.base_version}{suffix}" + + +def worker_image() -> str: + override: Final = os.environ.get("LENS_WORKER_IMAGE", "") + if override: + return override + tag: Final = release_tag() + return f"ghcr.io/berriai/litellm-lens-worker:{tag}" if tag else "litellm-lens-worker:local" diff --git a/litellm/proxy/lens/worker.py b/litellm/proxy/lens/worker.py index f9e746489d91..06455dc1a9a6 100644 --- a/litellm/proxy/lens/worker.py +++ b/litellm/proxy/lens/worker.py @@ -11,6 +11,7 @@ from .analysis import AnalysisResponseError, analyze_sample, validation_details from .models import Claim, Coverage, ExecutionContent, ModelRequest, ModelResult, Progress, Result, Sample +from .release import PROTOCOL_VERSION, release_tag logger: Final = logging.getLogger("litellm.lens.worker") @@ -120,8 +121,26 @@ async def model_request(self, path: str, body: ModelRequest, attempt: int = 0) - await self.sleep(2**attempt) return await self.model_request(path, body, attempt + 1) + async def report_unreadable_claim(self, identity: ClaimIdentity) -> None: + failure: Final = await self.client.post( + f"/lens/worker/{identity.lens_id}/{identity.job.id}/result", + json=Result( + coverage=Coverage(), + error="The worker could not read this investigation. Update the worker to match the gateway, then retry.", + ).model_dump(), + ) + if failure.status_code != 409: + failure.raise_for_status() + logger.warning("Worker could not read a claimed investigation; reported a version compatibility failure") + async def run_once(self) -> bool: - response: Final = await self.client.post("/lens/worker/claim", params=MappingProxyType({"protocol_version": 3})) + response: Final = await self.client.post( + "/lens/worker/claim", + params=MappingProxyType({"protocol_version": str(PROTOCOL_VERSION), "worker_release": release_tag()}), + ) + if response.status_code == 409: + logger.warning("Lens worker cannot claim work: %s", response.text) + return False response.raise_for_status() payload: Final = response.json() if payload is None: @@ -129,17 +148,7 @@ async def run_once(self) -> bool: try: claim: Final = Claim.model_validate(payload) except ValidationError: - identity: Final = ClaimIdentity.model_validate(payload) - failure: Final = await self.client.post( - f"/lens/worker/{identity.lens_id}/{identity.job.id}/result", - json=Result( - coverage=Coverage(), - error="The worker could not read this investigation. Update the worker to match the gateway, then retry.", - ).model_dump(), - ) - if failure.status_code != 409: - failure.raise_for_status() - logger.warning("Worker could not read a claimed investigation; reported a version compatibility failure") + await self.report_unreadable_claim(ClaimIdentity.model_validate(payload)) return True prefix: Final = f"/lens/worker/{claim.lens_id}/{claim.job.id}" diff --git a/tests/proxy_behavior/lens/test_lifecycle.py b/tests/proxy_behavior/lens/test_lifecycle.py index 1dade952dd8e..d5d177844364 100644 --- a/tests/proxy_behavior/lens/test_lifecycle.py +++ b/tests/proxy_behavior/lens/test_lifecycle.py @@ -29,6 +29,7 @@ Scope, Worker, ) +from litellm.proxy.lens.release import PROTOCOL_VERSION, release_tag from litellm.proxy.lens.repository import Database, LensRepository, Row from litellm.proxy.lens.state import can_access from litellm.proxy.utils import PrismaClient, ProxyLogging @@ -349,8 +350,9 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: authenticated_legacy: Final = await endpoints.worker_auth(credentials) assert authenticated_legacy.analysis_key_id is None with pytest.raises(HTTPException) as needs_billing: - await endpoints.claim(authenticated_legacy, protocol_version=2) + await endpoints.claim(authenticated_legacy, protocol_version=PROTOCOL_VERSION, worker_release=release_tag()) assert needs_billing.value.status_code == 409 + assert "Assign an analysis key" in needs_billing.value.detail assert await endpoints.heartbeat(lens.id, claimed.job.id, authenticated_legacy) finished: Final = await endpoints.result( lens.id, claimed.job.id, Result(coverage=Coverage(screened=2)), authenticated_legacy, storage=None diff --git a/tests/unit/proxy/lens/test_endpoints.py b/tests/unit/proxy/lens/test_endpoints.py index 40a6fd55044b..cfbdebba843c 100644 --- a/tests/unit/proxy/lens/test_endpoints.py +++ b/tests/unit/proxy/lens/test_endpoints.py @@ -148,13 +148,14 @@ def test_invalid_explicit_execution_ids_are_rejected(identity: str) -> None: assert error.value.status_code == 422 +@pytest.mark.parametrize("protocol_version", (1, 2, 3)) @pytest.mark.asyncio -async def test_incompatible_worker_is_rejected_before_claiming_work() -> None: +async def test_incompatible_worker_is_rejected_before_claiming_work(protocol_version: int) -> None: from litellm.proxy.lens.endpoints import claim from tests.unit.proxy.lens.test_state import worker with pytest.raises(HTTPException) as error: - await claim(worker(), protocol_version=1) + await claim(worker(), protocol_version=protocol_version) assert error.value.status_code == 409 assert "Upgrade" in error.value.detail @@ -207,3 +208,20 @@ async def test_preview_reports_calendar_overflow_as_a_validation_error() -> None await preview_sample(body, UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), None) assert error.value.status_code == 422 assert "supported calendar range" in error.value.detail + + +@pytest.mark.asyncio +@pytest.mark.parametrize("worker_release", ("", "v1.2.2", "branch-main-old")) +async def test_different_release_is_rejected_before_accessing_jobs( + monkeypatch: pytest.MonkeyPatch, worker_release: str +) -> None: + from litellm.proxy.lens.endpoints import claim + from litellm.proxy.lens.release import PROTOCOL_VERSION + from tests.unit.proxy.lens.test_state import worker + + monkeypatch.setenv("LITELLM_RELEASE_TAG", "v1.2.3") + monkeypatch.delenv("LENS_WORKER_IMAGE", raising=False) + with pytest.raises(HTTPException) as error: + await claim(worker(), protocol_version=PROTOCOL_VERSION, worker_release=worker_release) + assert error.value.status_code == 409 + assert "ghcr.io/berriai/litellm-lens-worker:v1.2.3" in error.value.detail diff --git a/tests/unit/proxy/lens/test_release.py b/tests/unit/proxy/lens/test_release.py new file mode 100644 index 000000000000..eb62330558cf --- /dev/null +++ b/tests/unit/proxy/lens/test_release.py @@ -0,0 +1,39 @@ +from typing import Final + +import pytest + +from litellm.proxy.lens.release import worker_image + + +@pytest.mark.parametrize("tag", ("v1.2.3", "v1.2.3-rc.4", "v1.2.3-dev.5", "branch-main-1234567")) +def test_install_command_follows_the_gateway_release(monkeypatch: pytest.MonkeyPatch, tag: str) -> None: + monkeypatch.setenv("LITELLM_RELEASE_TAG", tag) + monkeypatch.delenv("LENS_WORKER_IMAGE", raising=False) + assert worker_image() == f"ghcr.io/berriai/litellm-lens-worker:{tag}" + + +def test_private_registry_override_keeps_its_exact_digest(monkeypatch: pytest.MonkeyPatch) -> None: + image: Final = "registry.example/lens-worker@sha256:" + "a" * 64 + monkeypatch.setenv("LITELLM_RELEASE_TAG", "branch-main-1234567") + monkeypatch.setenv("LENS_WORKER_IMAGE", image) + assert worker_image() == image + + +@pytest.mark.parametrize( + "installed,expected", + (("1.2.3", "v1.2.3"), ("1.2.3rc4", "v1.2.3-rc.4"), ("1.2.3.dev5", "v1.2.3-dev.5")), +) +def test_python_installs_recommend_the_matching_worker( + monkeypatch: pytest.MonkeyPatch, installed: str, expected: str +) -> None: + from litellm.proxy.lens import release + + def installed_version(name: str) -> str: + assert name == "litellm" + return installed + + monkeypatch.delenv("LITELLM_RELEASE_TAG", raising=False) + monkeypatch.delenv("LENS_WORKER_IMAGE", raising=False) + monkeypatch.setattr(release, "version", installed_version) + assert release.release_tag() == expected + assert worker_image() == f"ghcr.io/berriai/litellm-lens-worker:{expected}" diff --git a/tests/unit/proxy/lens/test_worker.py b/tests/unit/proxy/lens/test_worker.py index 17bac711e395..7983aec8af45 100644 --- a/tests/unit/proxy/lens/test_worker.py +++ b/tests/unit/proxy/lens/test_worker.py @@ -417,3 +417,22 @@ async def handle(request: httpx.Request) -> httpx.Response: assert result.error == "" assert result.coverage.screened == 1 and result.coverage.unassessable == 0 assert attempts.qsize() == 2 and saved.empty() + + +@pytest.mark.asyncio +async def test_worker_announces_release_and_waits_on_incompatible_gateway( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + from litellm.proxy.lens.release import PROTOCOL_VERSION + + monkeypatch.setenv("LITELLM_RELEASE_TAG", "v1.2.3") + + def handle(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/lens/worker/claim" + assert request.url.params["protocol_version"] == str(PROTOCOL_VERSION) + assert request.url.params["worker_release"] == "v1.2.3" + return httpx.Response(409, json={"detail": "Upgrade the Lens worker to v1.2.4"}) + + async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client: + assert not await LensWorker(client).run_once() + assert "Upgrade the Lens worker to v1.2.4" in caplog.text diff --git a/ui/litellm-dashboard/src/components/lens/setup/worker/WorkerDialog.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/setup/worker/WorkerDialog.integration.test.tsx index 10ee40a87f9d..719b83cfe6bf 100644 --- a/ui/litellm-dashboard/src/components/lens/setup/worker/WorkerDialog.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/setup/worker/WorkerDialog.integration.test.tsx @@ -12,6 +12,7 @@ vi.mock("@/components/networking", () => ({ const created = { token: "lens-test-token", + image: "ghcr.io/berriai/litellm-lens-worker:v1.2.3", worker: { id: "worker", name: "Lens worker", @@ -57,7 +58,11 @@ describe("Worker setup", () => { expect(command).toContain("LITELLM_URL=https://gateway.example/proxy"); expect(command).toContain("LENS_WORKER_TOKEN=lens-test-token"); expect(command).toContain("--add-host host.docker.internal:host-gateway"); - expect(command).toContain("ghcr.io/berriai/litellm-lens-worker@sha256:"); + expect(command).toContain(created.image); + await user.click(screen.getByText("Using Docker Compose or Helm?")); + await user.click(screen.getByRole("button", { name: "Copy worker token" })); + expect(await navigator.clipboard.readText()).toBe(created.token); + expect(screen.getByRole("button", { name: "Token copied" })).toBeVisible(); }); it("assigns billing to an existing worker without replacing its access token", async () => { const user = userEvent.setup(); diff --git a/ui/litellm-dashboard/src/components/lens/setup/worker/WorkerInstall.tsx b/ui/litellm-dashboard/src/components/lens/setup/worker/WorkerInstall.tsx index 49f4df82edd3..efeef194652e 100644 --- a/ui/litellm-dashboard/src/components/lens/setup/worker/WorkerInstall.tsx +++ b/ui/litellm-dashboard/src/components/lens/setup/worker/WorkerInstall.tsx @@ -1,5 +1,6 @@ "use client"; +import { useState } from "react"; import { Button } from "@/components/ui/button"; import { CheckCircle2, Copy, Loader2 } from "lucide-react"; @@ -25,6 +26,7 @@ export function WorkerInstall({ onReady?: () => void; onClose: () => void; }) { + const [tokenCopied, setTokenCopied] = useState(false); return (
{!connected && ( @@ -34,7 +36,7 @@ export function WorkerInstall({ className="w-full gap-2" onClick={async () => { try { - await navigator.clipboard.writeText(workerSetupCommand(address, created.token)); + await navigator.clipboard.writeText(workerSetupCommand(address, created.token, created.image)); setCopied(true); } catch { setError("Clipboard access failed. Allow clipboard access and try again."); @@ -51,9 +53,30 @@ export function WorkerInstall({ aria-label="Docker command preview" className="mt-3 max-h-48 overflow-auto rounded-md bg-muted/40 p-3 text-xs leading-5" > - {workerSetupCommand(address, created.token)} + {workerSetupCommand(address, created.token, created.image)} +
+ Using Docker Compose or Helm? +

+ Save this private token as LENS_WORKER_TOKEN in Compose or in your Helm worker token secret. Keep it for + future upgrades. +

+ +
)} {!connected && ( diff --git a/ui/litellm-dashboard/src/components/lens/setup/worker/workerCommand.ts b/ui/litellm-dashboard/src/components/lens/setup/worker/workerCommand.ts index 0b5c1c347d62..0e022aaec26e 100644 --- a/ui/litellm-dashboard/src/components/lens/setup/worker/workerCommand.ts +++ b/ui/litellm-dashboard/src/components/lens/setup/worker/workerCommand.ts @@ -1,23 +1,20 @@ import { proxyBaseUrl } from "@/components/networking"; import { serverRootPath } from "@/lib/serverRootPath"; -export const LENS_WORKER_IMAGE = - "ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b"; - export function initialProxyAddress(): string { const url = new URL(proxyBaseUrl || serverRootPath, window.location.origin); if (["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)) url.hostname = "host.docker.internal"; return url.toString().replace(/\/$/, ""); } -export function workerSetupCommand(address: string, token: string): string { +export function workerSetupCommand(address: string, token: string, image: string): string { const quote = (value: string) => "'" + value.replaceAll("'", "'\\''") + "'"; return [ "docker run -d --restart unless-stopped --read-only --cap-drop ALL", " --tmpfs /tmp:rw,noexec,nosuid,size=1g", - " --security-opt no-new-privileges --platform linux/amd64 --add-host host.docker.internal:host-gateway", + " --security-opt no-new-privileges --add-host host.docker.internal:host-gateway", ` -e ${quote("LITELLM_URL=" + address)}`, ` -e ${quote("LENS_WORKER_TOKEN=" + token)}`, - ` ${LENS_WORKER_IMAGE}`, + ` ${quote(image)}`, ].join(" \\\n"); } diff --git a/ui/litellm-dashboard/src/components/lens/setup/worker/workerSchema.test.ts b/ui/litellm-dashboard/src/components/lens/setup/worker/workerSchema.test.ts index 01e3099260d7..fed67e0d497f 100644 --- a/ui/litellm-dashboard/src/components/lens/setup/worker/workerSchema.test.ts +++ b/ui/litellm-dashboard/src/components/lens/setup/worker/workerSchema.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { validateWorkerAddress, analysisAccessSchema, workerFormSchema } from "./workerSchema"; -import { workerSetupCommand, LENS_WORKER_IMAGE } from "./workerCommand"; +import { workerSetupCommand } from "./workerCommand"; const workerDefaults = { useExisting: false, @@ -26,14 +26,13 @@ describe("worker setup", () => { }); it("quotes apostrophes literally and retains the pinned image and runtime restrictions", () => { - const command = workerSetupCommand("https://gateway.example/proxy?name=it's", "token'quoted"); + const image = "registry.example/lens-worker:v1.2.3-rc.4"; + const command = workerSetupCommand("https://gateway.example/proxy?name=it's", "token'quoted", image); expect(command).toContain("'LITELLM_URL=https://gateway.example/proxy?name=it'\\''s'"); expect(command).toContain("'LENS_WORKER_TOKEN=token'\\''quoted'"); expect(command).toContain("--read-only --cap-drop ALL"); - expect(command).toContain( - "--security-opt no-new-privileges --platform linux/amd64 --add-host host.docker.internal:host-gateway", - ); - expect(command.split("\n").at(-1)?.trim()).toBe(LENS_WORKER_IMAGE); + expect(command).toContain("--security-opt no-new-privileges --add-host host.docker.internal:host-gateway"); + expect(command.split("\n").at(-1)?.trim()).toBe(`'${image}'`); }); }); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3ed4103a227e..434bccd1349b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -49490,6 +49490,8 @@ export interface components { }; /** WorkerCreated */ WorkerCreated: { + /** Image */ + image: string; /** Token */ token: string; worker: components["schemas"]["Worker"]; @@ -62278,6 +62280,7 @@ export interface operations { parameters: { query?: { protocol_version?: number; + worker_release?: string; }; header?: never; path?: never; From fb475d66ccf603e596a73ab8a0070bc16c812880 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 13:32:07 -0700 Subject: [PATCH 2/8] test(lens): exercise bundled Compose startup and restart in CI --- .github/workflows/image-scan.yml | 10 ++- deploy/lens/README.md | 2 +- tests/e2e/migrations/lens_compose_smoke.sh | 71 ++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/migrations/lens_compose_smoke.sh diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index 0695720733ff..8fb11f403701 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -15,6 +15,9 @@ on: - gateway/main.py - backend/Dockerfile - backend/main.py + - deploy/lens/** + - litellm/proxy/lens/release.py + - tests/e2e/migrations/lens_compose_smoke.sh - docker/component_entrypoint.sh - docker/entrypoint.sh - litellm/proxy/prisma_migration.py @@ -113,7 +116,7 @@ jobs: persist-credentials: false - name: Build runtime image - run: docker build -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} . + run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} . - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 @@ -127,6 +130,11 @@ jobs: python -m pip install "pytest==9.0.3" python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v + - name: Verify the bundled Lens Compose installation and restart + env: + LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }} + run: bash tests/e2e/migrations/lens_compose_smoke.sh + migrations-image: name: migrations-image runs-on: ubuntu-latest diff --git a/deploy/lens/README.md b/deploy/lens/README.md index fb658f32059e..5bb07e6a04d6 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -187,6 +187,6 @@ Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy L Released gateway and worker images carry `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched. During a rolling upgrade, workers wait for a gateway from their release -The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Worker-only Compose accepts `LITELLM_VERSION` (without `v`) or an explicit `LENS_WORKER_IMAGE`. Release workers are available as `ghcr.io/berriai/litellm-lens-worker:vX.Y.Z` and `docker.io/berriai/litellm-lens-worker:vX.Y.Z`, including matching RC/dev suffixes, on amd64 and arm64 +The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Worker-only Compose accepts `LITELLM_VERSION` (without `v`) or an explicit `LENS_WORKER_IMAGE`. Release workers are available as `ghcr.io/berriai/litellm-lens-worker:vX.Y.Z` and `docker.io/litellm/litellm-lens-worker:vX.Y.Z`, including matching RC/dev suffixes, on amd64 and arm64 For source development, use `make lens-dev`. Custom paired builds must pass the same `--build-arg LITELLM_RELEASE_TAG=` to the proxy/backend and worker. The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds diff --git a/tests/e2e/migrations/lens_compose_smoke.sh b/tests/e2e/migrations/lens_compose_smoke.sh new file mode 100644 index 000000000000..f2443749b40c --- /dev/null +++ b/tests/e2e/migrations/lens_compose_smoke.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +qa_dir=$(mktemp -d) +master_key="sk-$(openssl rand -hex 32)" +compose=(docker compose -p lens-compose-ci --env-file "$qa_dir/env" -f deploy/lens/stack.yaml) +cleanup() { + "${compose[@]}" --profile lens down -v --remove-orphans >/dev/null 2>&1 || true + rm -rf "$qa_dir" +} +trap cleanup EXIT +umask 077 +printf 'LITELLM_VERSION=0.0.0-lens-ci\nLITELLM_PORT=4418\nLITELLM_MASTER_KEY=%s\nLITELLM_SALT_KEY=sk-%s\n' \ + "$master_key" "$(openssl rand -hex 32)" > "$qa_dir/env" +docker tag "${LITELLM_IMAGE:?Set LITELLM_IMAGE to the built gateway image}" ghcr.io/berriai/litellm:0.0.0-lens-ci +docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f deploy/lens/Dockerfile \ + -t ghcr.io/berriai/litellm-lens-worker:v0.0.0-lens-ci . +"${compose[@]}" up -d + +api() { + curl --fail-with-body --silent --show-error --max-time 30 \ + -H "Authorization: Bearer $master_key" -H 'Content-Type: application/json' \ + "http://127.0.0.1:4418$1" "${@:2}" +} +ready=false +for attempt in $(seq 1 90); do + if api /health/liveliness > /dev/null 2>&1; then ready=true; break; fi + sleep 2 +done +if [[ "$ready" != true ]]; then "${compose[@]}" logs litellm; exit 1; fi +api /key/generate -d '{"key_alias":"Lens Compose CI","models":["lens-compose-ci"],"max_budget":1}' > "$qa_dir/key.json" +key_id=$(jq -r '.token_id // empty' "$qa_dir/key.json") +if [[ -z "$key_id" ]]; then + key_id=$(jq -rj '.key' "$qa_dir/key.json" | openssl dgst -sha256 | awk '{print $NF}') +fi +jq -n --arg key "$key_id" '{name:"Lens Compose CI",analysis_key_id:$key}' > "$qa_dir/registration.json" +api /lens/workers/register -d "@$qa_dir/registration.json" > "$qa_dir/worker.json" +jq -e '.image == "ghcr.io/berriai/litellm-lens-worker:v0.0.0-lens-ci"' "$qa_dir/worker.json" > /dev/null +printf 'LENS_WORKER_TOKEN=%s\n' "$(jq -r '.token' "$qa_dir/worker.json")" >> "$qa_dir/env" +worker_id=$(jq -r '.worker.id' "$qa_dir/worker.json") +heartbeat_after=$(date -u +'%Y-%m-%dT%H:%M:%S') +"${compose[@]}" --profile lens up -d + +connected() { + for attempt in $(seq 1 60); do + if api /lens > "$qa_dir/lens.json" 2>/dev/null && \ + jq -e --arg id "$worker_id" --arg since "$heartbeat_after" \ + '.workers[] | select(.id == $id and .last_seen > $since)' "$qa_dir/lens.json" > /dev/null; then + return 0 + fi + sleep 2 + done + "${compose[@]}" --profile lens logs lens-worker + return 1 +} +connected +printf 'Fresh Compose stack: matching worker image and authenticated heartbeat passed\n' + +status=$(curl --silent --show-error -o "$qa_dir/mismatch.json" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer $(jq -r '.token' "$qa_dir/worker.json")" \ + 'http://127.0.0.1:4418/lens/worker/claim?protocol_version=4&worker_release=v0.0.0-old') +[[ "$status" == 409 ]] +jq -e '.detail | contains("Upgrade the Lens worker")' "$qa_dir/mismatch.json" > /dev/null + +"${compose[@]}" --profile lens restart litellm lens-worker +heartbeat_after=$(date -u +'%Y-%m-%dT%H:%M:%S') +connected +api /lens > "$qa_dir/restarted.json" +jq -e --arg id "$worker_id" --arg key "$key_id" \ + '.workers[] | select(.id == $id and .analysis_key_id == $key)' "$qa_dir/restarted.json" > /dev/null +printf 'Compose restart: worker identity, token and billing assignment preserved; wrong release rejected\n' From a2234a795611946493e136120e5c590038b6708b Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 13:46:17 -0700 Subject: [PATCH 3/8] fix(lens): refund failed model requests without a response --- litellm/proxy/lens/inference.py | 11 +++++------ tests/proxy_behavior/lens/test_lifecycle.py | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/lens/inference.py b/litellm/proxy/lens/inference.py index b5c4d128c6e4..93c6202c495e 100644 --- a/litellm/proxy/lens/inference.py +++ b/litellm/proxy/lens/inference.py @@ -192,7 +192,7 @@ def reserve(e: Lens) -> Lens: current, active.model_copy(update=MappingProxyType({"cost": active.cost + estimate})) ).model_copy(update=MappingProxyType({"spent": current.spent + estimate})) - def settle(e: Lens, cost: float) -> Lens: + def settle(e: Lens, cost: float, step: Step | None = None) -> Lens: charged: Final = next((j for j in e.jobs if j.id == job.id), None) adjusted: Final = ( e.model_copy(update=MappingProxyType({"spent": max(0, e.spent - estimate + cost)})) @@ -201,12 +201,10 @@ def settle(e: Lens, cost: float) -> Lens: ) if charged is None: return adjusted + settled: Final = charged.model_copy(update=MappingProxyType({"cost": max(0, charged.cost - estimate + cost)})) return replace_job( adjusted, - add_step( - charged.model_copy(update=MappingProxyType({"cost": max(0, charged.cost - estimate + cost)})), - model_step(response, body, job.settings.model, cost), - ), + add_step(settled, step) if step else settled, ) @asynccontextmanager @@ -243,7 +241,8 @@ async def reserve_budget() -> AsyncIterator[None]: response, billed_cost = await complete(worker.analysis_key_id, data, reserve_budget, request) cost: Final = billed_cost if billed_cost is not None else completion_charge(deployments, response, estimate) - await repo.update(lens.id, lambda e: settle(e, cost)) + step: Final = model_step(response, body, job.settings.model, cost) + await repo.update(lens.id, lambda e: settle(e, cost, step)) parsed: Final = Completion.model_validate_json(response.model_dump_json()) choice: Final = parsed.choices[0] return ModelResult( diff --git a/tests/proxy_behavior/lens/test_lifecycle.py b/tests/proxy_behavior/lens/test_lifecycle.py index d5d177844364..5bd41f889d88 100644 --- a/tests/proxy_behavior/lens/test_lifecycle.py +++ b/tests/proxy_behavior/lens/test_lifecycle.py @@ -443,6 +443,7 @@ async def test_failed_model_requests_release_lens_budget_reservations(lens_datab stored: Final = await endpoints.get_lens(lens.id, worker.scope) assert stored.spent == 0 assert stored.jobs[0].cost == 0 + assert not any(step.kind == "model" for step in stored.jobs[0].steps) finally: await lens_database.db.execute_raw('DELETE FROM "LiteLLM_LensRun" WHERE lens_id=$1', lens.id) await lens_database.db.execute_raw('DELETE FROM "LiteLLM_Lens" WHERE id=$1', lens.id) From cbd403778a61ff8dc11e330d5b762cefb1c33518 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 13:51:43 -0700 Subject: [PATCH 4/8] test(lens): verify trace persistence in the bundled stack --- tests/e2e/migrations/lens_compose_smoke.sh | 25 +++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/e2e/migrations/lens_compose_smoke.sh b/tests/e2e/migrations/lens_compose_smoke.sh index f2443749b40c..deaa63a9a3fa 100644 --- a/tests/e2e/migrations/lens_compose_smoke.sh +++ b/tests/e2e/migrations/lens_compose_smoke.sh @@ -28,6 +28,28 @@ for attempt in $(seq 1 90); do sleep 2 done if [[ "$ready" != true ]]; then "${compose[@]}" logs litellm; exit 1; fi +trace_id=$(openssl rand -hex 16) +span_id=$(openssl rand -hex 8) +start_ns="$(date +%s)000000000" +jq -n --arg trace "$trace_id" --arg span "$span_id" --arg at "$start_ns" \ + '{resourceSpans:[{resource:{attributes:[{key:"service.name",value:{stringValue:"lens-compose-ci"}}]}, + scopeSpans:[{scope:{name:"lens-compose-ci"},spans:[{traceId:$trace,spanId:$span,name:"Compose trace", + kind:1,startTimeUnixNano:$at,endTimeUnixNano:$at, + attributes:[{key:"openinference.span.kind",value:{stringValue:"AGENT"}}],status:{code:1}}]}]}]}' \ + > "$qa_dir/trace.json" +api /v1/traces -d "@$qa_dir/trace.json" > /dev/null +trace_saved() { + for attempt in $(seq 1 60); do + if api "/v1/traces/$trace_id" > "$qa_dir/saved-trace.json" 2>/dev/null && \ + jq -e --arg trace "$trace_id" --arg span "$span_id" \ + '.summary.trace_id == $trace and any(.spans[]; .span_id == $span)' "$qa_dir/saved-trace.json" > /dev/null; then + return 0 + fi + sleep 2 + done + return 1 +} +trace_saved api /key/generate -d '{"key_alias":"Lens Compose CI","models":["lens-compose-ci"],"max_budget":1}' > "$qa_dir/key.json" key_id=$(jq -r '.token_id // empty' "$qa_dir/key.json") if [[ -z "$key_id" ]]; then @@ -65,7 +87,8 @@ jq -e '.detail | contains("Upgrade the Lens worker")' "$qa_dir/mismatch.json" > "${compose[@]}" --profile lens restart litellm lens-worker heartbeat_after=$(date -u +'%Y-%m-%dT%H:%M:%S') connected +trace_saved api /lens > "$qa_dir/restarted.json" jq -e --arg id "$worker_id" --arg key "$key_id" \ '.workers[] | select(.id == $id and .analysis_key_id == $key)' "$qa_dir/restarted.json" > /dev/null -printf 'Compose restart: worker identity, token and billing assignment preserved; wrong release rejected\n' +printf 'Compose restart: trace, worker identity, token and billing assignment preserved; wrong release rejected\n' From 243a5c691978a13ff6f24ed25aa4c2e6a235befd Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 15:29:58 -0700 Subject: [PATCH 5/8] fix(lens): align Helm images and isolate Compose storage --- deploy/lens/README.md | 2 ++ deploy/lens/stack.yaml | 19 +++++++++--- helm/litellm/templates/_helpers.tpl | 6 ++++ .../litellm/templates/backend/deployment.yaml | 2 +- helm/litellm/templates/lens/deployment.yaml | 2 +- helm/litellm/tests/lens_worker_tests.yaml | 31 +++++++++++++++++++ tests/e2e/migrations/lens_compose_smoke.sh | 19 ++++++++++++ 7 files changed, 74 insertions(+), 7 deletions(-) diff --git a/deploy/lens/README.md b/deploy/lens/README.md index 5bb07e6a04d6..b6581ecf9d4b 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -17,6 +17,8 @@ curl -fSLo config.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LEN umask 077 printf 'LITELLM_VERSION=%s\nLITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' \ "$LENS_RELEASE" "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +printf 'POSTGRES_PASSWORD=%s\nCLICKHOUSE_PASSWORD=%s\n' \ + "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" >> .env docker compose up -d ``` diff --git a/deploy/lens/stack.yaml b/deploy/lens/stack.yaml index 0452b27fa64e..b60cd13a4bef 100644 --- a/deploy/lens/stack.yaml +++ b/deploy/lens/stack.yaml @@ -7,14 +7,15 @@ services: environment: LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?Set a strong master key} LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades} - DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm + DATABASE_URL: postgresql://litellm:${POSTGRES_PASSWORD:?Set a permanent database password}@db:5432/litellm STORE_MODEL_IN_DB: "True" - CLICKHOUSE_URL: http://default:local-lens@clickhouse:8123 + CLICKHOUSE_URL: http://default:${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}@clickhouse:8123 LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION} volumes: - ./config.yaml:/app/lens-config.yaml:ro ports: - "127.0.0.1:${LITELLM_PORT:-4000}:4000" + networks: [proxy, storage] depends_on: db: condition: service_healthy @@ -29,6 +30,7 @@ services: LITELLM_URL: http://litellm:4000 LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-} depends_on: [litellm] + networks: [proxy] restart: unless-stopped read_only: true tmpfs: @@ -41,7 +43,8 @@ services: environment: POSTGRES_DB: litellm POSTGRES_USER: litellm - POSTGRES_PASSWORD: litellm + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + networks: [storage] volumes: - postgres_data:/var/lib/postgresql/data healthcheck: @@ -55,16 +58,22 @@ services: image: clickhouse/clickhouse-server:26.9.6.6 environment: CLICKHOUSE_USER: default - CLICKHOUSE_PASSWORD: local-lens + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD} CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" volumes: - clickhouse_data:/var/lib/clickhouse healthcheck: - test: ["CMD", "clickhouse-client", "--user", "default", "--password", "local-lens", "--query", "SELECT 1"] + test: ["CMD", "clickhouse-client", "--user", "default", "--password", "${CLICKHOUSE_PASSWORD}", "--query", "SELECT 1"] interval: 5s timeout: 5s retries: 20 restart: unless-stopped + networks: [storage] + +networks: + proxy: + storage: + internal: true volumes: postgres_data: diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 20fd1a722dc9..402b8d323c6c 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -471,6 +471,12 @@ Directory of the collector's unix socket, shared by the gateway and collector containers through an emptyDir. Empty when the sidecar is off or gateway.collector.address is a tcp://127.0.0.1: address. */}} +{{- define "litellm.lensWorker.image" -}} +{{- $chartTag := ternary (printf "v%s" .Chart.AppVersion) .Chart.AppVersion (regexMatch "^[0-9]" .Chart.AppVersion) -}} +{{- $tag := .Values.lensWorker.image.tag | default .Values.backend.image.tag | default $chartTag -}} +{{- printf "%s:%s" .Values.lensWorker.image.repository $tag -}} +{{- end -}} + {{- define "litellm.gateway.collectorSocketDir" -}} {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index d56c0e1f52d3..5d3be1439bd0 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -58,7 +58,7 @@ spec: protocol: TCP env: - name: LENS_WORKER_IMAGE - value: "{{ .Values.lensWorker.image.repository }}:{{ .Values.lensWorker.image.tag | default .Chart.AppVersion }}" + value: {{ include "litellm.lensWorker.image" . | quote }} {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }} {{- if .Values.gateway.config.create }} - name: CONFIG_FILE_PATH diff --git a/helm/litellm/templates/lens/deployment.yaml b/helm/litellm/templates/lens/deployment.yaml index 89db8cb3646d..787581b9ad10 100644 --- a/helm/litellm/templates/lens/deployment.yaml +++ b/helm/litellm/templates/lens/deployment.yaml @@ -32,7 +32,7 @@ spec: type: RuntimeDefault containers: - name: lens-worker - image: "{{ .Values.lensWorker.image.repository }}:{{ .Values.lensWorker.image.tag | default .Chart.AppVersion }}" + image: {{ include "litellm.lensWorker.image" . | quote }} imagePullPolicy: {{ .Values.lensWorker.image.pullPolicy }} securityContext: allowPrivilegeEscalation: false diff --git a/helm/litellm/tests/lens_worker_tests.yaml b/helm/litellm/tests/lens_worker_tests.yaml index 232ad9e26b45..f283cf8e268f 100644 --- a/helm/litellm/tests/lens_worker_tests.yaml +++ b/helm/litellm/tests/lens_worker_tests.yaml @@ -71,3 +71,34 @@ tests: - equal: path: spec.template.spec.containers[0].env[0].value value: https://gateway.example/proxy + - it: prefixes a numeric chart release with v + template: lens/deployment.yaml + chart: + appVersion: 1.2.3-rc.4 + set: + lensWorker.enabled: true + lensWorker.tokenSecret.name: lens-credential + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-rc.4 + - it: follows a backend image override when no worker tag is set + template: lens/deployment.yaml + set: + backend.image.tag: branch-main-1234567 + lensWorker.enabled: true + lensWorker.tokenSecret.name: lens-credential + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm-lens-worker:branch-main-1234567 + - it: recommends the overridden backend release for standalone installers + template: backend/deployment.yaml + set: + backend.image.tag: v1.2.3-dev.4 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_WORKER_IMAGE + value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4 diff --git a/tests/e2e/migrations/lens_compose_smoke.sh b/tests/e2e/migrations/lens_compose_smoke.sh index deaa63a9a3fa..c0844a26fd21 100644 --- a/tests/e2e/migrations/lens_compose_smoke.sh +++ b/tests/e2e/migrations/lens_compose_smoke.sh @@ -12,6 +12,8 @@ trap cleanup EXIT umask 077 printf 'LITELLM_VERSION=0.0.0-lens-ci\nLITELLM_PORT=4418\nLITELLM_MASTER_KEY=%s\nLITELLM_SALT_KEY=sk-%s\n' \ "$master_key" "$(openssl rand -hex 32)" > "$qa_dir/env" +printf 'POSTGRES_PASSWORD=%s\nCLICKHOUSE_PASSWORD=%s\n' \ + "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" >> "$qa_dir/env" docker tag "${LITELLM_IMAGE:?Set LITELLM_IMAGE to the built gateway image}" ghcr.io/berriai/litellm:0.0.0-lens-ci docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f deploy/lens/Dockerfile \ -t ghcr.io/berriai/litellm-lens-worker:v0.0.0-lens-ci . @@ -78,6 +80,23 @@ connected() { connected printf 'Fresh Compose stack: matching worker image and authenticated heartbeat passed\n' +for target in db:5432 clickhouse:8123; do + service=${target%:*} + port=${target#*:} + address=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$("${compose[@]}" ps -q "$service")") + "${compose[@]}" exec -T lens-worker python -c ' +import socket, sys +for host in (sys.argv[1], sys.argv[2]): + try: + connection = socket.create_connection((host, int(sys.argv[3])), timeout=2) + except OSError: + continue + connection.close() + raise SystemExit("Worker can reach a datastore directly") +' "$service" "$address" "$port" +done +printf 'Worker can reach the proxy but cannot connect directly to PostgreSQL or ClickHouse\n' + status=$(curl --silent --show-error -o "$qa_dir/mismatch.json" -w '%{http_code}' -X POST \ -H "Authorization: Bearer $(jq -r '.token' "$qa_dir/worker.json")" \ 'http://127.0.0.1:4418/lens/worker/claim?protocol_version=4&worker_release=v0.0.0-old') From 34f04abf41eb3a914ef2b8a85740b80b6c8a9cf7 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 15:33:00 -0700 Subject: [PATCH 6/8] fix(lens): reject worker builds without release identity --- .github/workflows/lens-worker.yml | 7 +++++++ deploy/lens/Dockerfile | 1 + 2 files changed, 8 insertions(+) diff --git a/.github/workflows/lens-worker.yml b/.github/workflows/lens-worker.yml index cf47eb2edb26..e3c47f35c36f 100644 --- a/.github/workflows/lens-worker.yml +++ b/.github/workflows/lens-worker.yml @@ -35,6 +35,13 @@ jobs: persist-credentials: false - name: Build Lens worker run: docker build --build-arg LITELLM_RELEASE_TAG=sha-${{ github.sha }} -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} . + - name: Reject custom builds without a matching release tag + run: | + if docker build --progress plain -f deploy/lens/Dockerfile -t lens-worker:unversioned . > missing-tag.log 2>&1; then + echo "::error::An unversioned worker build unexpectedly succeeded" + exit 1 + fi + grep -F 'LITELLM_RELEASE_TAG: Pass --build-arg LITELLM_RELEASE_TAG matching the gateway' missing-tag.log - name: Verify standalone imports with a read-only filesystem run: | docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \ diff --git a/deploy/lens/Dockerfile b/deploy/lens/Dockerfile index b1142fe4d71c..3d7ddbe832ff 100644 --- a/deploy/lens/Dockerfile +++ b/deploy/lens/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.12-slim ARG LITELLM_RELEASE_TAG="" +RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}" ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} WORKDIR /app RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7 From 3defa90b09c065921d5e5bf1b0fddea101ade3f9 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 15:40:14 -0700 Subject: [PATCH 7/8] fix(lens): encode Compose credentials and normalize worker versions --- deploy/lens/stack.yaml | 15 +++++++++++++-- helm/litellm/templates/_helpers.tpl | 5 +++-- helm/litellm/tests/lens_worker_tests.yaml | 10 ++++++++++ tests/e2e/migrations/lens_compose_smoke.sh | 2 +- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/deploy/lens/stack.yaml b/deploy/lens/stack.yaml index b60cd13a4bef..ab559e27b193 100644 --- a/deploy/lens/stack.yaml +++ b/deploy/lens/stack.yaml @@ -3,13 +3,24 @@ name: litellm-lens services: litellm: image: ghcr.io/berriai/litellm:${LITELLM_VERSION:?Set LITELLM_VERSION to a published release, without the v prefix} + entrypoint: + - python3 + - -c + - | + import os, sys + from urllib.parse import quote + postgres_password = quote(os.environ["POSTGRES_PASSWORD"], safe="") + clickhouse_password = quote(os.environ["CLICKHOUSE_PASSWORD"], safe="") + os.environ["DATABASE_URL"] = f"postgresql://litellm:{postgres_password}@db:5432/litellm" + os.environ["CLICKHOUSE_URL"] = f"http://default:{clickhouse_password}@clickhouse:8123" + os.execv("docker/prod_entrypoint.sh", ["docker/prod_entrypoint.sh", *sys.argv[1:]]) command: ["--config", "/app/lens-config.yaml", "--port", "4000"] environment: LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?Set a strong master key} LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades} - DATABASE_URL: postgresql://litellm:${POSTGRES_PASSWORD:?Set a permanent database password}@db:5432/litellm + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set a permanent database password} STORE_MODEL_IN_DB: "True" - CLICKHOUSE_URL: http://default:${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}@clickhouse:8123 + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password} LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION} volumes: - ./config.yaml:/app/lens-config.yaml:ro diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 402b8d323c6c..2b7fdfb5fd78 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -472,8 +472,9 @@ collector containers through an emptyDir. Empty when the sidecar is off or gateway.collector.address is a tcp://127.0.0.1: address. */}} {{- define "litellm.lensWorker.image" -}} -{{- $chartTag := ternary (printf "v%s" .Chart.AppVersion) .Chart.AppVersion (regexMatch "^[0-9]" .Chart.AppVersion) -}} -{{- $tag := .Values.lensWorker.image.tag | default .Values.backend.image.tag | default $chartTag -}} +{{- $backendTag := .Values.backend.image.tag | default .Chart.AppVersion -}} +{{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}} +{{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}} {{- printf "%s:%s" .Values.lensWorker.image.repository $tag -}} {{- end -}} diff --git a/helm/litellm/tests/lens_worker_tests.yaml b/helm/litellm/tests/lens_worker_tests.yaml index f283cf8e268f..5230f14efb49 100644 --- a/helm/litellm/tests/lens_worker_tests.yaml +++ b/helm/litellm/tests/lens_worker_tests.yaml @@ -102,3 +102,13 @@ tests: content: name: LENS_WORKER_IMAGE value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4 + - it: normalizes a numeric backend tag to the published worker tag + template: lens/deployment.yaml + set: + backend.image.tag: 1.2.3-dev.4 + lensWorker.enabled: true + lensWorker.tokenSecret.name: lens-credential + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4 diff --git a/tests/e2e/migrations/lens_compose_smoke.sh b/tests/e2e/migrations/lens_compose_smoke.sh index c0844a26fd21..985e29a04950 100644 --- a/tests/e2e/migrations/lens_compose_smoke.sh +++ b/tests/e2e/migrations/lens_compose_smoke.sh @@ -12,7 +12,7 @@ trap cleanup EXIT umask 077 printf 'LITELLM_VERSION=0.0.0-lens-ci\nLITELLM_PORT=4418\nLITELLM_MASTER_KEY=%s\nLITELLM_SALT_KEY=sk-%s\n' \ "$master_key" "$(openssl rand -hex 32)" > "$qa_dir/env" -printf 'POSTGRES_PASSWORD=%s\nCLICKHOUSE_PASSWORD=%s\n' \ +printf 'POSTGRES_PASSWORD=%s:/?#@%%\nCLICKHOUSE_PASSWORD=%s:/?#@%%\n' \ "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" >> "$qa_dir/env" docker tag "${LITELLM_IMAGE:?Set LITELLM_IMAGE to the built gateway image}" ghcr.io/berriai/litellm:0.0.0-lens-ci docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f deploy/lens/Dockerfile \ From 61703efa3eb11cd1edbed72a36696b93bbb1baef Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 16:18:59 -0700 Subject: [PATCH 8/8] fix(lens): refuse worker recommendations for unidentified builds --- deploy/lens/README.md | 4 +- litellm/proxy/lens/endpoints.py | 18 +++++++-- litellm/proxy/lens/release.py | 21 ++++++---- scripts/lens_dev.sh | 6 ++- tests/e2e/migrations/lens_compose_smoke.sh | 30 ++++++++++++++ tests/proxy_behavior/lens/test_lifecycle.py | 3 +- tests/unit/proxy/lens/test_endpoints.py | 27 +++++++++++-- tests/unit/proxy/lens/test_release.py | 43 +++++++++++++++++++-- tests/unit/test_lens_dev.py | 14 +++++++ 9 files changed, 146 insertions(+), 20 deletions(-) diff --git a/deploy/lens/README.md b/deploy/lens/README.md index b6581ecf9d4b..afd2f409e210 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -191,4 +191,6 @@ Released gateway and worker images carry `LITELLM_RELEASE_TAG`. A worker announc The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Worker-only Compose accepts `LITELLM_VERSION` (without `v`) or an explicit `LENS_WORKER_IMAGE`. Release workers are available as `ghcr.io/berriai/litellm-lens-worker:vX.Y.Z` and `docker.io/litellm/litellm-lens-worker:vX.Y.Z`, including matching RC/dev suffixes, on amd64 and arm64 -For source development, use `make lens-dev`. Custom paired builds must pass the same `--build-arg LITELLM_RELEASE_TAG=` to the proxy/backend and worker. The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds +For source development, use `make lens-dev`, which gives the proxy and source worker the same commit identity. For custom containers, build both from the same checkout with `--build-arg LITELLM_RELEASE_TAG=sha-$(git rev-parse HEAD)` and set the proxy's `LENS_WORKER_IMAGE` to the worker image you built. An unlabelled custom build refuses worker setup and claims instead of guessing from the Python package version. Normal package-index installations use their installed release version + +The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds. The public commit-tagged worker workflow publishes on Lens-related changes, so an arbitrary `main` commit may require building your own pair; do not substitute the newest available worker diff --git a/litellm/proxy/lens/endpoints.py b/litellm/proxy/lens/endpoints.py index 0023ee1173c2..3ae9db0a600d 100644 --- a/litellm/proxy/lens/endpoints.py +++ b/litellm/proxy/lens/endpoints.py @@ -422,9 +422,20 @@ class WorkerName(WorkerBilling): name: str = Field(default="Lens worker", min_length=1) +def configured_worker_image() -> str: + if image := worker_image(): + return image + raise HTTPException( + 503, + "This LiteLLM build has no release identity. Use a published release, make lens-dev, " + "or build the gateway and worker from the same commit with the same LITELLM_RELEASE_TAG.", + ) + + @router.post("/workers/register", response_model=WorkerCreated) async def register_worker(body: WorkerName, auth: Auth) -> WorkerCreated: scope: Final = user_scope(auth, write=True) + image: Final = configured_worker_image() await validate_key(body.analysis_key_id) token: Final = "lens-" + secrets.token_urlsafe(40) worker: Final = Worker( @@ -435,7 +446,7 @@ async def register_worker(body: WorkerName, auth: Auth) -> WorkerCreated: last_seen=datetime(1970, 1, 1, tzinfo=timezone.utc), ) await repository().save_worker(worker, hashlib.sha256(token.encode()).hexdigest()) - return WorkerCreated(worker=worker, token=token, image=worker_image()) + return WorkerCreated(worker=worker, token=token, image=image) @router.put("/workers/{worker_id}/billing-key", response_model=Worker) @@ -468,9 +479,10 @@ async def revoke_worker(worker_id: str, auth: Auth) -> bool: @router.post("/worker/claim", response_model=Claim | None) async def claim(worker: WorkerAuth, protocol_version: int = 1, worker_release: str = "") -> Claim | None: + image: Final = configured_worker_image() expected: Final = release_tag() - if protocol_version != PROTOCOL_VERSION or (expected and worker_release != expected): - raise HTTPException(409, f"Upgrade the Lens worker to {worker_image()} and retry") + if protocol_version != PROTOCOL_VERSION or worker_release != expected: + raise HTTPException(409, f"Upgrade the Lens worker to {image} and retry") if worker.analysis_key_id is None: raise HTTPException(409, "Assign an analysis key to this worker in Lens setup") now: Final = datetime.now(timezone.utc) diff --git a/litellm/proxy/lens/release.py b/litellm/proxy/lens/release.py index 7abc89351390..0338858a6a55 100644 --- a/litellm/proxy/lens/release.py +++ b/litellm/proxy/lens/release.py @@ -1,28 +1,35 @@ import os -from importlib.metadata import PackageNotFoundError, version +from importlib.metadata import PackageNotFoundError, distribution +from pathlib import Path from typing import Final PROTOCOL_VERSION: Final = 4 def release_tag() -> str: - if tag := os.environ.get("LITELLM_RELEASE_TAG", ""): - return tag + if "LITELLM_RELEASE_TAG" in os.environ: + return os.environ["LITELLM_RELEASE_TAG"] try: - installed: Final = version("litellm") + installed: Final = distribution("litellm") except PackageNotFoundError: return "" + if installed.read_text("direct_url.json") is not None: + return "" + if Path(str(installed.locate_file("litellm/proxy/lens/release.py"))).resolve() != Path(__file__).resolve(): + return "" from packaging.version import Version - parsed: Final = Version(installed) + parsed: Final = Version(installed.version) suffix: Final = f"-dev.{parsed.dev}" if parsed.dev is not None else f"-rc.{parsed.pre[1]}" if parsed.pre else "" return f"v{parsed.base_version}{suffix}" def worker_image() -> str: + tag: Final = release_tag() + if not tag: + return "" override: Final = os.environ.get("LENS_WORKER_IMAGE", "") if override: return override - tag: Final = release_tag() - return f"ghcr.io/berriai/litellm-lens-worker:{tag}" if tag else "litellm-lens-worker:local" + return f"ghcr.io/berriai/litellm-lens-worker:{tag}" diff --git a/scripts/lens_dev.sh b/scripts/lens_dev.sh index 7ab4b9bb433f..46408fdaf19a 100755 --- a/scripts/lens_dev.sh +++ b/scripts/lens_dev.sh @@ -13,6 +13,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +source_release_tag="sha-$(git -C "$repo_root" rev-parse HEAD)" proxy_port="${LENS_DEV_PROXY_PORT:-4000}" ui_port="${LENS_DEV_UI_PORT:-3000}" state_dir="${LENS_DEV_STATE_DIR:-$repo_root/.lens-dev}" @@ -108,6 +109,8 @@ proxy_env() { unset ANTHROPIC_BASE_URL ANTHROPIC_AUTH_TOKEN ANTHROPIC_CUSTOM_HEADERS OPENAI_BASE_URL OPENAI_API_BASE for var in $(compgen -e | grep '^REDIS_' || true); do unset "$var"; done eval "$1" + export LITELLM_RELEASE_TAG="$source_release_tag" + export LENS_WORKER_IMAGE=litellm-lens-worker:local export LITELLM_MODE=PRODUCTION export LITELLM_MASTER_KEY="$master_key" if [ "$master_key" = sk-1234 ]; then export LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true; fi @@ -236,7 +239,8 @@ main() { wait_for_proxy "$proxy_pid" ensure_worker_token - LITELLM_MODE=PRODUCTION LITELLM_URL="$proxy_url" LENS_WORKER_TOKEN="$(cat "$token_file")" \ + LITELLM_RELEASE_TAG="$source_release_tag" \ + LITELLM_MODE=PRODUCTION LITELLM_URL="$proxy_url" LENS_WORKER_TOKEN="$(cat "$token_file")" \ "$py" -c "import asyncio, logging; from litellm.proxy.lens.worker import main; logging.basicConfig(level=logging.INFO); asyncio.run(main())" \ < /dev/null > "$log_dir/worker.log" 2>&1 & pids+=("$!") diff --git a/tests/e2e/migrations/lens_compose_smoke.sh b/tests/e2e/migrations/lens_compose_smoke.sh index 985e29a04950..69a13b58d880 100644 --- a/tests/e2e/migrations/lens_compose_smoke.sh +++ b/tests/e2e/migrations/lens_compose_smoke.sh @@ -111,3 +111,33 @@ api /lens > "$qa_dir/restarted.json" jq -e --arg id "$worker_id" --arg key "$key_id" \ '.workers[] | select(.id == $id and .analysis_key_id == $key)' "$qa_dir/restarted.json" > /dev/null printf 'Compose restart: trace, worker identity, token and billing assignment preserved; wrong release rejected\n' + +cat > "$qa_dir/unversioned.yaml" <<'EOF' +services: + litellm: + environment: + LITELLM_RELEASE_TAG: "" +EOF +"${compose[@]}" -f "$qa_dir/unversioned.yaml" up -d litellm +for attempt in $(seq 1 90); do + if api /health/liveliness > /dev/null 2>&1; then break; fi + sleep 2 +done +api /health/liveliness > /dev/null +status=$(curl --silent --show-error --max-time 30 -o "$qa_dir/unversioned-registration.json" -w '%{http_code}' \ + -H "Authorization: Bearer $master_key" -H 'Content-Type: application/json' \ + -d "@$qa_dir/registration.json" 'http://127.0.0.1:4418/lens/workers/register') +[[ "$status" == 503 ]] +jq -e '.detail | contains("no release identity")' "$qa_dir/unversioned-registration.json" > /dev/null +status=$(curl --silent --show-error --max-time 30 -o "$qa_dir/unversioned-claim.json" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer $(jq -r '.token' "$qa_dir/worker.json")" \ + 'http://127.0.0.1:4418/lens/worker/claim?protocol_version=4&worker_release=') +[[ "$status" == 503 ]] +jq -e '.detail | contains("no release identity")' "$qa_dir/unversioned-claim.json" > /dev/null +api /lens > "$qa_dir/unversioned-workers.json" +jq -e --arg id "$worker_id" '.workers | length == 1 and .[0].id == $id' "$qa_dir/unversioned-workers.json" > /dev/null +"${compose[@]}" up -d litellm +heartbeat_after=$(date -u +'%Y-%m-%dT%H:%M:%S') +connected +trace_saved +printf 'Unversioned gateway: setup and claims refused without guessing; original worker and trace recovered\n' diff --git a/tests/proxy_behavior/lens/test_lifecycle.py b/tests/proxy_behavior/lens/test_lifecycle.py index 5bd41f889d88..e3027474baa0 100644 --- a/tests/proxy_behavior/lens/test_lifecycle.py +++ b/tests/proxy_behavior/lens/test_lifecycle.py @@ -36,7 +36,8 @@ @pytest_asyncio.fixture(loop_scope="function") -async def lens_database() -> AsyncIterator[PrismaClient]: +async def lens_database(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[PrismaClient]: + monkeypatch.setenv("LITELLM_RELEASE_TAG", "v0.0.0-lens-lifecycle") original_db: Final = proxy_server.prisma_client original_router: Final = proxy_server.llm_router original_settings: Final = proxy_server.general_settings diff --git a/tests/unit/proxy/lens/test_endpoints.py b/tests/unit/proxy/lens/test_endpoints.py index 972e5f960316..a0d01878277f 100644 --- a/tests/unit/proxy/lens/test_endpoints.py +++ b/tests/unit/proxy/lens/test_endpoints.py @@ -6,16 +6,16 @@ from pydantic import ValidationError import litellm -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm import Router +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.lens.endpoints import ( list_agents, run_settings, run_window, user_scope, + validate_model, watchable, watching, - validate_model, worker_supports_model, ) from litellm.proxy.lens.models import Lens, LensSettings, RunRequest, Scope @@ -161,10 +161,13 @@ def test_invalid_explicit_execution_ids_are_rejected(identity: str) -> None: @pytest.mark.parametrize("protocol_version", (1, 2, 3)) @pytest.mark.asyncio -async def test_incompatible_worker_is_rejected_before_claiming_work(protocol_version: int) -> None: +async def test_incompatible_worker_is_rejected_before_claiming_work( + protocol_version: int, monkeypatch: pytest.MonkeyPatch +) -> None: from litellm.proxy.lens.endpoints import claim from tests.unit.proxy.lens.test_state import worker + monkeypatch.setenv("LITELLM_RELEASE_TAG", "v1.2.3") with pytest.raises(HTTPException) as error: await claim(worker(), protocol_version=protocol_version) assert error.value.status_code == 409 @@ -309,3 +312,21 @@ async def test_different_release_is_rejected_before_accessing_jobs( await claim(worker(), protocol_version=PROTOCOL_VERSION, worker_release=worker_release) assert error.value.status_code == 409 assert "ghcr.io/berriai/litellm-lens-worker:v1.2.3" in error.value.detail + + +@pytest.mark.asyncio +async def test_unknown_gateway_release_refuses_registration_and_claims(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.lens.endpoints import WorkerName, claim, register_worker + from litellm.proxy.lens.release import PROTOCOL_VERSION + from tests.unit.proxy.lens.test_state import worker + + monkeypatch.setenv("LITELLM_RELEASE_TAG", "") + monkeypatch.setenv("LENS_WORKER_IMAGE", "registry.example/lens-worker:old") + with pytest.raises(HTTPException) as registration_error: + await register_worker(WorkerName(analysis_key_id="a" * 64), UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)) + assert registration_error.value.status_code == 503 + assert "LITELLM_RELEASE_TAG" in registration_error.value.detail + with pytest.raises(HTTPException) as claim_error: + await claim(worker(), protocol_version=PROTOCOL_VERSION, worker_release="") + assert claim_error.value.status_code == 503 + assert claim_error.value.detail == registration_error.value.detail diff --git a/tests/unit/proxy/lens/test_release.py b/tests/unit/proxy/lens/test_release.py index eb62330558cf..95d637465c4b 100644 --- a/tests/unit/proxy/lens/test_release.py +++ b/tests/unit/proxy/lens/test_release.py @@ -1,3 +1,5 @@ +from importlib.metadata import Distribution, PackageNotFoundError, PathDistribution +from pathlib import Path from typing import Final import pytest @@ -24,16 +26,49 @@ def test_private_registry_override_keeps_its_exact_digest(monkeypatch: pytest.Mo (("1.2.3", "v1.2.3"), ("1.2.3rc4", "v1.2.3-rc.4"), ("1.2.3.dev5", "v1.2.3-dev.5")), ) def test_python_installs_recommend_the_matching_worker( - monkeypatch: pytest.MonkeyPatch, installed: str, expected: str + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, installed: str, expected: str ) -> None: from litellm.proxy.lens import release - def installed_version(name: str) -> str: + metadata: Final = tmp_path / "litellm.dist-info" + metadata.mkdir() + metadata.joinpath("METADATA").write_text(f"Name: litellm\nVersion: {installed}\n") + + def installed_distribution(name: str) -> Distribution: assert name == "litellm" - return installed + return PathDistribution(metadata) monkeypatch.delenv("LITELLM_RELEASE_TAG", raising=False) monkeypatch.delenv("LENS_WORKER_IMAGE", raising=False) - monkeypatch.setattr(release, "version", installed_version) + monkeypatch.setattr(release, "distribution", installed_distribution) + monkeypatch.setattr(release, "__file__", str(tmp_path / "litellm/proxy/lens/release.py")) assert release.release_tag() == expected assert worker_image() == f"ghcr.io/berriai/litellm-lens-worker:{expected}" + + +@pytest.mark.parametrize("source", ("checkout", "direct-install", "unversioned-container", "missing-package")) +def test_unknown_source_never_falls_back_to_a_package_version_or_image_override( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, source: str +) -> None: + from litellm.proxy.lens import release + + metadata: Final = tmp_path / "litellm.dist-info" + metadata.mkdir() + metadata.joinpath("METADATA").write_text("Name: litellm\nVersion: 1.2.3\n") + if source == "direct-install": + metadata.joinpath("direct_url.json").write_text('{"url":"file:///checkout","dir_info":{"editable":true}}') + + def installed_distribution(name: str) -> Distribution: + if source == "missing-package": + raise PackageNotFoundError(name) + return PathDistribution(metadata) + + monkeypatch.delenv("LITELLM_RELEASE_TAG", raising=False) + monkeypatch.setenv("LENS_WORKER_IMAGE", "registry.example/lens-worker:old") + monkeypatch.setattr(release, "distribution", installed_distribution) + if source != "checkout": + monkeypatch.setattr(release, "__file__", str(tmp_path / "litellm/proxy/lens/release.py")) + if source == "unversioned-container": + monkeypatch.setenv("LITELLM_RELEASE_TAG", "") + assert release.release_tag() == "" + assert worker_image() == "" diff --git a/tests/unit/test_lens_dev.py b/tests/unit/test_lens_dev.py index e14b27fd27ef..cbc8aced8ac4 100644 --- a/tests/unit/test_lens_dev.py +++ b/tests/unit/test_lens_dev.py @@ -2,6 +2,7 @@ import subprocess import sys from pathlib import Path +from typing import Final ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "lens_dev.sh" @@ -123,6 +124,19 @@ def test_proxy_env_permits_the_weak_key_only_when_chosen(tmp_path): assert "LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true" in proc.stdout +def test_source_development_overrides_an_inherited_release_with_its_own_commit(tmp_path: Path) -> None: + proc: Final = _run( + tmp_path, + 'proxy_env "export LITELLM_RELEASE_TAG=v0.0.0-old"; ' + 'test "$LITELLM_RELEASE_TAG" = "sha-$(git -C "$repo_root" rev-parse HEAD)"; ' + 'printf "%s" "$LENS_WORKER_IMAGE"', + LITELLM_RELEASE_TAG="v0.0.0-old", + LENS_WORKER_IMAGE="registry.example/lens-worker:old", + ) + assert proc.returncode == 0, proc.stderr + assert proc.stdout == "litellm-lens-worker:local" + + def test_external_database_url_never_starts_compose_postgres(tmp_path): docker = tmp_path / "bin" / "docker" proc = _run(