[Test] UI - E2E: Add Playwright tests with local PostgreSQL

.github/workflows/test-litellm-ui-e2e.yml

@@ -0,0 +1,99 @@
+name: UI E2E Playwright Tests
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  ui_e2e_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: "22"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-ui-e2e-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-ui-e2e-
+            ${{ runner.os }}-poetry-
+
+      - name: Install Python dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy" --quiet
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Install Playwright
+        run: |
+          cd tests/ui_e2e_tests
+          npm install --silent
+          npx playwright install --with-deps chromium
+
+      - name: Run UI E2E tests
+        env:
+          CI: "true"
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+        run: |
+          cd tests/ui_e2e_tests
+          ./run_e2e.sh
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: playwright-results
+          path: |
+            tests/ui_e2e_tests/test-results/
+            tests/ui_e2e_tests/playwright-report/
+          retention-days: 7

tests/ui_e2e_tests/constants.ts

@@ -0,0 +1,40 @@
+export const ADMIN_STORAGE_PATH = "admin.storageState.json";
+
+// Page enum — maps to ?page= query parameter values in the UI
+export enum Page {
+  ApiKeys = "api-keys",
+  Teams = "teams",
+  AdminSettings = "settings",
+}
+
+// Test user credentials — all users have password "test" (hashed in seed.sql)
+export enum Role {
+  ProxyAdmin = "proxy_admin",
+  ProxyAdminViewer = "proxy_admin_viewer",
+  InternalUser = "internal_user",
+  InternalUserViewer = "internal_user_viewer",
+  TeamAdmin = "team_admin",
+}
+
+export const users: Record<Role, { email: string; password: string }> = {
+  [Role.ProxyAdmin]: {
+    email: "admin",
+    password: process.env.LITELLM_MASTER_KEY || "sk-1234",
+  },
+  [Role.ProxyAdminViewer]: {
+    email: "adminviewer@test.local",
+    password: "test",
+  },
+  [Role.InternalUser]: {
+    email: "internal@test.local",
+    password: "test",
+  },
+  [Role.InternalUserViewer]: {
+    email: "viewer@test.local",
+    password: "test",
+  },
+  [Role.TeamAdmin]: {
+    email: "teamadmin@test.local",
+    password: "test",
+  },
+};

tests/ui_e2e_tests/fixtures/config.yml

@@ -0,0 +1,16 @@
+model_list:
+  - model_name: fake-openai-gpt-4
+    litellm_params:
+      model: openai/fake-gpt-4
+      api_base: os.environ/MOCK_LLM_URL
+      api_key: fake-key
+  - model_name: fake-anthropic-claude
+    litellm_params:
+      model: openai/fake-claude
+      api_base: os.environ/MOCK_LLM_URL
+      api_key: fake-key
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  database_url: os.environ/DATABASE_URL
+  store_prompts_in_spend_logs: true

tests/ui_e2e_tests/fixtures/mock_llm_server/server.py

@@ -0,0 +1,118 @@
+"""
+Mock LLM server for UI e2e tests.
+Responds to OpenAI-format endpoints with canned responses.
+"""
+
+import time
+import json
+import uuid
+
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+
+
+app = FastAPI(title="Mock LLM Server")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/health")
+async def health():
+    return {"status": "ok"}
+
+
+@app.get("/v1/models")
+@app.get("/models")
+async def list_models():
+    return {
+        "object": "list",
+        "data": [
+            {"id": "fake-gpt-4", "object": "model", "owned_by": "mock"},
+            {"id": "fake-claude", "object": "model", "owned_by": "mock"},
+        ],
+    }
+
+
+@app.post("/v1/chat/completions")
+@app.post("/chat/completions")
+async def chat_completions(request: Request):
+    body = await request.json()
+    model = body.get("model", "mock-model")
+    stream = body.get("stream", False)
+
+    response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
+    created = int(time.time())
+
+    if stream:
+        async def stream_generator():
+            chunk = {
+                "id": response_id,
+                "object": "chat.completion.chunk",
+                "created": created,
+                "model": model,
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {"role": "assistant", "content": "This is a mock response."},
+                        "finish_reason": None,
+                    }
+                ],
+            }
+            yield f"data: {json.dumps(chunk)}\n\n"
+
+            done_chunk = {
+                "id": response_id,
+                "object": "chat.completion.chunk",
+                "created": created,
+                "model": model,
+                "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+            }
+            yield f"data: {json.dumps(done_chunk)}\n\n"
+            yield "data: [DONE]\n\n"
+
+        return StreamingResponse(
+            stream_generator(), media_type="text/event-stream"
+        )
+
+    return {
+        "id": response_id,
+        "object": "chat.completion",
+        "created": created,
+        "model": model,
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": "This is a mock response."},
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18},
+    }
+
+
+@app.post("/v1/embeddings")
+@app.post("/embeddings")
+async def embeddings(request: Request):
+    body = await request.json()
+    inputs = body.get("input", [""])
+    if isinstance(inputs, str):
+        inputs = [inputs]
+    return {
+        "object": "list",
+        "data": [
+            {"object": "embedding", "index": i, "embedding": [0.0] * 1536}
+            for i in range(len(inputs))
+        ],
+        "model": body.get("model", "mock-embedding"),
+        "usage": {"prompt_tokens": 5, "total_tokens": 5},
+    }
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="127.0.0.1", port=8090)

tests/ui_e2e_tests/fixtures/seed.sql

@@ -0,0 +1,103 @@
+-- UI E2E Test Database Seed
+-- Run with: psql $DATABASE_URL -f seed.sql
+
+-- ============================================================
+-- 1. Budget Table (must be first — referenced by org FK)
+-- ============================================================
+INSERT INTO "LiteLLM_BudgetTable" (
+    budget_id, max_budget, created_by, updated_by
+) VALUES (
+    'e2e-budget-org', 1000.0, 'e2e-proxy-admin', 'e2e-proxy-admin'
+) ON CONFLICT (budget_id) DO NOTHING;
+
+-- ============================================================
+-- 2. Organization
+-- ============================================================
+INSERT INTO "LiteLLM_OrganizationTable" (
+    organization_id, organization_alias, budget_id, metadata, models, spend,
+    model_spend, created_by, updated_by
+) VALUES (
+    'e2e-org-main', 'E2E Organization', 'e2e-budget-org', '{}'::jsonb,
+    ARRAY[]::text[], 0.0, '{}'::jsonb, 'e2e-proxy-admin', 'e2e-proxy-admin'
+) ON CONFLICT (organization_id) DO NOTHING;
+
+-- ============================================================
+-- 3. Users (password is scrypt hash of "test")
+-- ============================================================
+INSERT INTO "LiteLLM_UserTable" (
+    user_id, user_email, user_role, password, teams, models, metadata,
+    spend, model_spend, model_max_budget
+) VALUES
+(
+    'e2e-proxy-admin', 'admin@test.local', 'proxy_admin', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY['e2e-team-crud']::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+),
+(
+    'e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+),
+(
+    'e2e-internal-user', 'internal@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY['e2e-team-crud', 'e2e-team-org']::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+),
+(
+    'e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+),
+(
+    'e2e-team-admin', 'teamadmin@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr',
+    ARRAY['e2e-team-crud', 'e2e-team-delete']::text[], ARRAY[]::text[], '{}'::jsonb,
+    0.0, '{}'::jsonb, '{}'::jsonb
+)
+ON CONFLICT (user_id) DO NOTHING;
+
+-- ============================================================
+-- 4. Teams
+-- ============================================================
+INSERT INTO "LiteLLM_TeamTable" (
+    team_id, team_alias, organization_id, admins, members,
+    members_with_roles, metadata, models, spend, model_spend,
+    model_max_budget, blocked
+) VALUES
+(
+    'e2e-team-crud', 'E2E Team CRUD', NULL,
+    ARRAY['e2e-team-admin']::text[],
+    ARRAY['e2e-team-admin', 'e2e-internal-user']::text[],
+    '[{"role": "admin", "user_id": "e2e-team-admin"}, {"role": "user", "user_id": "e2e-internal-user"}]'::jsonb,
+    '{}'::jsonb,
+    ARRAY['fake-openai-gpt-4', 'fake-anthropic-claude']::text[],
+    0.0, '{}'::jsonb, '{}'::jsonb, false
+),
+(
+    'e2e-team-delete', 'E2E Team Delete', NULL,
+    ARRAY['e2e-team-admin']::text[],
+    ARRAY['e2e-team-admin']::text[],
+    '[{"role": "admin", "user_id": "e2e-team-admin"}]'::jsonb,
+    '{}'::jsonb,
+    ARRAY['fake-openai-gpt-4']::text[],
+    0.0, '{}'::jsonb, '{}'::jsonb, false
+),
+(
+    'e2e-team-org', 'E2E Team In Org', 'e2e-org-main',
+    ARRAY[]::text[],
+    ARRAY['e2e-internal-user']::text[],
+    '[{"role": "user", "user_id": "e2e-internal-user"}]'::jsonb,
+    '{}'::jsonb,
+    ARRAY['fake-openai-gpt-4']::text[],
+    0.0, '{}'::jsonb, '{}'::jsonb, false
+)
+ON CONFLICT (team_id) DO NOTHING;
+
+-- ============================================================
+-- 5. Team Memberships
+-- ============================================================
+INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend) VALUES
+    ('e2e-team-admin', 'e2e-team-crud', 0.0),
+    ('e2e-internal-user', 'e2e-team-crud', 0.0),
+    ('e2e-team-admin', 'e2e-team-delete', 0.0),
+    ('e2e-internal-user', 'e2e-team-org', 0.0)
+ON CONFLICT (user_id, team_id) DO NOTHING;