From 6eb78e46c966f5f663abf9c00b28dbefbd094bba Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 09:49:55 +0200 Subject: [PATCH 01/28] fix(ui): resolve login redirect loop when reverse proxy adds HttpOnly to cookies When LiteLLM is behind nginx-ingress or similar with security-hardened configs, the reverse proxy adds HttpOnly to all Set-Cookie headers. This makes the JWT token unreadable by JavaScript, causing an infinite login redirect loop. Fix by returning the JWT token in the /v2/login response body so the frontend can set a JS-accessible cookie directly. Fixes #19663 Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/proxy_server.py | 2 +- tests/test_litellm/proxy/test_proxy_server.py | 8 +++--- .../src/components/networking.test.ts | 1 + .../src/components/networking.tsx | 6 ++++- .../src/utils/cookieUtils.test.ts | 26 ++++++++++++++++++- ui/litellm-dashboard/src/utils/cookieUtils.ts | 10 +++++++ 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 77e2a88796e5..ddfba140ff2c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11030,7 +11030,7 @@ async def login_v2(request: Request): # noqa: PLR0915 litellm_dashboard_ui += "?login=success" json_response = JSONResponse( - content={"redirect_url": litellm_dashboard_ui}, + content={"redirect_url": litellm_dashboard_ui, "token": jwt_token}, status_code=status.HTTP_200_OK, ) json_response.set_cookie(key="token", value=jwt_token) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 112a06b1731c..de3ffb9f8390 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -104,10 +104,10 @@ def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): ) assert response.status_code == 200 - assert ( - response.json() - == {"redirect_url": "http://testserver/ui/?login=success"} - ) + assert response.json() == { + "redirect_url": "http://testserver/ui/?login=success", + "token": "signed-token", + } assert response.cookies.get("token") == "signed-token" mock_authenticate_user.assert_awaited_once_with( diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index c57dcb97eb92..459492f9eddd 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -5,6 +5,7 @@ import * as Networking from "./networking"; vi.mock("@/utils/cookieUtils", () => ({ clearTokenCookies: vi.fn(), getCookie: vi.fn(), + setTokenCookie: vi.fn(), })); vi.mock("./molecules/notifications_manager", () => ({ diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 6b5ec5139e68..c8dd00cbb9a4 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -69,7 +69,7 @@ export const getInProductNudgesCall = async (accessToken: string) => { * Helper file for calls being made to proxy */ import { message } from "antd"; -import { clearTokenCookies } from "@/utils/cookieUtils"; +import { clearTokenCookies, setTokenCookie } from "@/utils/cookieUtils"; import { TagNewRequest, TagUpdateRequest, TagListResponse, TagInfoResponse } from "./tag_management/types"; import { Team } from "./key_team_helpers/key_list"; import { UserInfo } from "./view_users/types"; @@ -8976,6 +8976,7 @@ export interface LoginRequest { interface LoginResponse { redirect_url: string; + token?: string; } export const loginCall = async (username: string, password: string): Promise => { @@ -9003,6 +9004,9 @@ export const loginCall = async (username: string, password: string): Promise { beforeEach(() => { @@ -118,6 +118,30 @@ describe("cookieUtils", () => { }); }); + describe("setTokenCookie", () => { + it("should set a token cookie readable by getCookie", () => { + setTokenCookie("my-jwt-token"); + expect(getCookie("token")).toBe("my-jwt-token"); + }); + + it("should overwrite an existing token cookie", () => { + setTokenCookie("old-token"); + expect(getCookie("token")).toBe("old-token"); + + setTokenCookie("new-token"); + expect(getCookie("token")).toBe("new-token"); + }); + + it("should not throw when document is undefined (server-side rendering)", () => { + const originalDocument = global.document; + delete (global as any).document; + + expect(() => setTokenCookie("token")).not.toThrow(); + + global.document = originalDocument; + }); + }); + describe("getCookie", () => { it("should return cookie value when it exists", () => { document.cookie = "token=my-test-token; path=/"; diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 01add36542c1..3b7b3551ebbe 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -46,6 +46,16 @@ export function clearTokenCookies() { console.log("After clearing cookies:", document.cookie); } +/** + * Sets the token cookie from a login response body. + * This ensures the token is JS-accessible even when a reverse proxy adds HttpOnly to server-set cookies. + */ +export function setTokenCookie(token: string) { + if (typeof document === "undefined") return; + const isSecure = window.location.protocol === "https:"; + document.cookie = `token=${token}; Path=/; SameSite=Lax${isSecure ? "; Secure" : ""}`; +} + /** * Gets a cookie value by name * @param name The name of the cookie to retrieve From a8864ea12bc33c8acaa934fd25784110e72a018c Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 10:02:10 +0200 Subject: [PATCH 02/28] fix: address Greptile review feedback - Add window guard to setTokenCookie for SSR consistency with clearTokenCookies - Add SSR test for window undefined case - Add code comment explaining why JWT is included in response body Co-Authored-By: Claude Opus 4.6 --- litellm/proxy/proxy_server.py | 3 +++ ui/litellm-dashboard/src/utils/cookieUtils.test.ts | 12 ++++++++++++ ui/litellm-dashboard/src/utils/cookieUtils.ts | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ddfba140ff2c..c8fea12b272b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11029,6 +11029,9 @@ async def login_v2(request: Request): # noqa: PLR0915 litellm_dashboard_ui += "/ui/" litellm_dashboard_ui += "?login=success" + # Token is included in the response body so the UI can set a JS-accessible + # cookie even when a reverse proxy (e.g. nginx-ingress) adds HttpOnly to the + # server-set cookie, which would otherwise cause an infinite login redirect. json_response = JSONResponse( content={"redirect_url": litellm_dashboard_ui, "token": jwt_token}, status_code=status.HTTP_200_OK, diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts index 84935606d621..0e3ec1eb8c69 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts @@ -140,6 +140,18 @@ describe("cookieUtils", () => { global.document = originalDocument; }); + + it("should not throw when window is undefined (server-side rendering)", () => { + const originalWindow = global.window; + const originalDocument = global.document; + delete (global as any).window; + delete (global as any).document; + + expect(() => setTokenCookie("token")).not.toThrow(); + + global.window = originalWindow; + global.document = originalDocument; + }); }); describe("getCookie", () => { diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 3b7b3551ebbe..9a6f34a0acc1 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -51,7 +51,7 @@ export function clearTokenCookies() { * This ensures the token is JS-accessible even when a reverse proxy adds HttpOnly to server-set cookies. */ export function setTokenCookie(token: string) { - if (typeof document === "undefined") return; + if (typeof window === "undefined" || typeof document === "undefined") return; const isSecure = window.location.protocol === "https:"; document.cookie = `token=${token}; Path=/; SameSite=Lax${isSecure ? "; Secure" : ""}`; } From 2c5caac5310c0bdd4a2acfe666326532033a65f0 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 10:09:52 +0200 Subject: [PATCH 03/28] fix: address second round of Greptile review feedback - Add loginCall integration tests verifying setTokenCookie is called with token and skipped when absent (backward-compatibility path) - Use encodeURIComponent/decodeURIComponent in setTokenCookie/getCookie for defense-in-depth against non-standard token formats Co-Authored-By: Claude Opus 4.6 --- .../src/components/networking.test.ts | 32 +++++++++++++++++++ ui/litellm-dashboard/src/utils/cookieUtils.ts | 4 +-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 459492f9eddd..2f4d41f8bb34 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -80,6 +80,38 @@ describe("networking - expired session handling", () => { }); }); +describe("loginCall - setTokenCookie integration", () => { + const originalFetch = global.fetch; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + it("calls setTokenCookie when response includes token", async () => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ redirect_url: "/ui/?login=success", token: "my-jwt" }), + }) as any; + const { setTokenCookie } = await import("@/utils/cookieUtils"); + await Networking.loginCall("admin", "pass"); + expect(setTokenCookie).toHaveBeenCalledWith("my-jwt"); + }); + + it("does not call setTokenCookie when response has no token", async () => { + global.fetch = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ redirect_url: "/ui/?login=success" }), + }) as any; + const { setTokenCookie } = await import("@/utils/cookieUtils"); + await Networking.loginCall("admin", "pass"); + expect(setTokenCookie).not.toHaveBeenCalled(); + }); +}); + describe("daily activity helpers", () => { const startTime = new Date("2025-02-12T00:00:00.000Z"); const endTime = new Date("2025-02-19T00:00:00.000Z"); diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 9a6f34a0acc1..c1a9803d615a 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -53,7 +53,7 @@ export function clearTokenCookies() { export function setTokenCookie(token: string) { if (typeof window === "undefined" || typeof document === "undefined") return; const isSecure = window.location.protocol === "https:"; - document.cookie = `token=${token}; Path=/; SameSite=Lax${isSecure ? "; Secure" : ""}`; + document.cookie = `token=${encodeURIComponent(token)}; Path=/; SameSite=Lax${isSecure ? "; Secure" : ""}`; } /** @@ -64,5 +64,5 @@ export function setTokenCookie(token: string) { export function getCookie(name: string) { if (typeof document === "undefined") return null; const cookieValue = document.cookie.split("; ").find((row) => row.startsWith(name + "=")); - return cookieValue ? cookieValue.split("=")[1] : null; + return cookieValue ? decodeURIComponent(cookieValue.split("=")[1]) : null; } From 04f835ecfae672d4851262840cc7b380d60a03ae Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 10:15:21 +0200 Subject: [PATCH 04/28] Update ui/litellm-dashboard/src/utils/cookieUtils.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- ui/litellm-dashboard/src/utils/cookieUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index c1a9803d615a..07a782686c94 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -64,5 +64,5 @@ export function setTokenCookie(token: string) { export function getCookie(name: string) { if (typeof document === "undefined") return null; const cookieValue = document.cookie.split("; ").find((row) => row.startsWith(name + "=")); - return cookieValue ? decodeURIComponent(cookieValue.split("=")[1]) : null; + return cookieValue ? (() => { try { return decodeURIComponent(cookieValue.split("=")[1]); } catch { return cookieValue.split("=")[1]; } })() : null; } From 22409da93210166eb28540ee75f144866efcf2e8 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 10:22:32 +0200 Subject: [PATCH 05/28] Update ui/litellm-dashboard/src/utils/cookieUtils.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- ui/litellm-dashboard/src/utils/cookieUtils.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 07a782686c94..6fe43f30400d 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -64,5 +64,12 @@ export function setTokenCookie(token: string) { export function getCookie(name: string) { if (typeof document === "undefined") return null; const cookieValue = document.cookie.split("; ").find((row) => row.startsWith(name + "=")); - return cookieValue ? (() => { try { return decodeURIComponent(cookieValue.split("=")[1]); } catch { return cookieValue.split("=")[1]; } })() : null; + if (!cookieValue) return null; + const rawValue = cookieValue.split("=")[1]; + try { + return decodeURIComponent(rawValue); + } catch { + return rawValue; + } +} } From 461cf00408aac241fc294de407cd6e37a45e0b59 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 10:58:06 +0200 Subject: [PATCH 06/28] fix(ui): use sessionStorage instead of cookie for login token storage Replace setTokenCookie (which is a no-op when reverse proxy adds HttpOnly) with storeLoginToken using sessionStorage. Add sessionStorage fallback to getCookie so the token is found even when the cookie is HttpOnly. Also handle '=' in cookie values with .slice(1).join("=") and clear sessionStorage on logout. Co-Authored-By: Claude Opus 4.6 --- .../src/components/networking.test.ts | 16 +++--- .../src/components/networking.tsx | 4 +- .../src/utils/cookieUtils.test.ts | 55 +++++++++++-------- ui/litellm-dashboard/src/utils/cookieUtils.ts | 48 +++++++++++----- 4 files changed, 77 insertions(+), 46 deletions(-) diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 2f4d41f8bb34..3c107fa586f5 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -5,7 +5,7 @@ import * as Networking from "./networking"; vi.mock("@/utils/cookieUtils", () => ({ clearTokenCookies: vi.fn(), getCookie: vi.fn(), - setTokenCookie: vi.fn(), + storeLoginToken: vi.fn(), })); vi.mock("./molecules/notifications_manager", () => ({ @@ -80,7 +80,7 @@ describe("networking - expired session handling", () => { }); }); -describe("loginCall - setTokenCookie integration", () => { +describe("loginCall - storeLoginToken integration", () => { const originalFetch = global.fetch; beforeEach(() => { @@ -91,24 +91,24 @@ describe("loginCall - setTokenCookie integration", () => { global.fetch = originalFetch; }); - it("calls setTokenCookie when response includes token", async () => { + it("calls storeLoginToken when response includes token", async () => { global.fetch = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ redirect_url: "/ui/?login=success", token: "my-jwt" }), }) as any; - const { setTokenCookie } = await import("@/utils/cookieUtils"); + const { storeLoginToken } = await import("@/utils/cookieUtils"); await Networking.loginCall("admin", "pass"); - expect(setTokenCookie).toHaveBeenCalledWith("my-jwt"); + expect(storeLoginToken).toHaveBeenCalledWith("my-jwt"); }); - it("does not call setTokenCookie when response has no token", async () => { + it("does not call storeLoginToken when response has no token", async () => { global.fetch = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ redirect_url: "/ui/?login=success" }), }) as any; - const { setTokenCookie } = await import("@/utils/cookieUtils"); + const { storeLoginToken } = await import("@/utils/cookieUtils"); await Networking.loginCall("admin", "pass"); - expect(setTokenCookie).not.toHaveBeenCalled(); + expect(storeLoginToken).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index c8dd00cbb9a4..35d9576a8c5e 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -69,7 +69,7 @@ export const getInProductNudgesCall = async (accessToken: string) => { * Helper file for calls being made to proxy */ import { message } from "antd"; -import { clearTokenCookies, setTokenCookie } from "@/utils/cookieUtils"; +import { clearTokenCookies, storeLoginToken } from "@/utils/cookieUtils"; import { TagNewRequest, TagUpdateRequest, TagListResponse, TagInfoResponse } from "./tag_management/types"; import { Team } from "./key_team_helpers/key_list"; import { UserInfo } from "./view_users/types"; @@ -9005,7 +9005,7 @@ export const loginCall = async (username: string, password: string): Promise { beforeEach(() => { document.cookie.split(";").forEach((c) => { document.cookie = c.replace(/^ +/, "").replace(/=.*/, "=;expires=" + new Date().toUTCString() + ";path=/"); }); + sessionStorage.clear(); vi.spyOn(console, "log").mockImplementation(() => {}); }); @@ -116,41 +117,35 @@ describe("cookieUtils", () => { vi.restoreAllMocks(); }); - }); - describe("setTokenCookie", () => { - it("should set a token cookie readable by getCookie", () => { - setTokenCookie("my-jwt-token"); - expect(getCookie("token")).toBe("my-jwt-token"); + it("should clear sessionStorage token", () => { + sessionStorage.setItem("token", "stored-token"); + clearTokenCookies(); + expect(sessionStorage.getItem("token")).toBeNull(); }); + }); - it("should overwrite an existing token cookie", () => { - setTokenCookie("old-token"); - expect(getCookie("token")).toBe("old-token"); - - setTokenCookie("new-token"); - expect(getCookie("token")).toBe("new-token"); + describe("storeLoginToken", () => { + it("should store the token in sessionStorage", () => { + storeLoginToken("my-jwt-token"); + expect(sessionStorage.getItem("token")).toBe("my-jwt-token"); }); - it("should not throw when document is undefined (server-side rendering)", () => { - const originalDocument = global.document; - delete (global as any).document; - - expect(() => setTokenCookie("token")).not.toThrow(); + it("should overwrite an existing token in sessionStorage", () => { + storeLoginToken("old-token"); + expect(sessionStorage.getItem("token")).toBe("old-token"); - global.document = originalDocument; + storeLoginToken("new-token"); + expect(sessionStorage.getItem("token")).toBe("new-token"); }); it("should not throw when window is undefined (server-side rendering)", () => { const originalWindow = global.window; - const originalDocument = global.document; delete (global as any).window; - delete (global as any).document; - expect(() => setTokenCookie("token")).not.toThrow(); + expect(() => storeLoginToken("token")).not.toThrow(); global.window = originalWindow; - global.document = originalDocument; }); }); @@ -177,5 +172,21 @@ describe("cookieUtils", () => { expect(getCookie("token")).toBe("token-value"); expect(getCookie("other")).toBe("other-value"); }); + + it("should handle values containing '=' characters", () => { + document.cookie = "token=abc=def=ghi; path=/"; + expect(getCookie("token")).toBe("abc=def=ghi"); + }); + + it("should fall back to sessionStorage when cookie is not found", () => { + sessionStorage.setItem("token", "session-stored-jwt"); + expect(getCookie("token")).toBe("session-stored-jwt"); + }); + + it("should prefer cookie over sessionStorage", () => { + document.cookie = "token=cookie-value; path=/"; + sessionStorage.setItem("token", "session-value"); + expect(getCookie("token")).toBe("cookie-value"); + }); }); }); diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 6fe43f30400d..229a49a6a036 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -43,17 +43,27 @@ export function clearTokenCookies() { }); }); + try { + sessionStorage.removeItem("token"); + } catch { + // sessionStorage may be unavailable + } + console.log("After clearing cookies:", document.cookie); } /** - * Sets the token cookie from a login response body. - * This ensures the token is JS-accessible even when a reverse proxy adds HttpOnly to server-set cookies. + * Stores the login token in sessionStorage. + * This ensures the token is available even when a reverse proxy adds HttpOnly + * to server-set cookies, making them invisible to JavaScript. */ -export function setTokenCookie(token: string) { - if (typeof window === "undefined" || typeof document === "undefined") return; - const isSecure = window.location.protocol === "https:"; - document.cookie = `token=${encodeURIComponent(token)}; Path=/; SameSite=Lax${isSecure ? "; Secure" : ""}`; +export function storeLoginToken(token: string) { + if (typeof window === "undefined") return; + try { + sessionStorage.setItem("token", token); + } catch { + // sessionStorage may be unavailable (e.g. private browsing quota exceeded) + } } /** @@ -63,13 +73,23 @@ export function setTokenCookie(token: string) { */ export function getCookie(name: string) { if (typeof document === "undefined") return null; - const cookieValue = document.cookie.split("; ").find((row) => row.startsWith(name + "=")); - if (!cookieValue) return null; - const rawValue = cookieValue.split("=")[1]; - try { - return decodeURIComponent(rawValue); - } catch { - return rawValue; + const row = document.cookie.split("; ").find((r) => r.startsWith(name + "=")); + if (row) { + const raw = row.split("=").slice(1).join("="); + try { + return decodeURIComponent(raw); + } catch { + return raw; + } } -} + // Fallback to sessionStorage — covers the case where a reverse proxy + // added HttpOnly to the server-set cookie, making it invisible to JS. + if (typeof window !== "undefined") { + try { + return sessionStorage.getItem(name); + } catch { + return null; + } + } + return null; } From 523d1c388f934e868de282d39e121f5fe69befc6 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 11:06:43 +0200 Subject: [PATCH 07/28] fix(ui): use shared getCookie in page.tsx and user_dashboard.tsx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace local getCookie functions in page.tsx and user_dashboard.tsx with the shared one from cookieUtils that has the sessionStorage fallback. Without this, the HttpOnly cookie fix was incomplete — page.tsx (the dashboard entry point) could not read the token, causing the redirect loop to persist. Also scope the sessionStorage fallback to the "token" key only, and clear sessionStorage in page.tsx deleteCookie. Co-Authored-By: Claude Opus 4.6 --- ui/litellm-dashboard/src/app/page.tsx | 18 ++++++------------ .../src/components/user_dashboard.tsx | 8 ++------ .../src/utils/cookieUtils.test.ts | 5 +++++ ui/litellm-dashboard/src/utils/cookieUtils.ts | 2 +- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 5f2921203ffb..1aa555be4fe2 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -43,6 +43,7 @@ import ToolPoliciesView from "@/components/ToolPoliciesView"; import SpendLogsTable from "@/components/view_logs"; import ViewUserDashboard from "@/components/view_users"; import { ThemeProvider } from "@/contexts/ThemeContext"; +import { getCookie } from "@/utils/cookieUtils"; import { isJwtExpired } from "@/utils/jwtUtils"; import { buildLoginUrlWithReturn, consumeReturnUrl, normalizeUrlForCompare, storeReturnUrl } from "@/utils/returnUrlUtils"; import { formatUserRole, isAdminRole } from "@/utils/roles"; @@ -52,21 +53,14 @@ import { useSearchParams } from "next/navigation"; import { Suspense, useEffect, useMemo, useRef, useState } from "react"; import { ConfigProvider, theme } from "antd"; -function getCookie(name: string) { - // Safer cookie read + decoding; handles '=' inside values - const match = document.cookie.split("; ").find((row) => row.startsWith(name + "=")); - if (!match) return null; - const value = match.slice(name.length + 1); - try { - return decodeURIComponent(value); - } catch { - return value; - } -} - function deleteCookie(name: string, path = "/") { // Best-effort client-side clear (works for non-HttpOnly cookies without Domain) document.cookie = `${name}=; Max-Age=0; Path=${path}`; + try { + sessionStorage.removeItem(name); + } catch { + // sessionStorage may be unavailable + } } interface ProxySettings { diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index dc8d2d9cdd7f..79b1a7ca6e5a 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -1,5 +1,5 @@ "use client"; -import { clearTokenCookies } from "@/utils/cookieUtils"; +import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; import { Col, Grid } from "@tremor/react"; import { Typography } from "antd"; import { jwtDecode } from "jwt-decode"; @@ -35,11 +35,7 @@ export type UserInfo = { spend: number; }; -function getCookie(name: string) { - console.log("COOKIES", document.cookie); - const cookieValue = document.cookie.split("; ").find((row) => row.startsWith(name + "=")); - return cookieValue ? cookieValue.split("=")[1] : null; -} + interface UserDashboardProps { userID: string | null; diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts index eedf4949242b..d7418db3178c 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts @@ -188,5 +188,10 @@ describe("cookieUtils", () => { sessionStorage.setItem("token", "session-value"); expect(getCookie("token")).toBe("cookie-value"); }); + + it("should not fall back to sessionStorage for non-token keys", () => { + sessionStorage.setItem("other", "other-value"); + expect(getCookie("other")).toBeNull(); + }); }); }); diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 229a49a6a036..4282d645c0e7 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -84,7 +84,7 @@ export function getCookie(name: string) { } // Fallback to sessionStorage — covers the case where a reverse proxy // added HttpOnly to the server-set cookie, making it invisible to JS. - if (typeof window !== "undefined") { + if (name === "token" && typeof window !== "undefined") { try { return sessionStorage.getItem(name); } catch { From acdc6145b38394b244b2abe19b4ae7a2a8936a35 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 11:12:35 +0200 Subject: [PATCH 08/28] fix(ui): scope deleteCookie sessionStorage cleanup to token key only Also document the sessionStorage cross-tab trade-off: per-tab scope means users behind an HttpOnly proxy must log in once per tab, but this is intentional to avoid localStorage XSS exposure. Co-Authored-By: Claude Opus 4.6 --- ui/litellm-dashboard/src/app/page.tsx | 10 ++++++---- ui/litellm-dashboard/src/utils/cookieUtils.ts | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 1aa555be4fe2..93b7b19deae3 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -56,10 +56,12 @@ import { ConfigProvider, theme } from "antd"; function deleteCookie(name: string, path = "/") { // Best-effort client-side clear (works for non-HttpOnly cookies without Domain) document.cookie = `${name}=; Max-Age=0; Path=${path}`; - try { - sessionStorage.removeItem(name); - } catch { - // sessionStorage may be unavailable + if (name === "token") { + try { + sessionStorage.removeItem(name); + } catch { + // sessionStorage may be unavailable + } } } diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 4282d645c0e7..8566f0d1aa72 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -56,6 +56,10 @@ export function clearTokenCookies() { * Stores the login token in sessionStorage. * This ensures the token is available even when a reverse proxy adds HttpOnly * to server-set cookies, making them invisible to JavaScript. + * + * Note: sessionStorage is per-tab, so users behind an HttpOnly proxy must log + * in once per tab. We intentionally avoid localStorage here because it persists + * after browser close and is readable by any injected script (XSS). */ export function storeLoginToken(token: string) { if (typeof window === "undefined") return; From a607754b358831f21e37f553c1c1511abb53dbb0 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 11:33:53 +0200 Subject: [PATCH 09/28] Update ui/litellm-dashboard/src/utils/cookieUtils.ts Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- ui/litellm-dashboard/src/utils/cookieUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 8566f0d1aa72..815a8b615268 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -49,7 +49,7 @@ export function clearTokenCookies() { // sessionStorage may be unavailable } - console.log("After clearing cookies:", document.cookie); +} } /** From 20ad73fda48e9745f939912ec853e58e6a86e334 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 11:40:01 +0200 Subject: [PATCH 10/28] Apply suggestion from @greptile-apps[bot] Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- ui/litellm-dashboard/src/utils/cookieUtils.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 815a8b615268..71f389e2e8ef 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -50,7 +50,6 @@ export function clearTokenCookies() { } } -} /** * Stores the login token in sessionStorage. From 06260df412600e5e2294bcab265048ba122868c9 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 12:00:56 +0200 Subject: [PATCH 11/28] style: remove stray double blank line in user_dashboard.tsx Co-Authored-By: Claude Opus 4.6 --- ui/litellm-dashboard/src/components/user_dashboard.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index 79b1a7ca6e5a..cdfbd7d300d8 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -35,8 +35,6 @@ export type UserInfo = { spend: number; }; - - interface UserDashboardProps { userID: string | null; userRole: string | null; From 35bd69e146a34c5e2d0ca6ed1a610f508fc55b3b Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 12:06:43 +0200 Subject: [PATCH 12/28] fix(ui): guard storeLoginToken against empty/whitespace-only tokens Co-Authored-By: Claude Opus 4.6 --- ui/litellm-dashboard/src/utils/cookieUtils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 71f389e2e8ef..dc80d68aacb1 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -62,6 +62,7 @@ export function clearTokenCookies() { */ export function storeLoginToken(token: string) { if (typeof window === "undefined") return; + if (!token || !token.trim()) return; try { sessionStorage.setItem("token", token); } catch { From 01f884411468ef558a64c0d85411bbb0a8bd036b Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 13 Mar 2026 12:15:39 +0200 Subject: [PATCH 13/28] fix(ui): preserve sessionStorage token across beforeunload clear The existing beforeunload handler calls sessionStorage.clear() to flush cached UI data on page refresh. This also wiped the token stored by storeLoginToken, re-introducing the redirect loop after any page refresh in the HttpOnly proxy scenario. Now the token is saved and restored across the clear. Co-Authored-By: Claude Opus 4.6 --- ui/litellm-dashboard/src/components/user_dashboard.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index cdfbd7d300d8..65f25fd4593a 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -97,7 +97,11 @@ const UserDashboard: React.FC = ({ // They are only cleared on logout useEffect(() => { const handleBeforeUnload = () => { + const token = sessionStorage.getItem("token"); sessionStorage.clear(); + if (token) { + sessionStorage.setItem("token", token); + } }; window.addEventListener("beforeunload", handleBeforeUnload); return () => window.removeEventListener("beforeunload", handleBeforeUnload); From a0937607a469dc088c7b2d19d8a33c5eaab2e554 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Thu, 19 Mar 2026 23:50:02 +0200 Subject: [PATCH 14/28] fix(ui): set JS-accessible cookie at /ui path as HttpOnly workaround MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sessionStorage alone is unreliable. Also set the token via document.cookie at path=/ui — nginx only adds HttpOnly to server-set Set-Cookie headers, so a JS-set cookie is always readable. Co-Authored-By: Claude Opus 4.6 (1M context) --- ui/litellm-dashboard/src/app/page.tsx | 2 ++ ui/litellm-dashboard/src/utils/cookieUtils.ts | 25 ++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 93b7b19deae3..8a748849baaa 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -56,7 +56,9 @@ import { ConfigProvider, theme } from "antd"; function deleteCookie(name: string, path = "/") { // Best-effort client-side clear (works for non-HttpOnly cookies without Domain) document.cookie = `${name}=; Max-Age=0; Path=${path}`; + // Also clear the JS-set cookie at /ui (used to work around HttpOnly proxies) if (name === "token") { + document.cookie = `${name}=; Max-Age=0; Path=/ui`; try { sessionStorage.removeItem(name); } catch { diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index dc80d68aacb1..c212f3b0a3b9 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -52,17 +52,30 @@ export function clearTokenCookies() { } /** - * Stores the login token in sessionStorage. - * This ensures the token is available even when a reverse proxy adds HttpOnly - * to server-set cookies, making them invisible to JavaScript. + * Stores the login token so the UI can read it even when a reverse proxy + * (e.g. nginx-ingress) adds HttpOnly to the server-set cookie. * - * Note: sessionStorage is per-tab, so users behind an HttpOnly proxy must log - * in once per tab. We intentionally avoid localStorage here because it persists - * after browser close and is readable by any injected script (XSS). + * Strategy: + * 1. Set a JS-accessible cookie at path "/ui". Because nginx only modifies + * server-set Set-Cookie headers, a cookie created via document.cookie will + * never carry HttpOnly. Using path "/ui" avoids colliding with the + * server-set HttpOnly cookie at path "/". + * 2. Also store in sessionStorage as a secondary fallback. */ export function storeLoginToken(token: string) { if (typeof window === "undefined") return; if (!token || !token.trim()) return; + + // 1. JS-accessible cookie at /ui — survives same-tab navigations and + // is readable by getCookie() via document.cookie. + try { + const secure = window.location.protocol === "https:" ? "; Secure" : ""; + document.cookie = `token=${encodeURIComponent(token)}; path=/ui; SameSite=Lax${secure}`; + } catch { + // cookie setting may fail in restrictive environments + } + + // 2. sessionStorage backup try { sessionStorage.setItem("token", token); } catch { From bd878e742e51380745b36b9ec2ce1eafe7d7d90f Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Thu, 19 Mar 2026 23:53:35 +0200 Subject: [PATCH 15/28] fix(ui): use dynamic cookie path based on server_root_path Hardcoded path=/ui breaks when LiteLLM is deployed with a custom server_root_path. Now derives the cookie path from serverRootPath so it works at /ui, /myapp/ui, etc. Also reuse clearTokenCookies() in deleteCookie() to avoid duplication. Co-Authored-By: Claude Opus 4.6 (1M context) --- ui/litellm-dashboard/src/app/page.tsx | 10 ++-------- ui/litellm-dashboard/src/utils/cookieUtils.ts | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 8a748849baaa..9946cc539252 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -43,7 +43,7 @@ import ToolPoliciesView from "@/components/ToolPoliciesView"; import SpendLogsTable from "@/components/view_logs"; import ViewUserDashboard from "@/components/view_users"; import { ThemeProvider } from "@/contexts/ThemeContext"; -import { getCookie } from "@/utils/cookieUtils"; +import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; import { isJwtExpired } from "@/utils/jwtUtils"; import { buildLoginUrlWithReturn, consumeReturnUrl, normalizeUrlForCompare, storeReturnUrl } from "@/utils/returnUrlUtils"; import { formatUserRole, isAdminRole } from "@/utils/roles"; @@ -56,14 +56,8 @@ import { ConfigProvider, theme } from "antd"; function deleteCookie(name: string, path = "/") { // Best-effort client-side clear (works for non-HttpOnly cookies without Domain) document.cookie = `${name}=; Max-Age=0; Path=${path}`; - // Also clear the JS-set cookie at /ui (used to work around HttpOnly proxies) if (name === "token") { - document.cookie = `${name}=; Max-Age=0; Path=/ui`; - try { - sessionStorage.removeItem(name); - } catch { - // sessionStorage may be unavailable - } + clearTokenCookies(); } } diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index c212f3b0a3b9..928f640fc2d6 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -1,6 +1,17 @@ /** * Utility functions for managing cookies */ +import { serverRootPath } from "@/components/networking"; + +/** + * Returns the cookie path for the UI. + * Respects server_root_path so the cookie works when LiteLLM is + * deployed behind a subpath (e.g. /myapp/ui instead of /ui). + */ +function getUiCookiePath(): string { + const root = serverRootPath === "/" ? "" : serverRootPath; + return `${root}/ui`; +} /** * Clears the token cookie from both root and /ui paths @@ -16,7 +27,8 @@ export function clearTokenCookies() { // Clear with various combinations of path and SameSite // Include current path in case of custom server root path const currentPath = window.location.pathname; - const paths = ["/", "/ui"]; + const uiCookiePath = getUiCookiePath(); + const paths = ["/", uiCookiePath]; // Add the current path directory if it's different from root and /ui if (currentPath && currentPath !== "/" && !currentPath.startsWith("/ui")) { @@ -70,7 +82,8 @@ export function storeLoginToken(token: string) { // is readable by getCookie() via document.cookie. try { const secure = window.location.protocol === "https:" ? "; Secure" : ""; - document.cookie = `token=${encodeURIComponent(token)}; path=/ui; SameSite=Lax${secure}`; + const cookiePath = getUiCookiePath(); + document.cookie = `token=${encodeURIComponent(token)}; path=${cookiePath}; SameSite=Lax${secure}`; } catch { // cookie setting may fail in restrictive environments } From 40eaab5693df1b695a6c6281309fb661a4673b3d Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 20 Mar 2026 00:07:06 +0200 Subject: [PATCH 16/28] refactor(ui): remove circular dependency in cookieUtils.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Derive the UI cookie path from window.location.pathname instead of importing serverRootPath from networking.tsx. This breaks the cookieUtils → networking → cookieUtils cycle that could cause serverRootPath to be undefined under certain bundler configurations. Co-Authored-By: Claude Opus 4.6 (1M context) --- ui/litellm-dashboard/src/utils/cookieUtils.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 928f640fc2d6..583f41e84a89 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -1,16 +1,18 @@ /** * Utility functions for managing cookies */ -import { serverRootPath } from "@/components/networking"; /** * Returns the cookie path for the UI. - * Respects server_root_path so the cookie works when LiteLLM is - * deployed behind a subpath (e.g. /myapp/ui instead of /ui). + * Derives the path from window.location.pathname so it works when + * LiteLLM is deployed behind a subpath (e.g. /myapp/ui instead of /ui). + * No imports from networking.tsx to avoid circular dependencies. */ function getUiCookiePath(): string { - const root = serverRootPath === "/" ? "" : serverRootPath; - return `${root}/ui`; + if (typeof window === "undefined") return "/ui"; + const idx = window.location.pathname.indexOf("/ui"); + if (idx >= 0) return window.location.pathname.substring(0, idx + 3); + return "/ui"; } /** From 11974cae354237bf473f515ab171117136653745 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Fri, 20 Mar 2026 00:19:48 +0200 Subject: [PATCH 17/28] fix(ui): harden getUiCookiePath regex and add missing tests - Use regex /\/ui(?=\/|$)/ to match "/ui" only as a full path segment, preventing false matches on paths like "/my-ui-tool/login". - Add unit tests for storeLoginToken empty/whitespace guard and cookie-at-/ui-path behavior. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/utils/cookieUtils.test.ts | 19 +++++++++++++++++++ ui/litellm-dashboard/src/utils/cookieUtils.ts | 8 ++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts index d7418db3178c..c7bd27a6a85e 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts @@ -147,6 +147,25 @@ describe("cookieUtils", () => { global.window = originalWindow; }); + + it("should not store empty string token", () => { + storeLoginToken(""); + expect(sessionStorage.getItem("token")).toBeNull(); + }); + + it("should not store whitespace-only token", () => { + storeLoginToken(" "); + expect(sessionStorage.getItem("token")).toBeNull(); + }); + + it("should set a JS-accessible cookie at /ui path", () => { + const cookieSpy = vi.spyOn(document, "cookie", "set"); + storeLoginToken("my-jwt-token"); + expect(cookieSpy).toHaveBeenCalledWith( + expect.stringContaining("path=/ui") + ); + vi.restoreAllMocks(); + }); }); describe("getCookie", () => { diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.ts b/ui/litellm-dashboard/src/utils/cookieUtils.ts index 583f41e84a89..b4493744ad44 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.ts @@ -10,8 +10,12 @@ */ function getUiCookiePath(): string { if (typeof window === "undefined") return "/ui"; - const idx = window.location.pathname.indexOf("/ui"); - if (idx >= 0) return window.location.pathname.substring(0, idx + 3); + // Match "/ui" only as a full path segment (followed by "/" or end of string) + // to avoid false matches like "/my-ui-tool/login" → "/my-ui". + const match = window.location.pathname.match(/\/ui(?=\/|$)/); + if (match && match.index !== undefined) { + return window.location.pathname.substring(0, match.index + 3); + } return "/ui"; } From c16fdceb66a5062f6c00140c4d90e79ae7e31edc Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Mon, 23 Mar 2026 09:38:50 +0200 Subject: [PATCH 18/28] style: fix Black formatting in audit_logs.py Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/proxy/management_helpers/audit_logs.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index b9020222f1fe..7599e11bdef0 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -51,7 +51,11 @@ def _build_audit_log_payload( if request_data.updated_at is not None: updated_at = request_data.updated_at.isoformat() - table_name_str: str = request_data.table_name.value if isinstance(request_data.table_name, LitellmTableNames) else str(request_data.table_name) + table_name_str: str = ( + request_data.table_name.value + if isinstance(request_data.table_name, LitellmTableNames) + else str(request_data.table_name) + ) return StandardAuditLogPayload( id=request_data.id, @@ -89,7 +93,9 @@ async def _dispatch_audit_log_to_callbacks( for callback in litellm.audit_log_callbacks: try: - resolved: Optional[CustomLogger] = callback if isinstance(callback, CustomLogger) else None + resolved: Optional[CustomLogger] = ( + callback if isinstance(callback, CustomLogger) else None + ) if isinstance(callback, str): resolved = _resolve_audit_log_callback(callback) if resolved is None: @@ -138,9 +144,7 @@ async def create_object_audit_log( return _changed_by = ( - litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name + litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name ) await create_audit_log_for_update( From 2905d47bd4013b81fbaa01cf2c7402d51360f1e6 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Mon, 30 Mar 2026 09:24:41 +0300 Subject: [PATCH 19/28] fix CI: formatting, test params, remove token from login JSON Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/integrations/prometheus.py | 46 +-- .../bedrock/chat/converse_transformation.py | 315 +++++++++++++----- litellm/proxy/auth/auth_checks.py | 12 +- .../proxy/common_utils/reset_budget_job.py | 12 +- litellm/proxy/management_endpoints/ui_sso.py | 62 ++-- litellm/proxy/proxy_server.py | 77 ++--- .../proxy/auth/test_user_api_key_auth.py | 8 +- 7 files changed, 344 insertions(+), 188 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 294b9d464f58..d022af689d3a 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -1087,9 +1087,11 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti ), client_ip=standard_logging_payload["metadata"].get("requester_ip_address"), user_agent=standard_logging_payload["metadata"].get("user_agent"), - stream=str(standard_logging_payload.get("stream")) - if litellm.prometheus_emit_stream_label - else None, + stream=( + str(standard_logging_payload.get("stream")) + if litellm.prometheus_emit_stream_label + else None + ), ) if ( @@ -1755,9 +1757,11 @@ async def async_post_call_failure_hook( client_ip=_metadata.get("requester_ip_address"), user_agent=_metadata.get("user_agent"), model_id=model_id, - stream=str(request_data.get("stream")) - if litellm.prometheus_emit_stream_label - else None, + stream=( + str(request_data.get("stream")) + if litellm.prometheus_emit_stream_label + else None + ), ) _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric( @@ -2081,9 +2085,9 @@ def set_llm_deployment_success_metrics( ): try: verbose_logger.debug("setting remaining tokens requests metric") - standard_logging_payload: Optional[ - StandardLoggingPayload - ] = request_kwargs.get("standard_logging_object") + standard_logging_payload: Optional[StandardLoggingPayload] = ( + request_kwargs.get("standard_logging_object") + ) if standard_logging_payload is None: return @@ -2716,9 +2720,7 @@ async def _initialize_api_key_budget_metrics(self): ) return - async def fetch_keys( - page_size: int, page: int - ) -> Tuple[ + async def fetch_keys(page_size: int, page: int) -> Tuple[ List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]], Optional[int], ]: @@ -2789,9 +2791,7 @@ async def _initialize_org_budget_metrics(self): ) return - async def fetch_orgs( - page_size: int, page: int - ) -> Tuple[list, Optional[int]]: + async def fetch_orgs(page_size: int, page: int) -> Tuple[list, Optional[int]]: skip = (page - 1) * page_size orgs = await prisma_client.db.litellm_organizationtable.find_many( skip=skip, @@ -2911,9 +2911,11 @@ async def _set_org_list_budget_metrics(self, orgs: list): org_alias=org.organization_alias or "", spend=org.spend or 0.0, max_budget=budget_table.max_budget if budget_table else None, - budget_reset_at=getattr(budget_table, "budget_reset_at", None) - if budget_table - else None, + budget_reset_at=( + getattr(budget_table, "budget_reset_at", None) + if budget_table + else None + ), ) async def _set_team_budget_metrics_after_api_request( @@ -3395,10 +3397,10 @@ def initialize_budget_metrics_cron_job(scheduler: AsyncIOScheduler): from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES from litellm.integrations.custom_logger import CustomLogger - prometheus_loggers: List[ - CustomLogger - ] = litellm.logging_callback_manager.get_custom_loggers_for_type( - callback_type=PrometheusLogger + prometheus_loggers: List[CustomLogger] = ( + litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=PrometheusLogger + ) ) # we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers)) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 90b501a9cbb0..e01f29948107 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -160,7 +160,8 @@ def _convert_consecutive_user_messages_to_guarded_text( if isinstance(content, list): has_guarded_text = any( - isinstance(item, dict) and item.get("type") == "guarded_text" for item in content + isinstance(item, dict) and item.get("type") == "guarded_text" + for item in content ) if has_guarded_text: continue # Skip this message if it already has guarded_text @@ -321,9 +322,13 @@ def _is_nova_2_model(self, model: str) -> bool: # Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.) # Also check for nova-2/ spec prefix for imported models - return model_without_region.startswith("amazon.nova-2-") or model_without_region.startswith("nova-2/") + return model_without_region.startswith( + "amazon.nova-2-" + ) or model_without_region.startswith("nova-2/") - def _map_web_search_options(self, web_search_options: dict, model: str) -> Optional[BedrockToolBlock]: + def _map_web_search_options( + self, web_search_options: dict, model: str + ) -> Optional[BedrockToolBlock]: """ Map web_search_options to Nova grounding systemTool. @@ -352,7 +357,9 @@ def _map_web_search_options(self, web_search_options: dict, model: str) -> Optio # (unlike Anthropic), so we just enable grounding with no options return BedrockToolBlock(systemTool={"name": "nova_grounding"}) - def _transform_reasoning_effort_to_reasoning_config(self, reasoning_effort: str) -> dict: + def _transform_reasoning_effort_to_reasoning_config( + self, reasoning_effort: str + ) -> dict: """ Transform reasoning_effort parameter to Nova 2 reasoningConfig structure. @@ -397,7 +404,9 @@ def _transform_reasoning_effort_to_reasoning_config(self, reasoning_effort: str) } } - def _handle_reasoning_effort_parameter(self, model: str, reasoning_effort: str, optional_params: dict) -> None: + def _handle_reasoning_effort_parameter( + self, model: str, reasoning_effort: str, optional_params: dict + ) -> None: """ Handle the reasoning_effort parameter based on the model type. @@ -434,7 +443,9 @@ def _handle_reasoning_effort_parameter(self, model: str, reasoning_effort: str, optional_params["reasoning_effort"] = reasoning_effort elif self._is_nova_2_model(model): # Nova 2 models: transform to reasoningConfig - reasoning_config = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort) + reasoning_config = self._transform_reasoning_effort_to_reasoning_config( + reasoning_effort + ) optional_params.update(reasoning_config) else: # Anthropic and other models: convert to thinking parameter @@ -478,7 +489,9 @@ def get_supported_openai_params(self, model: str) -> List[str]: "parallel_tool_calls", ] - if "arn" in model: # we can't infer the model from the arn, so just add all params + if ( + "arn" in model + ): # we can't infer the model from the arn, so just add all params supported_params.append("tools") supported_params.append("tool_choice") supported_params.append("thinking") @@ -500,7 +513,9 @@ def get_supported_openai_params(self, model: str) -> List[str]: or base_model.startswith("meta.llama3-3") or base_model.startswith("meta.llama4") or base_model.startswith("amazon.nova") - or supports_function_calling(model=model, custom_llm_provider=self.custom_llm_provider) + or supports_function_calling( + model=model, custom_llm_provider=self.custom_llm_provider + ) ): supported_params.append("tools") @@ -510,7 +525,9 @@ def get_supported_openai_params(self, model: str) -> List[str]: if litellm.utils.supports_tool_choice( model=model, custom_llm_provider=self.custom_llm_provider - ) or litellm.utils.supports_tool_choice(model=base_model, custom_llm_provider=self.custom_llm_provider): + ) or litellm.utils.supports_tool_choice( + model=base_model, custom_llm_provider=self.custom_llm_provider + ): # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html supported_params.append("tool_choice") @@ -529,7 +546,9 @@ def get_supported_openai_params(self, model: str) -> List[str]: model=model, custom_llm_provider=self.custom_llm_provider, ) - or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) + or supports_reasoning( + model=base_model, custom_llm_provider=self.custom_llm_provider + ) ): supported_params.append("thinking") supported_params.append("reasoning_effort") @@ -554,7 +573,9 @@ def map_tool_choice_values( return ToolChoiceValuesBlock(auto={}) elif isinstance(tool_choice, dict): # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html - specific_tool = SpecificToolChoiceBlock(name=tool_choice.get("function", {}).get("name", "")) + specific_tool = SpecificToolChoiceBlock( + name=tool_choice.get("function", {}).get("name", "") + ) return ToolChoiceValuesBlock(tool=specific_tool) else: raise litellm.utils.UnsupportedParamsError( @@ -574,9 +595,15 @@ def get_supported_video_types(self) -> List[str]: return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"] def get_all_supported_content_types(self) -> List[str]: - return self.get_supported_image_types() + self.get_supported_document_types() + self.get_supported_video_types() + return ( + self.get_supported_image_types() + + self.get_supported_document_types() + + self.get_supported_video_types() + ) - def is_computer_use_tool_used(self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str) -> bool: + def is_computer_use_tool_used( + self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str + ) -> bool: """Check if computer use tools are being used in the request.""" if tools is None: return False @@ -589,7 +616,9 @@ def is_computer_use_tool_used(self, tools: Optional[List[OpenAIChatCompletionToo return True return False - def _transform_computer_use_tools(self, computer_use_tools: List[OpenAIChatCompletionToolParam]) -> List[dict]: + def _transform_computer_use_tools( + self, computer_use_tools: List[OpenAIChatCompletionToolParam] + ) -> List[dict]: """Transform computer use tools to Bedrock format.""" transformed_tools: List[dict] = [] @@ -631,7 +660,9 @@ def _transform_computer_use_tools(self, computer_use_tools: List[OpenAIChatCompl def _separate_computer_use_tools( self, tools: List[OpenAIChatCompletionToolParam], model: str - ) -> Tuple[List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]]: + ) -> Tuple[ + List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam] + ]: """ Separate computer use tools from regular function tools. @@ -740,18 +771,25 @@ def _add_additional_properties_to_schema(schema: dict) -> dict: # Recurse into nested schemas if "properties" in result and isinstance(result["properties"], dict): result["properties"] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result["properties"].items() + k: AmazonConverseConfig._add_additional_properties_to_schema(v) + for k, v in result["properties"].items() } if "items" in result and isinstance(result["items"], dict): - result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"]) + result["items"] = AmazonConverseConfig._add_additional_properties_to_schema( + result["items"] + ) for defs_key in ("$defs", "definitions"): if defs_key in result and isinstance(result[defs_key], dict): result[defs_key] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items() + k: AmazonConverseConfig._add_additional_properties_to_schema(v) + for k, v in result[defs_key].items() } for key in ("anyOf", "allOf", "oneOf"): if key in result and isinstance(result[key], list): - result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]] + result[key] = [ + AmazonConverseConfig._add_additional_properties_to_schema(item) + for item in result[key] + ] return result @@ -781,7 +819,9 @@ def _create_output_config_for_response_format( } """ if json_schema is not None: - json_schema = AmazonConverseConfig._add_additional_properties_to_schema(json_schema) + json_schema = AmazonConverseConfig._add_additional_properties_to_schema( + json_schema + ) schema_str = json.dumps(json_schema) if json_schema is not None else "{}" json_schema_def: JsonSchemaDefinition = {"schema": schema_str} if name is not None: @@ -803,9 +843,14 @@ def _apply_tool_call_transformation( non_default_params: dict, optional_params: dict, ): - optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=tools) + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, tools=tools + ) - if "meta.llama3-3-70b-instruct-v1:0" in model and non_default_params.get("stream", False) is True: + if ( + "meta.llama3-3-70b-instruct-v1:0" in model + and non_default_params.get("stream", False) is True + ): optional_params["fake_stream"] = True def map_openai_params( @@ -944,7 +989,10 @@ def _translate_response_format_param( if "type" in value and value["type"] == "text": return optional_params - if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None: + if ( + self._supports_native_structured_outputs(model, self.custom_llm_provider) + and json_schema is not None + ): # Use Bedrock's native structured outputs API (outputConfig.textFormat) # No synthetic tool injection, no fake_stream needed. # Requires an explicit schema — json_object with no schema falls through @@ -962,10 +1010,14 @@ def _translate_response_format_param( json_schema=json_schema, description=description, ) - optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) + optional_params = self._add_tools_to_optional_params( + optional_params=optional_params, tools=[_tool] + ) if ( - litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) + litellm.utils.supports_tool_choice( + model=model, custom_llm_provider=self.custom_llm_provider + ) and not is_thinking_enabled ): optional_params["tool_choice"] = ToolChoiceValuesBlock( @@ -977,7 +1029,9 @@ def _translate_response_format_param( optional_params["json_mode"] = True return optional_params - def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict): + def update_optional_params_with_thinking_tokens( + self, non_default_params: dict, optional_params: dict + ): """ Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget. @@ -995,9 +1049,13 @@ def update_optional_params_with_thinking_tokens(self, non_default_params: dict, is_thinking_enabled = self.is_thinking_enabled(optional_params) is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params) if is_thinking_enabled and not is_max_tokens_in_request: - thinking_token_budget = cast(dict, optional_params["thinking"]).get("budget_tokens", None) + thinking_token_budget = cast(dict, optional_params["thinking"]).get( + "budget_tokens", None + ) if thinking_token_budget is not None: - optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS + optional_params["maxTokens"] = ( + thinking_token_budget + DEFAULT_MAX_TOKENS + ) @overload def _get_cache_point_block( @@ -1063,15 +1121,23 @@ def _transform_system_message( if message["role"] == "system": system_prompt_indices.append(idx) if isinstance(message["content"], str) and message["content"]: - system_content_blocks.append(SystemContentBlock(text=message["content"])) - cache_block = self._get_cache_point_block(message, block_type="system", model=model) + system_content_blocks.append( + SystemContentBlock(text=message["content"]) + ) + cache_block = self._get_cache_point_block( + message, block_type="system", model=model + ) if cache_block: system_content_blocks.append(cache_block) elif isinstance(message["content"], list): for m in message["content"]: if m.get("type") == "text" and m.get("text"): - system_content_blocks.append(SystemContentBlock(text=m["text"])) - cache_block = self._get_cache_point_block(m, block_type="system", model=model) + system_content_blocks.append( + SystemContentBlock(text=m["text"]) + ) + cache_block = self._get_cache_point_block( + m, block_type="system", model=model + ) if cache_block: system_content_blocks.append(cache_block) if len(system_prompt_indices) > 0: @@ -1109,10 +1175,16 @@ def _prepare_request_params( # Exceptions should not be stored in optional_params (this is a defensive fix) cleaned_params = filter_exceptions_from_params(optional_params) inference_params = safe_deep_copy(cleaned_params) - supported_converse_params = list(AmazonConverseConfig.__annotations__.keys()) + ["top_k"] + supported_converse_params = list( + AmazonConverseConfig.__annotations__.keys() + ) + ["top_k"] supported_tool_call_params = ["tools", "tool_choice"] supported_config_params = list(self.get_config_blocks().keys()) - total_supported_params = supported_converse_params + supported_tool_call_params + supported_config_params + total_supported_params = ( + supported_converse_params + + supported_tool_call_params + + supported_config_params + ) inference_params.pop("json_mode", None) # used for handling json_schema # Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and # will reject `output_config` if it leaks through pass-through routes. @@ -1123,15 +1195,25 @@ def _prepare_request_params( if request_metadata is not None: self._validate_request_metadata(request_metadata) - output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None) - inference_params.pop("output_config", None) # Bedrock Converse doesn't support it + output_config: Optional[OutputConfigBlock] = inference_params.pop( + "outputConfig", None + ) + inference_params.pop( + "output_config", None + ) # Bedrock Converse doesn't support it # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' - additional_request_params = {k: v for k, v in inference_params.items() if k not in total_supported_params} - inference_params = {k: v for k, v in inference_params.items() if k in total_supported_params} + additional_request_params = { + k: v for k, v in inference_params.items() if k not in total_supported_params + } + inference_params = { + k: v for k, v in inference_params.items() if k in total_supported_params + } # Handle parallel_tool_calls configuration - parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) + parallel_tool_use_config = additional_request_params.pop( + "_parallel_tool_use_config", None + ) if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): for key, value in parallel_tool_use_config.items(): if ( @@ -1146,7 +1228,9 @@ def _prepare_request_params( additional_request_params.pop("parallel_tool_calls", None) # Only set the topK value in for models that support it - additional_request_params.update(self._handle_top_k_value(model, inference_params)) + additional_request_params.update( + self._handle_top_k_value(model, inference_params) + ) # Filter out internal/MCP-related parameters that shouldn't be sent to the API # These are LiteLLM internal parameters, not API parameters @@ -1155,7 +1239,9 @@ def _prepare_request_params( # Filter out non-serializable objects (exceptions, callables, logging objects, etc.) # from additional_request_params to prevent JSON serialization errors # This filters: Exception objects, callable objects (functions), Logging objects, etc. - additional_request_params = filter_exceptions_from_params(additional_request_params) + additional_request_params = filter_exceptions_from_params( + additional_request_params + ) return ( inference_params, @@ -1202,7 +1288,9 @@ def _process_tools_and_beta( # Only separate tools if computer use tools are actually present if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): # Separate computer use tools from regular function tools - computer_use_tools, regular_tools = self._separate_computer_use_tools(filtered_tools, model) + computer_use_tools, regular_tools = self._separate_computer_use_tools( + filtered_tools, model + ) # Process regular function tools using existing logic bedrock_tools = _bedrock_tools_pt(regular_tools) @@ -1263,7 +1351,9 @@ def _process_tools_and_beta( anthropic_beta_list.append(computer_use_header) # Transform computer use tools to proper Bedrock format - transformed_computer_tools = self._transform_computer_use_tools(computer_use_tools) + transformed_computer_tools = self._transform_computer_use_tools( + computer_use_tools + ) additional_request_params["tools"] = transformed_computer_tools else: # No computer use tools, process all tools as regular tools @@ -1292,9 +1382,15 @@ def _transform_request_helper( """ Bedrock doesn't support tool calling without `tools=` param specified. """ - if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages): + if ( + "tools" not in optional_params + and messages is not None + and has_tool_call_blocks(messages) + ): if litellm.modify_params: - optional_params["tools"] = add_dummy_tool(custom_llm_provider="bedrock_converse") + optional_params["tools"] = add_dummy_tool( + custom_llm_provider="bedrock_converse" + ) else: raise litellm.UnsupportedParamsError( message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.", @@ -1348,7 +1444,9 @@ def _transform_request_helper( bedrock_tool_config: Optional[ToolConfigBlock] = None if len(bedrock_tools) > 0: - tool_choice_values: ToolChoiceValuesBlock = inference_params.pop("tool_choice", None) + tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( + "tool_choice", None + ) bedrock_tool_config = ToolConfigBlock( tools=bedrock_tools, ) @@ -1358,7 +1456,9 @@ def _transform_request_helper( data: CommonRequestObject = { "additionalModelRequestFields": additional_request_params, "system": system_content_blocks, - "inferenceConfig": self._transform_inference_params(inference_params=inference_params), + "inferenceConfig": self._transform_inference_params( + inference_params=inference_params + ), } # Handle all config blocks @@ -1388,10 +1488,14 @@ async def _async_transform_request( litellm_params: dict, headers: Optional[dict] = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message(messages, model=model) + messages, system_content_blocks = self._transform_system_message( + messages, model=model + ) # Convert last user message to guarded_text if guardrailConfig is present - messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + messages = self._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) ## TRANSFORMATION ## _data: CommonRequestObject = self._transform_request_helper( @@ -1402,11 +1506,13 @@ async def _async_transform_request( headers=headers, ) - bedrock_messages = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model=model, - llm_provider="bedrock_converse", - user_continue_message=litellm_params.pop("user_continue_message", None), + bedrock_messages = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model=model, + llm_provider="bedrock_converse", + user_continue_message=litellm_params.pop("user_continue_message", None), + ) ) data: RequestObject = {"messages": bedrock_messages, **_data} @@ -1440,10 +1546,14 @@ def _transform_request( litellm_params: dict, headers: Optional[dict] = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message(messages, model=model) + messages, system_content_blocks = self._transform_system_message( + messages, model=model + ) # Convert last user message to guarded_text if guardrailConfig is present - messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) + messages = self._convert_consecutive_user_messages_to_guarded_text( + messages, optional_params + ) _data: CommonRequestObject = self._transform_request_helper( model=model, @@ -1492,7 +1602,9 @@ def transform_response( encoding=encoding, ) - def _transform_reasoning_content(self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]) -> str: + def _transform_reasoning_content( + self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock] + ) -> str: """ Extract the reasoning text from the reasoning content blocks @@ -1508,7 +1620,9 @@ def _transform_thinking_blocks( self, thinking_blocks: List[BedrockConverseReasoningContentBlock] ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]: """Return a consistent format for thinking blocks between Anthropic and Bedrock.""" - thinking_blocks_list: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] = [] + thinking_blocks_list: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = [] for block in thinking_blocks: if "reasoningText" in block: _thinking_block = ChatCompletionThinkingBlock(type="thinking") @@ -1544,11 +1658,21 @@ def _transform_usage( cache_creation_input_tokens = usage["cacheWriteInputTokens"] input_tokens += cache_creation_input_tokens - prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=cache_read_input_tokens) - reasoning_tokens = token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0 + prompt_tokens_details = PromptTokensDetailsWrapper( + cached_tokens=cache_read_input_tokens + ) + reasoning_tokens = ( + token_counter(text=reasoning_content, count_response_tokens=True) + if reasoning_content + else 0 + ) completion_tokens_details = CompletionTokensDetailsWrapper( reasoning_tokens=reasoning_tokens, - text_tokens=(output_tokens - reasoning_tokens if reasoning_tokens > 0 else output_tokens), + text_tokens=( + output_tokens - reasoning_tokens + if reasoning_tokens > 0 + else output_tokens + ), ) openai_usage = Usage( prompt_tokens=input_tokens, @@ -1563,7 +1687,9 @@ def _transform_usage( def get_tool_call_names( self, - tools: Optional[Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]] = None, + tools: Optional[ + Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]] + ] = None, ) -> List[str]: if tools is None: return [] @@ -1602,8 +1728,13 @@ def apply_tool_call_transformation_if_needed( try: tool_call_names = self.get_tool_call_names(tools) json_content = json.loads(message.content) - if json_content.get("type") == "function" and json_content.get("name") in tool_call_names: - tool_calls = [ChatCompletionMessageToolCall(function=Function(**json_content))] + if ( + json_content.get("type") == "function" + and json_content.get("name") in tool_call_names + ): + tool_calls = [ + ChatCompletionMessageToolCall(function=Function(**json_content)) + ] message.tool_calls = tool_calls message.content = None @@ -1613,9 +1744,7 @@ def apply_tool_call_transformation_if_needed( return message, returned_finish_reason - def _translate_message_content( - self, content_blocks: List[ContentBlock] - ) -> Tuple[ + def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[ str, List[ChatCompletionToolCallChunk], Optional[List[BedrockConverseReasoningContentBlock]], @@ -1632,7 +1761,9 @@ def _translate_message_content( """ content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( + None + ) citationsContentBlocks: Optional[List[CitationsContentBlock]] = None for idx, content in enumerate(content_blocks): """ @@ -1649,7 +1780,9 @@ def _translate_message_content( if "toolUse" in content: ## check tool name was formatted by litellm _response_tool_name = content["toolUse"]["name"] - response_tool_name = get_bedrock_tool_name(response_tool_name=_response_tool_name) + response_tool_name = get_bedrock_tool_name( + response_tool_name=_response_tool_name + ) _function_chunk = ChatCompletionToolCallFunctionChunk( name=response_tool_name, arguments=json.dumps(content["toolUse"]["input"]), @@ -1700,7 +1833,11 @@ def _unwrap_bedrock_properties(json_str: str) -> str: """ try: response_data = json.loads(json_str) - if isinstance(response_data, dict) and "properties" in response_data and len(response_data) == 1: + if ( + isinstance(response_data, dict) + and "properties" in response_data + and len(response_data) == 1 + ): response_data = response_data["properties"] return json.dumps(response_data) except json.JSONDecodeError: @@ -1724,7 +1861,11 @@ def _filter_json_mode_tools( if not json_mode or not tools: return tools if tools else None - json_tool_indices = [i for i, t in enumerate(tools) if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME] + json_tool_indices = [ + i + for i, t in enumerate(tools) + if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME + ] if not json_tool_indices: # No json_tool_call found, return tools unchanged @@ -1732,10 +1873,14 @@ def _filter_json_mode_tools( if len(json_tool_indices) == len(tools): # All tools are json_tool_call — convert first one to content - verbose_logger.debug("Processing JSON tool call response for response_format") + verbose_logger.debug( + "Processing JSON tool call response for response_format" + ) json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments") if json_mode_content_str is not None: - json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_content_str) + json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties( + json_mode_content_str + ) chat_completion_message["content"] = json_mode_content_str return None @@ -1745,9 +1890,13 @@ def _filter_json_mode_tools( first_idx = json_tool_indices[0] json_mode_args = tools[first_idx]["function"].get("arguments") if json_mode_args is not None: - json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_args) + json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties( + json_mode_args + ) existing = chat_completion_message.get("content") or "" - chat_completion_message["content"] = existing + json_mode_args if existing else json_mode_args + chat_completion_message["content"] = ( + existing + json_mode_args if existing else json_mode_args + ) real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices] return real_tools if real_tools else None @@ -1825,7 +1974,9 @@ def _transform_response( # noqa: PLR0915 chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( + None + ) citationsContentBlocks: Optional[List[CitationsContentBlock]] = None if message is not None: @@ -1844,11 +1995,17 @@ def _transform_response( # noqa: PLR0915 provider_specific_fields["citationsContent"] = citationsContentBlocks if provider_specific_fields: - chat_completion_message["provider_specific_fields"] = provider_specific_fields + chat_completion_message["provider_specific_fields"] = ( + provider_specific_fields + ) if reasoningContentBlocks is not None: - chat_completion_message["reasoning_content"] = self._transform_reasoning_content(reasoningContentBlocks) - chat_completion_message["thinking_blocks"] = self._transform_thinking_blocks(reasoningContentBlocks) + chat_completion_message["reasoning_content"] = ( + self._transform_reasoning_content(reasoningContentBlocks) + ) + chat_completion_message["thinking_blocks"] = ( + self._transform_thinking_blocks(reasoningContentBlocks) + ) chat_completion_message["content"] = content_str filtered_tools = self._filter_json_mode_tools( json_mode=json_mode, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 0efc3dab638f..ed600b26e832 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -8,6 +8,7 @@ 2. If user is in budget 3. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget """ + import asyncio import re import time @@ -414,9 +415,9 @@ async def common_checks( # noqa: PLR0915 model=_model, team_object=team_object, llm_router=llm_router, - team_model_aliases=valid_token.team_model_aliases - if valid_token - else None, + team_model_aliases=( + valid_token.team_model_aliases if valid_token else None + ), ): raise ProxyException( message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", @@ -3078,10 +3079,7 @@ async def _team_max_budget_check( BudgetExceededError if the team is over it's max budget. Triggers a budget alert if the team is over it's max budget. """ - if ( - team_object is not None - and team_object.max_budget is not None - ): + if team_object is not None and team_object.max_budget is not None: from litellm.proxy.proxy_server import get_current_spend # Read spend from cross-pod counter (Redis-first) or cached object (fallback) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 5db919659eda..bcfaed243981 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -66,10 +66,8 @@ async def reset_budget_for_litellm_team_members( try: from litellm.proxy.proxy_server import spend_counter_cache - memberships = ( - await self.prisma_client.db.litellm_teammembership.find_many( - where={"budget_id": {"in": budget_ids}} - ) + memberships = await self.prisma_client.db.litellm_teammembership.find_many( + where={"budget_id": {"in": budget_ids}} ) for m in memberships: counter_key = f"spend:team_member:{m.user_id}:{m.team_id}" @@ -574,7 +572,11 @@ async def _reset_budget_common( counter_key = None if item_type == "key" and hasattr(item, "token") and item.token is not None: counter_key = f"spend:key:{item.token}" - elif item_type == "team" and hasattr(item, "team_id") and item.team_id is not None: + elif ( + item_type == "team" + and hasattr(item, "team_id") + and item.team_id is not None + ): counter_key = f"spend:team:{item.team_id}" if counter_key is not None: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index a5a62813b9fd..02d7b9baf4a4 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -722,9 +722,9 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: import ast try: - generic_user_role_mappings_data: Dict[ - LitellmUserRoles, List[str] - ] = ast.literal_eval(generic_role_mappings) + generic_user_role_mappings_data: Dict[LitellmUserRoles, List[str]] = ( + ast.literal_eval(generic_role_mappings) + ) if isinstance(generic_user_role_mappings_data, dict): from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings @@ -827,7 +827,9 @@ async def get_generic_sso_response( ], # sso specific jwt handler - used for restricted sso group access control generic_client_id: str, redirect_url: str, -) -> Tuple[Union[OpenID, dict], Optional[dict], Optional[dict]]: # (result, received_response, access_token_payload) +) -> Tuple[ + Union[OpenID, dict], Optional[dict], Optional[dict] +]: # (result, received_response, access_token_payload) # make generic sso provider from fastapi_sso.sso.base import DiscoveryDocument from fastapi_sso.sso.generic import create_provider @@ -879,9 +881,9 @@ def response_convertor(response, client): verbose_proxy_logger.debug("calling generic_sso.verify_and_process") additional_generic_sso_headers_dict = _parse_generic_sso_headers() - code_verifier: Optional[ - str - ] = None # assigned inside try; initialized for type tracking + code_verifier: Optional[str] = ( + None # assigned inside try; initialized for type tracking + ) access_token_payload: Optional[dict] = None # decoded JWT access token claims try: @@ -1231,9 +1233,11 @@ async def _sync_user_role_from_jwt_role_map( user_info.user_role = mapped_role.value await user_api_key_cache.async_set_cache( key=user_info.user_id, - value=user_info.model_dump() - if hasattr(user_info, "model_dump") - else dict(user_info), + value=( + user_info.model_dump() + if hasattr(user_info, "model_dump") + else dict(user_info) + ), ) @@ -1259,9 +1263,9 @@ def apply_user_info_values_to_sso_user_defined_values( else: # SSO didn't provide a valid role, fall back to DB role or default if user_info is None or user_info.user_role is None: - user_defined_values[ - "user_role" - ] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value + user_defined_values["user_role"] = ( + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value + ) verbose_proxy_logger.debug( "No SSO or DB role found, using default: INTERNAL_USER_VIEW_ONLY" ) @@ -1369,12 +1373,14 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: ) elif generic_client_id is not None: - result, received_response, access_token_payload = await get_generic_sso_response( - request=request, - jwt_handler=jwt_handler, - generic_client_id=generic_client_id, - redirect_url=redirect_url, - sso_jwt_handler=sso_jwt_handler, + result, received_response, access_token_payload = ( + await get_generic_sso_response( + request=request, + jwt_handler=jwt_handler, + generic_client_id=generic_client_id, + redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, + ) ) if result is None: @@ -1697,9 +1703,9 @@ async def insert_sso_user( if user_defined_values.get("max_budget") is None: user_defined_values["max_budget"] = litellm.max_internal_user_budget if user_defined_values.get("budget_duration") is None: - user_defined_values[ - "budget_duration" - ] = litellm.internal_user_budget_duration + user_defined_values["budget_duration"] = ( + litellm.internal_user_budget_duration + ) if user_defined_values["user_role"] is None: user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -3342,9 +3348,9 @@ async def get_microsoft_callback_response( # if user is trying to get the raw sso response for debugging, return the raw sso response if return_raw_sso_response: - original_msft_result[ - MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY - ] = user_team_ids + original_msft_result[MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY] = ( + user_team_ids + ) original_msft_result["app_roles"] = app_roles return original_msft_result or {} @@ -3463,9 +3469,9 @@ async def get_user_groups_from_graph_api( # Fetch user membership from Microsoft Graph API all_group_ids = [] - next_link: Optional[ - str - ] = MicrosoftSSOHandler.graph_api_user_groups_endpoint + next_link: Optional[str] = ( + MicrosoftSSOHandler.graph_api_user_groups_endpoint + ) auth_headers = {"Authorization": f"Bearer {access_token}"} page_count = 0 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ab73d44accaa..317b8f761792 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -639,9 +639,9 @@ def generate_feedback_box(): server_root_path = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional[ - "EnterpriseLicenseData" -] = _license_check.airgapped_license_data +premium_user_data: Optional["EnterpriseLicenseData"] = ( + _license_check.airgapped_license_data +) global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -1524,9 +1524,9 @@ async def root_redirect(): config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional[ - "ClientSession" -] = None # Global shared session for connection reuse +shared_aiohttp_session: Optional["ClientSession"] = ( + None # Global shared session for connection reuse +) user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1537,13 +1537,13 @@ async def root_redirect(): dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[ - RedisCache -] = None # redis cache used for tracking spend, tpm/rpm limits +redis_usage_cache: Optional[RedisCache] = ( + None # redis cache used for tracking spend, tpm/rpm limits +) polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False -native_background_mode: List[ - str -] = [] # Models that should use native provider background mode instead of polling +native_background_mode: List[str] = ( + [] +) # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None user_custom_key_generate = None @@ -1714,9 +1714,7 @@ async def get_current_spend(counter_key: str, fallback_spend: float) -> float: # 1. Try Redis first (cross-pod authoritative) if spend_counter_cache.redis_cache is not None: try: - val = await spend_counter_cache.redis_cache.async_get_cache( - key=counter_key - ) + val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key) if val is not None: return float(val) except Exception as e: @@ -1820,9 +1818,7 @@ async def _init_and_increment_spend_counter( key=counter_key, value=base_spend ) - await spend_counter_cache.async_increment_cache( - key=counter_key, value=increment - ) + await spend_counter_cache.async_increment_cache(key=counter_key, value=increment) async def update_cache( # noqa: PLR0915 @@ -2031,9 +2027,9 @@ async def _update_team_cache(): _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[ - LiteLLM_TeamTable - ] = await user_api_key_cache.async_get_cache(key=_id) + existing_spend_obj: Optional[LiteLLM_TeamTable] = ( + await user_api_key_cache.async_get_cache(key=_id) + ) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -2154,11 +2150,9 @@ def run_ollama_serve(): with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - verbose_proxy_logger.debug( - f""" + verbose_proxy_logger.debug(f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` - """ - ) + """) def _get_process_rss_mb() -> Optional[float]: @@ -5206,10 +5200,10 @@ async def _init_guardrails_in_db(self, prisma_client: PrismaClient): ) try: - guardrails_in_db: List[ - Guardrail - ] = await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client + guardrails_in_db: List[Guardrail] = ( + await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client + ) ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -5591,9 +5585,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ[ - "AZURE_API_VERSION" - ] = api_version # set this for azure - litellm can read this from the env + os.environ["AZURE_API_VERSION"] = ( + api_version # set this for azure - litellm can read this from the env + ) if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -5930,9 +5924,9 @@ def _validate_redis_transaction_buffer_config( """ from litellm.secret_managers.main import str_to_bool - _use_redis_transaction_buffer: Optional[ - Union[bool, str] - ] = general_settings.get("use_redis_transaction_buffer", False) + _use_redis_transaction_buffer: Optional[Union[bool, str]] = ( + general_settings.get("use_redis_transaction_buffer", False) + ) if isinstance(_use_redis_transaction_buffer, str): _use_redis_transaction_buffer = str_to_bool(_use_redis_transaction_buffer) @@ -11318,14 +11312,11 @@ async def login_v2(request: Request): # noqa: PLR0915 litellm_dashboard_ui += "/ui/" litellm_dashboard_ui += "?login=success" - # Token is included in the response body so the UI can set a JS-accessible - # cookie even when a reverse proxy (e.g. nginx-ingress) adds HttpOnly to the - # server-set cookie, which would otherwise cause an infinite login redirect. json_response = JSONResponse( - content={"redirect_url": litellm_dashboard_ui, "token": jwt_token}, + content={"redirect_url": litellm_dashboard_ui}, status_code=status.HTTP_200_OK, ) - json_response.set_cookie(key="token", value=jwt_token) + json_response.set_cookie(key="token", value=jwt_token, httponly=True) return json_response except Exception as e: verbose_proxy_logger.exception( @@ -12530,9 +12521,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[ - idx - ].field_description = sub_field_info.description + nested_fields[idx].field_description = ( + sub_field_info.description + ) idx += 1 _stored_in_db = None diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index ca70b4bfa8ea..74c7f9bca5f3 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -28,7 +28,7 @@ def test_get_api_key(): assert get_api_key( custom_litellm_key_header=None, api_key=bearer_token, - AZURE_AI_API_KEY_header=None, + azure_api_key_header=None, anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -59,7 +59,7 @@ def test_get_api_key_with_custom_litellm_key_header( assert get_api_key( custom_litellm_key_header=custom_litellm_key_header, api_key=None, - AZURE_AI_API_KEY_header=None, + azure_api_key_header=None, anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -371,7 +371,7 @@ async def test_proxy_admin_expired_key_from_cache(): await _user_api_key_auth_builder( request=request, api_key=f"Bearer {api_key}", # Add Bearer prefix - AZURE_AI_API_KEY_header="", + azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -842,7 +842,7 @@ async def test_user_api_key_auth_builder_no_blocking_calls(): await _user_api_key_auth_builder( request=request, api_key=f"Bearer {api_key}", - AZURE_AI_API_KEY_header="", + azure_api_key_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, From b2c3334c888a9dbb60fd94cce12f43e8f3aa7e82 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Mon, 30 Mar 2026 09:35:46 +0300 Subject: [PATCH 20/28] fix: reformat with Black 23.x to match CI Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/integrations/prometheus.py | 18 +++--- .../bedrock/chat/converse_transformation.py | 34 +++++----- .../chat/guardrail_translation/handler.py | 6 +- litellm/proxy/management_endpoints/ui_sso.py | 54 ++++++++-------- litellm/proxy/proxy_server.py | 62 ++++++++++--------- 5 files changed, 91 insertions(+), 83 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index d022af689d3a..5d1ff8329301 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2085,9 +2085,9 @@ def set_llm_deployment_success_metrics( ): try: verbose_logger.debug("setting remaining tokens requests metric") - standard_logging_payload: Optional[StandardLoggingPayload] = ( - request_kwargs.get("standard_logging_object") - ) + standard_logging_payload: Optional[ + StandardLoggingPayload + ] = request_kwargs.get("standard_logging_object") if standard_logging_payload is None: return @@ -2720,7 +2720,9 @@ async def _initialize_api_key_budget_metrics(self): ) return - async def fetch_keys(page_size: int, page: int) -> Tuple[ + async def fetch_keys( + page_size: int, page: int + ) -> Tuple[ List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]], Optional[int], ]: @@ -3397,10 +3399,10 @@ def initialize_budget_metrics_cron_job(scheduler: AsyncIOScheduler): from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES from litellm.integrations.custom_logger import CustomLogger - prometheus_loggers: List[CustomLogger] = ( - litellm.logging_callback_manager.get_custom_loggers_for_type( - callback_type=PrometheusLogger - ) + prometheus_loggers: List[ + CustomLogger + ] = litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=PrometheusLogger ) # we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers)) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index e01f29948107..5cfb00d69b61 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1744,7 +1744,9 @@ def apply_tool_call_transformation_if_needed( return message, returned_finish_reason - def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[ + def _translate_message_content( + self, content_blocks: List[ContentBlock] + ) -> Tuple[ str, List[ChatCompletionToolCallChunk], Optional[List[BedrockConverseReasoningContentBlock]], @@ -1761,9 +1763,9 @@ def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tupl """ content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( - None - ) + reasoningContentBlocks: Optional[ + List[BedrockConverseReasoningContentBlock] + ] = None citationsContentBlocks: Optional[List[CitationsContentBlock]] = None for idx, content in enumerate(content_blocks): """ @@ -1974,9 +1976,9 @@ def _transform_response( # noqa: PLR0915 chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( - None - ) + reasoningContentBlocks: Optional[ + List[BedrockConverseReasoningContentBlock] + ] = None citationsContentBlocks: Optional[List[CitationsContentBlock]] = None if message is not None: @@ -1995,17 +1997,17 @@ def _transform_response( # noqa: PLR0915 provider_specific_fields["citationsContent"] = citationsContentBlocks if provider_specific_fields: - chat_completion_message["provider_specific_fields"] = ( - provider_specific_fields - ) + chat_completion_message[ + "provider_specific_fields" + ] = provider_specific_fields if reasoningContentBlocks is not None: - chat_completion_message["reasoning_content"] = ( - self._transform_reasoning_content(reasoningContentBlocks) - ) - chat_completion_message["thinking_blocks"] = ( - self._transform_thinking_blocks(reasoningContentBlocks) - ) + chat_completion_message[ + "reasoning_content" + ] = self._transform_reasoning_content(reasoningContentBlocks) + chat_completion_message[ + "thinking_blocks" + ] = self._transform_thinking_blocks(reasoningContentBlocks) chat_completion_message["content"] = content_str filtered_tools = self._filter_json_mode_tools( json_mode=json_mode, diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 0ce9d57a65f7..f854cdb13d00 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -86,9 +86,9 @@ async def process_input_messages( if tool_calls_to_check: inputs["tool_calls"] = tool_calls_to_check # type: ignore if messages: - inputs["structured_messages"] = ( - messages # pass the openai /chat/completions messages to the guardrail, as-is - ) + inputs[ + "structured_messages" + ] = messages # pass the openai /chat/completions messages to the guardrail, as-is # Pass tools (function definitions) to the guardrail tools = data.get("tools") if tools: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 02d7b9baf4a4..7ab065ce3931 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -722,9 +722,9 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: import ast try: - generic_user_role_mappings_data: Dict[LitellmUserRoles, List[str]] = ( - ast.literal_eval(generic_role_mappings) - ) + generic_user_role_mappings_data: Dict[ + LitellmUserRoles, List[str] + ] = ast.literal_eval(generic_role_mappings) if isinstance(generic_user_role_mappings_data, dict): from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings @@ -881,9 +881,9 @@ def response_convertor(response, client): verbose_proxy_logger.debug("calling generic_sso.verify_and_process") additional_generic_sso_headers_dict = _parse_generic_sso_headers() - code_verifier: Optional[str] = ( - None # assigned inside try; initialized for type tracking - ) + code_verifier: Optional[ + str + ] = None # assigned inside try; initialized for type tracking access_token_payload: Optional[dict] = None # decoded JWT access token claims try: @@ -1263,9 +1263,9 @@ def apply_user_info_values_to_sso_user_defined_values( else: # SSO didn't provide a valid role, fall back to DB role or default if user_info is None or user_info.user_role is None: - user_defined_values["user_role"] = ( - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value - ) + user_defined_values[ + "user_role" + ] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value verbose_proxy_logger.debug( "No SSO or DB role found, using default: INTERNAL_USER_VIEW_ONLY" ) @@ -1373,14 +1373,16 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: ) elif generic_client_id is not None: - result, received_response, access_token_payload = ( - await get_generic_sso_response( - request=request, - jwt_handler=jwt_handler, - generic_client_id=generic_client_id, - redirect_url=redirect_url, - sso_jwt_handler=sso_jwt_handler, - ) + ( + result, + received_response, + access_token_payload, + ) = await get_generic_sso_response( + request=request, + jwt_handler=jwt_handler, + generic_client_id=generic_client_id, + redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, ) if result is None: @@ -1703,9 +1705,9 @@ async def insert_sso_user( if user_defined_values.get("max_budget") is None: user_defined_values["max_budget"] = litellm.max_internal_user_budget if user_defined_values.get("budget_duration") is None: - user_defined_values["budget_duration"] = ( - litellm.internal_user_budget_duration - ) + user_defined_values[ + "budget_duration" + ] = litellm.internal_user_budget_duration if user_defined_values["user_role"] is None: user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -3348,9 +3350,9 @@ async def get_microsoft_callback_response( # if user is trying to get the raw sso response for debugging, return the raw sso response if return_raw_sso_response: - original_msft_result[MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY] = ( - user_team_ids - ) + original_msft_result[ + MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY + ] = user_team_ids original_msft_result["app_roles"] = app_roles return original_msft_result or {} @@ -3469,9 +3471,9 @@ async def get_user_groups_from_graph_api( # Fetch user membership from Microsoft Graph API all_group_ids = [] - next_link: Optional[str] = ( - MicrosoftSSOHandler.graph_api_user_groups_endpoint - ) + next_link: Optional[ + str + ] = MicrosoftSSOHandler.graph_api_user_groups_endpoint auth_headers = {"Authorization": f"Bearer {access_token}"} page_count = 0 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 317b8f761792..e4b2b184bc09 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -639,9 +639,9 @@ def generate_feedback_box(): server_root_path = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional["EnterpriseLicenseData"] = ( - _license_check.airgapped_license_data -) +premium_user_data: Optional[ + "EnterpriseLicenseData" +] = _license_check.airgapped_license_data global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -1524,9 +1524,9 @@ async def root_redirect(): config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional["ClientSession"] = ( - None # Global shared session for connection reuse -) +shared_aiohttp_session: Optional[ + "ClientSession" +] = None # Global shared session for connection reuse user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1537,13 +1537,13 @@ async def root_redirect(): dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[RedisCache] = ( - None # redis cache used for tracking spend, tpm/rpm limits -) +redis_usage_cache: Optional[ + RedisCache +] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False -native_background_mode: List[str] = ( - [] -) # Models that should use native provider background mode instead of polling +native_background_mode: List[ + str +] = [] # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None user_custom_key_generate = None @@ -2027,9 +2027,9 @@ async def _update_team_cache(): _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[LiteLLM_TeamTable] = ( - await user_api_key_cache.async_get_cache(key=_id) - ) + existing_spend_obj: Optional[ + LiteLLM_TeamTable + ] = await user_api_key_cache.async_get_cache(key=_id) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -2150,9 +2150,11 @@ def run_ollama_serve(): with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - verbose_proxy_logger.debug(f""" + verbose_proxy_logger.debug( + f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` - """) + """ + ) def _get_process_rss_mb() -> Optional[float]: @@ -5200,10 +5202,10 @@ async def _init_guardrails_in_db(self, prisma_client: PrismaClient): ) try: - guardrails_in_db: List[Guardrail] = ( - await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client - ) + guardrails_in_db: List[ + Guardrail + ] = await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -5585,9 +5587,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ["AZURE_API_VERSION"] = ( - api_version # set this for azure - litellm can read this from the env - ) + os.environ[ + "AZURE_API_VERSION" + ] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -5924,9 +5926,9 @@ def _validate_redis_transaction_buffer_config( """ from litellm.secret_managers.main import str_to_bool - _use_redis_transaction_buffer: Optional[Union[bool, str]] = ( - general_settings.get("use_redis_transaction_buffer", False) - ) + _use_redis_transaction_buffer: Optional[ + Union[bool, str] + ] = general_settings.get("use_redis_transaction_buffer", False) if isinstance(_use_redis_transaction_buffer, str): _use_redis_transaction_buffer = str_to_bool(_use_redis_transaction_buffer) @@ -12521,9 +12523,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[idx].field_description = ( - sub_field_info.description - ) + nested_fields[ + idx + ].field_description = sub_field_info.description idx += 1 _stored_in_db = None From 086c41640c5749399f11aba298321d1d16e10a92 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Mon, 30 Mar 2026 09:43:17 +0300 Subject: [PATCH 21/28] fix: keep token in login JSON body for UI storeLoginToken flow Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/proxy/proxy_server.py | 2 +- tests/test_litellm/proxy/test_proxy_server.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e4b2b184bc09..5ef94783a8af 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11315,7 +11315,7 @@ async def login_v2(request: Request): # noqa: PLR0915 litellm_dashboard_ui += "?login=success" json_response = JSONResponse( - content={"redirect_url": litellm_dashboard_ui}, + content={"redirect_url": litellm_dashboard_ui, "token": jwt_token}, status_code=status.HTTP_200_OK, ) json_response.set_cookie(key="token", value=jwt_token, httponly=True) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 11bae9009a24..8fecb2f61682 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -104,7 +104,10 @@ def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): ) assert response.status_code == 200 - assert response.json() == {"redirect_url": "http://testserver/ui/?login=success"} + assert response.json() == { + "redirect_url": "http://testserver/ui/?login=success", + "token": "signed-token", + } assert response.cookies.get("token") == "signed-token" mock_authenticate_user.assert_awaited_once_with( From 866405f443eb2004ee56ed2f52c9047593d8cc6e Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Mon, 30 Mar 2026 09:54:24 +0300 Subject: [PATCH 22/28] fix: use storeLoginToken in exchangeLoginCode, add credentials include Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/proxy/proxy_server.py | 2 +- ui/litellm-dashboard/src/components/networking.tsx | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5ef94783a8af..de57e8cd969e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11486,7 +11486,7 @@ async def login_v3_exchange(request: Request): }, status_code=status.HTTP_200_OK, ) - json_response.set_cookie(key="token", value=cached_data["token"]) + json_response.set_cookie(key="token", value=cached_data["token"], httponly=True) return json_response except ProxyException: raise diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index b3e75f35287c..f3adb0fb1791 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -9166,6 +9166,7 @@ export const exchangeLoginCode = async (code: string, workerBaseUrl?: string | n const response = await fetch(`${base}/v3/login/exchange`, { method: "POST", body: JSON.stringify({ code }), + credentials: "include", headers: { "Content-Type": "application/json" }, }); @@ -9176,7 +9177,7 @@ export const exchangeLoginCode = async (code: string, workerBaseUrl?: string | n const data = await response.json(); if (data.token) { - document.cookie = `token=${data.token}; path=/; SameSite=Lax`; + storeLoginToken(data.token); } return data.token; }; From 0684a1e27521ada35cf8a4afbdff7aeaa87ff41d Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Tue, 31 Mar 2026 22:39:13 +0300 Subject: [PATCH 23/28] revert: remove unrelated changes from HttpOnly cookie fix branch Reset files not related to the login cookie fix back to main: - prometheus.py, bedrock converse, guardrail handler - auth_checks.py, reset_budget_job.py, audit_logs.py - test_user_api_key_auth.py Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/integrations/prometheus.py | 420 +----------------- .../bedrock/chat/converse_transformation.py | 68 +-- .../chat/guardrail_translation/handler.py | 43 +- litellm/proxy/auth/auth_checks.py | 302 +++++-------- .../proxy/common_utils/reset_budget_job.py | 86 +--- .../proxy/management_helpers/audit_logs.py | 128 +----- .../proxy/auth/test_user_api_key_auth.py | 342 ++++---------- 7 files changed, 264 insertions(+), 1125 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 5d1ff8329301..357e0229fc6a 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -65,17 +65,6 @@ def _get_cached_end_user_id_for_cost_tracking(): class PrometheusLogger(CustomLogger): # Class variables or attributes - - @staticmethod - def get_instance() -> Optional["PrometheusLogger"]: - """Find the PrometheusLogger instance from litellm.callbacks, if registered.""" - import litellm - - for cb in litellm.callbacks: - if isinstance(cb, PrometheusLogger): - return cb - return None - def __init__( # noqa: PLR0915 self, **kwargs, @@ -191,31 +180,6 @@ def __init__( # noqa: PLR0915 ), ) - # Remaining Budget for Org - self.litellm_remaining_org_budget_metric = self._gauge_factory( - "litellm_remaining_org_budget_metric", - "Remaining budget for org", - labelnames=self.get_labels_for_metric( - "litellm_remaining_org_budget_metric" - ), - ) - - # Max Budget for Org - self.litellm_org_max_budget_metric = self._gauge_factory( - "litellm_org_max_budget_metric", - "Maximum budget set for org", - labelnames=self.get_labels_for_metric("litellm_org_max_budget_metric"), - ) - - # Org Budget Reset At - self.litellm_org_budget_remaining_hours_metric = self._gauge_factory( - "litellm_org_budget_remaining_hours_metric", - "Remaining hours for org budget to be reset", - labelnames=self.get_labels_for_metric( - "litellm_org_budget_remaining_hours_metric" - ), - ) - # Remaining Budget for API Key self.litellm_remaining_api_key_budget_metric = self._gauge_factory( "litellm_remaining_api_key_budget_metric", @@ -476,76 +440,6 @@ def __init__( # noqa: PLR0915 labelnames=[], ) - ######################################## - # Managed Batch Metrics - ######################################## - self.litellm_managed_batch_created_total = self._counter_factory( - name="litellm_managed_batch_created_total", - documentation="Total number of managed batches created", - labelnames=[ - "model", - "api_provider", - "user", - "user_email", - "api_key_alias", - ], - ) - - self.litellm_managed_file_size_bytes = self._gauge_factory( - "litellm_managed_file_size_bytes", - "Size of the most recent managed batch file in bytes (last-seen value per label combination)", - labelnames=["purpose", "file_type", "model", "api_provider", "user"], - ) - - self.litellm_managed_batch_duration_seconds = self._histogram_factory( - "litellm_managed_batch_duration_seconds", - "Duration of completed managed batches in seconds (completed_at - created_at)", - labelnames=["model", "api_provider"], - buckets=BATCH_DURATION_BUCKETS, - ) - - self.litellm_managed_file_created_total = self._counter_factory( - name="litellm_managed_file_created_total", - documentation="Total number of managed files created", - labelnames=[ - "model", - "api_provider", - "user", - "user_email", - "api_key_alias", - ], - ) - - self.litellm_managed_file_deleted_total = self._counter_factory( - name="litellm_managed_file_deleted_total", - documentation="Total number of managed file deletions (success or blocked)", - labelnames=["result"], - ) - - self.litellm_check_batch_cost_jobs_polled = self._gauge_factory( - "litellm_check_batch_cost_jobs_polled", - "Number of unprocessed batches found by the last CheckBatchCost poll", - labelnames=[], - ) - - self.litellm_check_batch_cost_jobs_processed_total = self._counter_factory( - name="litellm_check_batch_cost_jobs_processed_total", - documentation="Total number of batches successfully cost-tracked by CheckBatchCost", - labelnames=["model", "api_provider"], - ) - - self.litellm_check_batch_cost_errors_total = self._counter_factory( - name="litellm_check_batch_cost_errors_total", - documentation="Total number of errors in CheckBatchCost by error type", - labelnames=["error_type"], - ) - - self.litellm_check_batch_cost_last_run_timestamp = self._gauge_factory( - "litellm_check_batch_cost_last_run_timestamp", - "Unix timestamp of the last CheckBatchCost job run", - labelnames=[], - ) - except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") raise e @@ -1028,9 +922,6 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti user_api_team_alias = standard_logging_payload["metadata"][ "user_api_key_team_alias" ] - user_api_key_org_id = standard_logging_payload["metadata"].get( - "user_api_key_org_id" - ) output_tokens = standard_logging_payload["completion_tokens"] tokens_used = standard_logging_payload["total_tokens"] response_cost = standard_logging_payload["response_cost"] @@ -1040,14 +931,10 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[ "metadata" ].get("user_api_key_auth_metadata") - spend_logs_metadata: Optional[dict] = standard_logging_payload["metadata"].get( - "spend_logs_metadata" - ) combined_metadata: Dict[str, Any] = { **(_requester_metadata if _requester_metadata else {}), **(user_api_key_auth_metadata if user_api_key_auth_metadata else {}), - **(spend_logs_metadata if spend_logs_metadata else {}), } if standard_logging_payload is not None and isinstance( standard_logging_payload, dict @@ -1087,11 +974,9 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti ), client_ip=standard_logging_payload["metadata"].get("requester_ip_address"), user_agent=standard_logging_payload["metadata"].get("user_agent"), - stream=( - str(standard_logging_payload.get("stream")) - if litellm.prometheus_emit_stream_label - else None - ), + stream=str(standard_logging_payload.get("stream")) + if litellm.prometheus_emit_stream_label + else None, ) if ( @@ -1141,7 +1026,6 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti litellm_params=litellm_params, response_cost=response_cost, user_id=user_id, - user_api_key_org_id=user_api_key_org_id, ) # set proxy virtual key rpm/tpm metrics @@ -1297,7 +1181,6 @@ async def _increment_remaining_budget_metrics( litellm_params: dict, response_cost: float, user_id: Optional[str] = None, - user_api_key_org_id: Optional[str] = None, ): _metadata = litellm_params.get("metadata") or {} _team_spend = _metadata.get("user_api_key_team_spend", None) @@ -1330,16 +1213,12 @@ async def _increment_remaining_budget_metrics( user_max_budget=_user_max_budget, response_cost=response_cost, ), - self._set_org_budget_metrics_after_api_request( - org_id=user_api_key_org_id, - response_cost=response_cost, - ), return_exceptions=True, ) for i, r in enumerate(results): if isinstance(r, Exception): verbose_logger.debug( - f"[Non-Blocking] Prometheus: Budget metric lookup {['key', 'team', 'user', 'org'][i]} failed: {r}" + f"[Non-Blocking] Prometheus: Budget metric lookup {['key', 'team', 'user'][i]} failed: {r}" ) def _increment_top_level_request_and_spend_metrics( @@ -1528,9 +1407,6 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti user_api_team_alias = standard_logging_payload["metadata"][ "user_api_key_team_alias" ] - user_api_key_org_id = standard_logging_payload["metadata"].get( - "user_api_key_org_id" - ) try: self.litellm_llm_api_failed_requests_metric.labels( @@ -1546,10 +1422,6 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti ), ).inc() self.set_llm_deployment_failure_metrics(kwargs) - await self._set_org_budget_metrics_after_api_request( - org_id=user_api_key_org_id, - response_cost=0, - ) except Exception as e: verbose_logger.exception( "prometheus Layer Error(): Exception occured - {}".format(str(e)) @@ -1757,11 +1629,9 @@ async def async_post_call_failure_hook( client_ip=_metadata.get("requester_ip_address"), user_agent=_metadata.get("user_agent"), model_id=model_id, - stream=( - str(request_data.get("stream")) - if litellm.prometheus_emit_stream_label - else None - ), + stream=str(request_data.get("stream")) + if litellm.prometheus_emit_stream_label + else None, ) _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric( @@ -2284,127 +2154,6 @@ def _record_guardrail_metrics( except Exception as e: verbose_logger.debug(f"Error recording guardrail metrics: {str(e)}") - ######################################## - # Managed Batch Metric Recording Methods - ######################################## - - def record_managed_batch_created( - self, - model: Optional[str], - api_provider: Optional[str], - user: Optional[str], - user_email: Optional[str], - api_key_alias: Optional[str], - ): - try: - self.litellm_managed_batch_created_total.labels( - model=model, - api_provider=api_provider, - user=user, - user_email=user_email, - api_key_alias=api_key_alias, - ).inc() - except Exception as e: - verbose_logger.warning(f"Error recording batch created metric: {e}") - - def record_managed_file_size( - self, - size_bytes: int, - purpose: str, - file_type: str, - model: Optional[str] = None, - api_provider: Optional[str] = None, - user: Optional[str] = None, - ): - """Record the size of a managed file. Uses a gauge (last-seen value per label combination).""" - try: - self.litellm_managed_file_size_bytes.labels( - purpose=purpose, - file_type=file_type, - model=model or "", - api_provider=api_provider or "", - user=user or "", - ).set(size_bytes) - except Exception as e: - verbose_logger.warning(f"Error recording file size metric: {e}") - - def record_managed_batch_duration( - self, - duration_seconds: float, - model: Optional[str] = None, - api_provider: Optional[str] = None, - ): - try: - self.litellm_managed_batch_duration_seconds.labels( - model=model or "", - api_provider=api_provider or "", - ).observe(duration_seconds) - except Exception as e: - verbose_logger.warning(f"Error recording batch duration metric: {e}") - - def record_managed_file_created( - self, - model: Optional[str], - api_provider: Optional[str], - user: Optional[str], - user_email: Optional[str], - api_key_alias: Optional[str], - ): - try: - self.litellm_managed_file_created_total.labels( - model=model, - api_provider=api_provider, - user=user, - user_email=user_email, - api_key_alias=api_key_alias, - ).inc() - except Exception as e: - verbose_logger.warning(f"Error recording file created metric: {e}") - - def record_managed_file_deleted(self, result: str): - """Record a managed file deletion attempt. result is 'success' or 'blocked'.""" - try: - self.litellm_managed_file_deleted_total.labels(result=result).inc() - except Exception as e: - verbose_logger.warning(f"Error recording file deleted metric: {e}") - - def record_check_batch_cost_run( - self, - jobs_polled: int, - processed_models: Optional[List[Tuple[Optional[str], Optional[str]]]] = None, - ): - """ - Record CheckBatchCost polling metrics. - - Args: - jobs_polled: Number of unprocessed batches found - processed_models: List of (model, api_provider) tuples for processed jobs - """ - import time - - try: - self.litellm_check_batch_cost_last_run_timestamp.set(time.time()) - self.litellm_check_batch_cost_jobs_polled.set(jobs_polled) - - if processed_models: - for model, api_provider in processed_models: - self.litellm_check_batch_cost_jobs_processed_total.labels( - model=model or "", - api_provider=api_provider or "", - ).inc() - except Exception as e: - verbose_logger.warning(f"Error recording check batch cost metrics: {e}") - - def record_check_batch_cost_error(self, error_type: str): - try: - self.litellm_check_batch_cost_errors_total.labels( - error_type=error_type, - ).inc() - except Exception as e: - verbose_logger.warning( - f"Error recording check batch cost error metric: {e}" - ) - @staticmethod def _get_exception_class_name(exception: Exception) -> str: exception_class_name = "" @@ -2781,35 +2530,6 @@ async def fetch_users( data_type="users", ) - async def _initialize_org_budget_metrics(self): - """ - Initialize org budget metrics by reusing the generic pagination logic. - """ - from litellm.proxy.proxy_server import prisma_client - - if prisma_client is None: - verbose_logger.debug( - "Prometheus: skipping org metrics initialization, DB not initialized" - ) - return - - async def fetch_orgs(page_size: int, page: int) -> Tuple[list, Optional[int]]: - skip = (page - 1) * page_size - orgs = await prisma_client.db.litellm_organizationtable.find_many( - skip=skip, - take=page_size, - order={"created_at": "desc"}, - include={"litellm_budget_table": True}, - ) - total_count = await prisma_client.db.litellm_organizationtable.count() - return orgs, total_count - - await self._initialize_budget_metrics( - data_fetch_function=fetch_orgs, - set_metrics_function=self._set_org_list_budget_metrics, - data_type="orgs", - ) - async def initialize_remaining_budget_metrics(self): """ Handler for initializing remaining budget metrics for all teams to avoid metric discrepancies. @@ -2844,11 +2564,10 @@ async def _initialize_remaining_budget_metrics(self): """ Helper to initialize remaining budget metrics for all teams, API keys, and users. """ - verbose_logger.debug("Emitting key, team, user, org budget metrics....") + verbose_logger.debug("Emitting key, team, user budget metrics....") await self._initialize_team_budget_metrics() await self._initialize_api_key_budget_metrics() await self._initialize_user_budget_metrics() - await self._initialize_org_budget_metrics() await self._initialize_user_and_team_count_metrics() async def _initialize_user_and_team_count_metrics(self): @@ -2904,22 +2623,6 @@ async def _set_user_list_budget_metrics(self, users: List[LiteLLM_UserTable]): for user in users: self._set_user_budget_metrics(user) - async def _set_org_list_budget_metrics(self, orgs: list): - """Helper function to set budget metrics for a list of orgs""" - for org in orgs: - budget_table = getattr(org, "litellm_budget_table", None) - self._set_org_budget_metrics( - org_id=org.organization_id or "", - org_alias=org.organization_alias or "", - spend=org.spend or 0.0, - max_budget=budget_table.max_budget if budget_table else None, - budget_reset_at=( - getattr(budget_table, "budget_reset_at", None) - if budget_table - else None - ), - ) - async def _set_team_budget_metrics_after_api_request( self, user_api_team: Optional[str], @@ -3041,113 +2744,6 @@ def _set_team_budget_metrics( ) ) - async def _set_org_budget_metrics_after_api_request( - self, - org_id: Optional[str], - response_cost: float, - ): - """ - Set org budget metrics after an LLM API request - - - Fetches org info via cache (get_org_object) - - Sets org budget metrics - """ - if not org_id: - return - - from litellm.proxy.auth.auth_checks import get_org_object - from litellm.proxy.proxy_server import prisma_client, user_api_key_cache - - if prisma_client is None: - return - - try: - org_info = await get_org_object( - org_id=org_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - include_budget_table=True, - ) - except Exception as e: - verbose_logger.debug( - f"[Non-Blocking] Prometheus: Error getting org info: {str(e)}" - ) - return - - if org_info is None: - return - - org_alias = org_info.organization_alias or "" - _total_org_spend = (org_info.spend or 0.0) + response_cost - budget_table = org_info.litellm_budget_table - max_budget = budget_table.max_budget if budget_table else None - budget_reset_at = ( - getattr(budget_table, "budget_reset_at", None) if budget_table else None - ) - - self._set_org_budget_metrics( - org_id=org_id, - org_alias=org_alias, - spend=_total_org_spend, - max_budget=max_budget, - budget_reset_at=budget_reset_at, - ) - - def _set_org_budget_metrics( - self, - org_id: str, - org_alias: str, - spend: float, - max_budget: Optional[float], - budget_reset_at: Optional[datetime], - ): - """ - Set org budget metrics for a single org - - - Remaining Budget - - Max Budget - - Budget Reset At - """ - enum_values = UserAPIKeyLabelValues( - org_id=org_id, - org_alias=org_alias, - ) - - _labels = prometheus_label_factory( - supported_enum_labels=self.get_labels_for_metric( - metric_name="litellm_remaining_org_budget_metric" - ), - enum_values=enum_values, - ) - self.litellm_remaining_org_budget_metric.labels(**_labels).set( - self._safe_get_remaining_budget( - max_budget=max_budget, - spend=spend, - ) - ) - - if max_budget is not None: - _labels = prometheus_label_factory( - supported_enum_labels=self.get_labels_for_metric( - metric_name="litellm_org_max_budget_metric" - ), - enum_values=enum_values, - ) - self.litellm_org_max_budget_metric.labels(**_labels).set(max_budget) - - if budget_reset_at is not None: - _labels = prometheus_label_factory( - supported_enum_labels=self.get_labels_for_metric( - metric_name="litellm_org_budget_remaining_hours_metric" - ), - enum_values=enum_values, - ) - self.litellm_org_budget_remaining_hours_metric.labels(**_labels).set( - self._get_remaining_hours_for_budget_reset( - budget_reset_at=budget_reset_at - ) - ) - def _set_key_budget_metrics(self, user_api_key_dict: UserAPIKeyAuth): """ Set virtual key budget metrics diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5cfb00d69b61..229457a73b48 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -91,6 +91,34 @@ "compact-2026-01-12", # The compact beta feature is not currently supported on the Converse and ConverseStream APIs ] +# Models that support Bedrock's native structured outputs API (outputConfig.textFormat) +# Uses substring matching against the Bedrock model ID +# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html +BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS = { + # Anthropic Claude 4.5+ + "claude-haiku-4-5", + "claude-sonnet-4-5", + "claude-opus-4-5", + "claude-opus-4-6", + # Qwen3 + "qwen3", + # DeepSeek + "deepseek-v3.1", + # Gemma 3 + "gemma-3", + # MiniMax + "minimax-m2", + # Mistral (magistral-small excluded: broken constrained decoding on Bedrock) + "ministral", + "mistral-large-3", + "voxtral", + # Moonshot + "kimi-k2", + # NVIDIA + "nemotron-nano", + # OpenAI (gpt-oss excluded: broken constrained decoding, works via tool-call fallback) +} + class AmazonConverseConfig(BaseConfig): """ @@ -465,7 +493,8 @@ def _clamp_thinking_budget_tokens(optional_params: dict) -> None: budget = thinking.get("budget_tokens") if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS: verbose_logger.debug( - "Bedrock requires thinking.budget_tokens >= %d, got %d. Clamping to minimum.", + "Bedrock requires thinking.budget_tokens >= %d, got %d. " + "Clamping to minimum.", BEDROCK_MIN_THINKING_BUDGET_TOKENS, budget, ) @@ -734,20 +763,10 @@ def _create_json_tool_call_for_response_format( return _tool @staticmethod - def _supports_native_structured_outputs( - model: str, custom_llm_provider: Optional[str] = None - ) -> bool: - """Check if the Bedrock model supports native structured outputs (outputConfig.textFormat). - - Delegates to the standard ``supports_native_structured_output`` utility - which looks up the flag in ``litellm.model_cost`` via - ``_get_model_info_helper``. - Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html - """ - from litellm.utils import supports_native_structured_output - - return supports_native_structured_output( - model=model, custom_llm_provider=custom_llm_provider + def _supports_native_structured_outputs(model: str) -> bool: + """Check if the Bedrock model supports native structured outputs (outputConfig.textFormat).""" + return any( + substring in model for substring in BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS ) @staticmethod @@ -894,9 +913,7 @@ def map_openai_params( ) if param == "tool_choice": _tool_choice_value = self.map_tool_choice_values( - model=model, - tool_choice=value, - drop_params=drop_params, # type: ignore + model=model, tool_choice=value, drop_params=drop_params # type: ignore ) if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value @@ -989,10 +1006,7 @@ def _translate_response_format_param( if "type" in value and value["type"] == "text": return optional_params - if ( - self._supports_native_structured_outputs(model, self.custom_llm_provider) - and json_schema is not None - ): + if self._supports_native_structured_outputs(model) and json_schema is not None: # Use Bedrock's native structured outputs API (outputConfig.textFormat) # No synthetic tool injection, no fake_stream needed. # Requires an explicit schema — json_object with no schema falls through @@ -1432,16 +1446,6 @@ def _transform_request_helper( original_tools, model, headers, additional_request_params ) - # Append cachePoint to tools if cache_control_injection_points has tool_config - cache_injection_points = additional_request_params.pop( - "cache_control_injection_points", None - ) - if cache_injection_points and len(bedrock_tools) > 0: - for point in cache_injection_points: - if point.get("location") == "tool_config": - bedrock_tools.append({"cachePoint": {"type": "default"}}) - break - bedrock_tool_config: Optional[ToolConfigBlock] = None if len(bedrock_tools) > 0: tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index f854cdb13d00..bab4c3b5eb75 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -260,7 +260,6 @@ async def process_output_response( guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: Optional[Any] = None, user_api_key_dict: Optional[Any] = None, - request_data: Optional[dict] = None, ) -> Any: """ Process output response by applying guardrails to text content. @@ -309,21 +308,15 @@ async def process_output_response( # Step 2: Apply guardrail to all texts and tool calls in batch if texts_to_check or tool_calls_to_check: - # Use the real request_data if provided (proxy path), otherwise - # create a standalone dict (SDK / direct-call path). - if request_data is None: - request_data = {"response": response} - else: - if "response" not in request_data: - request_data["response"] = response + # Create a request_data dict with response info and user API key metadata + request_data: dict = {"response": response} # Add user API key metadata with prefixed keys - if "litellm_metadata" not in request_data: - user_metadata = self.transform_user_api_key_dict_to_metadata( - user_api_key_dict - ) - if user_metadata: - request_data["litellm_metadata"] = user_metadata + user_metadata = self.transform_user_api_key_dict_to_metadata( + user_api_key_dict + ) + if user_metadata: + request_data["litellm_metadata"] = user_metadata inputs = GenericGuardrailAPIInputs(texts=texts_to_check) if images_to_check: @@ -371,7 +364,6 @@ async def process_output_streaming_response( guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: Optional[Any] = None, user_api_key_dict: Optional[Any] = None, - request_data: Optional[dict] = None, ) -> List["ModelResponseStream"]: """ Process output streaming responses by applying guardrails to text content. @@ -410,7 +402,6 @@ async def process_output_streaming_response( guardrail_to_apply=guardrail_to_apply, litellm_logging_obj=litellm_logging_obj, user_api_key_dict=user_api_key_dict, - request_data=request_data, ) return responses_so_far @@ -445,21 +436,15 @@ async def process_output_streaming_response( # Step 3: Apply guardrail to all combined texts in batch if texts_to_check: - # Use the real request_data if provided (proxy path), otherwise - # create a standalone dict (SDK / direct-call path). - if request_data is None: - request_data = {"responses": responses_so_far} - else: - if "responses" not in request_data: - request_data["responses"] = responses_so_far + # Create a request_data dict with response info and user API key metadata + request_data: dict = {"responses": responses_so_far} # Add user API key metadata with prefixed keys - if "litellm_metadata" not in request_data: - user_metadata = self.transform_user_api_key_dict_to_metadata( - user_api_key_dict - ) - if user_metadata: - request_data["litellm_metadata"] = user_metadata + user_metadata = self.transform_user_api_key_dict_to_metadata( + user_api_key_dict + ) + if user_metadata: + request_data["litellm_metadata"] = user_metadata inputs = GenericGuardrailAPIInputs(texts=texts_to_check) if images_to_check: diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index ed600b26e832..b4c123b361b9 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -8,7 +8,6 @@ 2. If user is in budget 3. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget """ - import asyncio import re import time @@ -30,7 +29,6 @@ DEFAULT_MAX_RECURSE_DEPTH, EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE, ) -from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.proxy._types import ( RBAC_ROLES, @@ -40,7 +38,6 @@ LiteLLM_EndUserTable, Litellm_EntityType, LiteLLM_JWTAuth, - LiteLLM_ManagedVectorStoresTable, LiteLLM_ObjectPermissionTable, LiteLLM_OrganizationMembershipTable, LiteLLM_OrganizationTable, @@ -405,26 +402,23 @@ async def common_checks( # noqa: PLR0915 # 1. If team is blocked if team_object is not None and team_object.blocked is True: raise Exception( - f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if you're an admin." + f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if your admin." ) # 2. If team can call model if _model and team_object: - with tracer.trace("litellm.proxy.auth.common_checks.can_team_access_model"): - if not await can_team_access_model( - model=_model, - team_object=team_object, - llm_router=llm_router, - team_model_aliases=( - valid_token.team_model_aliases if valid_token else None - ), - ): - raise ProxyException( - message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", - type=ProxyErrorTypes.team_model_access_denied, - param="model", - code=status.HTTP_401_UNAUTHORIZED, - ) + if not await can_team_access_model( + model=_model, + team_object=team_object, + llm_router=llm_router, + team_model_aliases=valid_token.team_model_aliases if valid_token else None, + ): + raise ProxyException( + message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", + type=ProxyErrorTypes.team_model_access_denied, + param="model", + code=status.HTTP_401_UNAUTHORIZED, + ) # Require trace id for agent keys when agent has require_trace_id_on_calls_by_agent if valid_token is not None and valid_token.agent_id: @@ -449,62 +443,54 @@ async def common_checks( # noqa: PLR0915 ## 2.1 If user can call model (if personal key) if _model and team_object is None and user_object is not None: - with tracer.trace("litellm.proxy.auth.common_checks.can_user_call_model"): - await can_user_call_model( - model=_model, - llm_router=llm_router, - user_object=user_object, - ) - - # 1.1 - 2.2 - 3.0.2 - 3.0.3: Project checks (blocked, model access, budget) - with tracer.trace("litellm.proxy.auth.common_checks.run_project_checks"): - await _run_project_checks( - project_object=project_object, - _model=_model, + await can_user_call_model( + model=_model, llm_router=llm_router, - skip_budget_checks=skip_budget_checks, - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, + user_object=user_object, ) + # 1.1 - 2.2 - 3.0.2 - 3.0.3: Project checks (blocked, model access, budget) + await _run_project_checks( + project_object=project_object, + _model=_model, + llm_router=llm_router, + skip_budget_checks=skip_budget_checks, + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + ) + # If this is a free model, skip all budget checks if not skip_budget_checks: # 3. If team is in budget - with tracer.trace("litellm.proxy.auth.common_checks.team_max_budget_check"): - await _team_max_budget_check( - team_object=team_object, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + await _team_max_budget_check( + team_object=team_object, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 3.0.5. If team is over soft budget (alert only, doesn't block) - with tracer.trace("litellm.proxy.auth.common_checks.team_soft_budget_check"): - await _team_soft_budget_check( - team_object=team_object, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + await _team_soft_budget_check( + team_object=team_object, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 3.1. If organization is in budget - with tracer.trace( - "litellm.proxy.auth.common_checks.organization_max_budget_check" - ): - await _organization_max_budget_check( - valid_token=valid_token, - team_object=team_object, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + await _organization_max_budget_check( + valid_token=valid_token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) - with tracer.trace("litellm.proxy.auth.common_checks.tag_max_budget_check"): - await _tag_max_budget_check( - request_body=request_body, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + await _tag_max_budget_check( + request_body=request_body, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 4. If user is in budget ## 4.1 check personal budget, if personal key @@ -522,15 +508,14 @@ async def common_checks( # noqa: PLR0915 ) ## 4.2 check team member budget, if team key - with tracer.trace("litellm.proxy.auth.common_checks.check_team_member_budget"): - await _check_team_member_budget( - team_object=team_object, - user_object=user_object, - valid_token=valid_token, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + await _check_team_member_budget( + team_object=team_object, + user_object=user_object, + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) # 5. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget if ( @@ -569,21 +554,19 @@ async def common_checks( # noqa: PLR0915 ) # 11. [OPTIONAL] Vector store checks - is the object allowed to access the vector store - with tracer.trace("litellm.proxy.auth.common_checks.vector_store_access_check"): - await vector_store_access_check( - request_body=request_body, - team_object=team_object, - valid_token=valid_token, - ) + await vector_store_access_check( + request_body=request_body, + team_object=team_object, + valid_token=valid_token, + ) # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path) - with tracer.trace("litellm.proxy.auth.common_checks.check_tools_allowlist"): - await check_tools_allowlist( - request_body=request_body, - valid_token=valid_token, - team_object=team_object, - route=route, - ) + await check_tools_allowlist( + request_body=request_body, + valid_token=valid_token, + team_object=team_object, + route=route, + ) return True @@ -2296,71 +2279,6 @@ async def get_object_permission( return None -@log_db_metrics -async def get_managed_vector_store_rows_by_uuids( - uuids: List[str], - prisma_client: Optional[PrismaClient], - user_api_key_cache: DualCache, - parent_otel_span: Optional[Span] = None, - proxy_logging_obj: Optional[ProxyLogging] = None, -) -> List[LiteLLM_ManagedVectorStoresTable]: - """ - Fetch managed vector store rows by their internal UUIDs. - - Follows the get_team_object / get_key_object / get_object_permission pattern: - cache-first lookup (in-memory / Redis), DB fallback only on cache miss. - Critical-path DB access must go through this helper to avoid raw Prisma - calls on the hot request path. - """ - if not uuids or prisma_client is None: - return [] - - result: List[LiteLLM_ManagedVectorStoresTable] = [] - cache_misses: List[str] = [] - - for uuid in uuids: - key = "managed_vector_store_id:{}".format(uuid) - cached = await user_api_key_cache.async_get_cache(key=key) - if cached is not None: - if isinstance(cached, dict): - result.append(LiteLLM_ManagedVectorStoresTable(**cached)) - elif isinstance(cached, LiteLLM_ManagedVectorStoresTable): - result.append(cached) - else: - cache_misses.append(uuid) - else: - cache_misses.append(uuid) - - if not cache_misses: - return result - - rows = await prisma_client.db.litellm_managedvectorstorestable.find_many( - where={"vector_store_id": {"in": cache_misses}}, - take=len(cache_misses), - ) - - for row in rows: - row_dict = ( - row.model_dump() - if hasattr(row, "model_dump") - else (row.dict() if hasattr(row, "dict") else None) - ) - if not isinstance(row_dict, dict) or not row_dict: - row_dict = dict(row) if hasattr(row, "__dict__") else {} - if not row_dict: - continue - cached_obj = LiteLLM_ManagedVectorStoresTable(**row_dict) - key = "managed_vector_store_id:{}".format(cached_obj.vector_store_id) - await user_api_key_cache.async_set_cache( - key=key, - value=row_dict, - ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, - ) - result.append(cached_obj) - - return result - - @log_db_metrics async def get_org_object( org_id: str, @@ -2877,15 +2795,7 @@ async def _virtual_key_max_budget_check( Triggers a budget alert if the token is over it's max budget. """ - if valid_token.max_budget is not None: - from litellm.proxy.proxy_server import get_current_spend - - # Read spend from cross-pod counter (Redis-first) or cached object (fallback) - spend = await get_current_spend( - counter_key=f"spend:key:{valid_token.token}", - fallback_spend=valid_token.spend or 0.0, - ) - + if valid_token.spend is not None and valid_token.max_budget is not None: #################################### # collect information for alerting # #################################### @@ -2897,7 +2807,7 @@ async def _virtual_key_max_budget_check( call_info = CallInfo( token=valid_token.token, - spend=spend, + spend=valid_token.spend, max_budget=valid_token.max_budget, soft_budget=valid_token.soft_budget, user_id=valid_token.user_id, @@ -2918,9 +2828,9 @@ async def _virtual_key_max_budget_check( # collect information for alerting # #################################### - if spend >= valid_token.max_budget: + if valid_token.spend >= valid_token.max_budget: raise litellm.BudgetExceededError( - current_cost=spend, + current_cost=valid_token.spend, max_budget=valid_token.max_budget, ) @@ -3051,14 +2961,6 @@ async def _check_team_member_budget( team_member_budget = team_membership.litellm_budget_table.max_budget team_member_spend = team_membership.spend or 0.0 - # Read from cross-pod counter (Redis-first) if available - from litellm.proxy.proxy_server import get_current_spend - - team_member_spend = await get_current_spend( - counter_key=f"spend:team_member:{valid_token.user_id}:{team_object.team_id}", - fallback_spend=team_member_spend, - ) - if team_member_spend >= team_member_budget: raise litellm.BudgetExceededError( current_cost=team_member_spend, @@ -3079,39 +2981,35 @@ async def _team_max_budget_check( BudgetExceededError if the team is over it's max budget. Triggers a budget alert if the team is over it's max budget. """ - if team_object is not None and team_object.max_budget is not None: - from litellm.proxy.proxy_server import get_current_spend - - # Read spend from cross-pod counter (Redis-first) or cached object (fallback) - spend = await get_current_spend( - counter_key=f"spend:team:{team_object.team_id}", - fallback_spend=team_object.spend or 0.0, - ) - - if spend > team_object.max_budget: - if valid_token: - call_info = CallInfo( - token=valid_token.token, - spend=spend, - max_budget=team_object.max_budget, - user_id=valid_token.user_id, - team_id=valid_token.team_id, - team_alias=valid_token.team_alias, - organization_id=valid_token.org_id, - event_group=Litellm_EntityType.TEAM, - ) - asyncio.create_task( - proxy_logging_obj.budget_alerts( - type="team_budget", - user_info=call_info, - ) - ) - - raise litellm.BudgetExceededError( - current_cost=spend, + if ( + team_object is not None + and team_object.max_budget is not None + and team_object.spend is not None + and team_object.spend > team_object.max_budget + ): + if valid_token: + call_info = CallInfo( + token=valid_token.token, + spend=team_object.spend, max_budget=team_object.max_budget, - message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}", + user_id=valid_token.user_id, + team_id=valid_token.team_id, + team_alias=valid_token.team_alias, + organization_id=valid_token.org_id, + event_group=Litellm_EntityType.TEAM, ) + asyncio.create_task( + proxy_logging_obj.budget_alerts( + type="team_budget", + user_info=call_info, + ) + ) + + raise litellm.BudgetExceededError( + current_cost=team_object.spend, + max_budget=team_object.max_budget, + message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {team_object.spend}, Max budget: {team_object.max_budget}", + ) async def _team_soft_budget_check( diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index bcfaed243981..674214b19e53 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -54,47 +54,16 @@ async def reset_budget_for_litellm_team_members( """ Resets the budget for all LiteLLM Team Members if their budget has expired """ - budget_ids = [ - budget.budget_id - for budget in budgets_to_reset - if budget.budget_id is not None - ] - - # Reset spend counters for affected team members. - # Reset Redis directly so a transient failure doesn't leave stale - # counters that get_current_spend would read as authoritative. - try: - from litellm.proxy.proxy_server import spend_counter_cache - - memberships = await self.prisma_client.db.litellm_teammembership.find_many( - where={"budget_id": {"in": budget_ids}} - ) - for m in memberships: - counter_key = f"spend:team_member:{m.user_id}:{m.team_id}" - # Always reset in-memory - spend_counter_cache.in_memory_cache.set_cache( - key=counter_key, value=0.0 - ) - # Explicitly reset Redis with warning on failure - if spend_counter_cache.redis_cache is not None: - try: - await spend_counter_cache.redis_cache.async_set_cache( - key=counter_key, value=0.0 - ) - except Exception as redis_err: - verbose_proxy_logger.warning( - "Failed to reset team member spend counter in Redis %s: %s. " - "Budget may be over-enforced until counter expires.", - counter_key, - redis_err, - ) - except Exception as e: - verbose_proxy_logger.warning( - "Failed to reset team member spend counters: %s", e - ) - return await self.prisma_client.db.litellm_teammembership.update_many( - where={"budget_id": {"in": budget_ids}}, + where={ + "budget_id": { + "in": [ + budget.budget_id + for budget in budgets_to_reset + if budget.budget_id is not None + ] + } + }, data={ "spend": 0, }, @@ -562,43 +531,6 @@ async def _reset_budget_common( """ try: item.spend = 0.0 - - # Reset the cross-pod spend counter. - # Reset Redis directly (not via DualCache) so a Redis failure - # doesn't silently leave a stale counter that get_current_spend - # would read as authoritative, permanently blocking the user. - from litellm.proxy.proxy_server import spend_counter_cache - - counter_key = None - if item_type == "key" and hasattr(item, "token") and item.token is not None: - counter_key = f"spend:key:{item.token}" - elif ( - item_type == "team" - and hasattr(item, "team_id") - and item.team_id is not None - ): - counter_key = f"spend:team:{item.team_id}" - - if counter_key is not None: - # Always reset in-memory (local fallback) - spend_counter_cache.in_memory_cache.set_cache( - key=counter_key, value=0.0 - ) - # Explicitly reset Redis with warning on failure - if spend_counter_cache.redis_cache is not None: - try: - await spend_counter_cache.redis_cache.async_set_cache( - key=counter_key, value=0.0 - ) - except Exception as redis_err: - verbose_proxy_logger.warning( - "Failed to reset spend counter in Redis for %s key=%s: %s. " - "Budget may be over-enforced until counter expires.", - item_type, - counter_key, - redis_err, - ) - if hasattr(item, "budget_duration") and item.budget_duration is not None: # Get standardized reset time based on budget duration from litellm.proxy.common_utils.timezone_utils import ( diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index 7599e11bdef0..ea082f468aee 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -2,15 +2,12 @@ Functions to create audit logs for LiteLLM Proxy """ -import asyncio import json +from litellm._uuid import uuid from datetime import datetime, timezone -from typing import Dict import litellm from litellm._logging import verbose_proxy_logger -from litellm._uuid import uuid -from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ( AUDIT_ACTIONS, LiteLLM_AuditLogs, @@ -18,99 +15,6 @@ Optional, UserAPIKeyAuth, ) -from litellm.types.utils import StandardAuditLogPayload - -_audit_log_callback_cache: Dict[str, CustomLogger] = {} - - -def _resolve_audit_log_callback(name: str) -> Optional[CustomLogger]: - """Resolve a string callback name to a CustomLogger instance, with caching.""" - if name in _audit_log_callback_cache: - return _audit_log_callback_cache[name] - - from litellm.litellm_core_utils.litellm_logging import ( - _init_custom_logger_compatible_class, - ) - - instance = _init_custom_logger_compatible_class( - logging_integration=name, # type: ignore - internal_usage_cache=None, - llm_router=None, - ) - - if instance is not None: - _audit_log_callback_cache[name] = instance - return instance - - -def _build_audit_log_payload( - request_data: LiteLLM_AuditLogs, -) -> StandardAuditLogPayload: - """Convert LiteLLM_AuditLogs to StandardAuditLogPayload for callback dispatch.""" - updated_at = "" - if request_data.updated_at is not None: - updated_at = request_data.updated_at.isoformat() - - table_name_str: str = ( - request_data.table_name.value - if isinstance(request_data.table_name, LitellmTableNames) - else str(request_data.table_name) - ) - - return StandardAuditLogPayload( - id=request_data.id, - updated_at=updated_at, - changed_by=request_data.changed_by or "", - changed_by_api_key=request_data.changed_by_api_key or "", - action=request_data.action, - table_name=table_name_str, - object_id=request_data.object_id, - before_value=request_data.before_value, - updated_values=request_data.updated_values, - ) - - -def _audit_log_task_done_callback(task: asyncio.Task) -> None: - """Log exceptions from audit log callback tasks so they don't slip through silently.""" - try: - exc = task.exception() - except asyncio.CancelledError: - return - if exc is not None: - verbose_proxy_logger.error( - "Audit log callback task failed: %s", exc, exc_info=exc - ) - - -async def _dispatch_audit_log_to_callbacks( - request_data: LiteLLM_AuditLogs, -) -> None: - """Dispatch audit log to all registered audit_log_callbacks.""" - if not litellm.audit_log_callbacks: - return - - payload = _build_audit_log_payload(request_data) - - for callback in litellm.audit_log_callbacks: - try: - resolved: Optional[CustomLogger] = ( - callback if isinstance(callback, CustomLogger) else None - ) - if isinstance(callback, str): - resolved = _resolve_audit_log_callback(callback) - if resolved is None: - verbose_proxy_logger.warning( - "Could not resolve audit log callback: %s", callback - ) - continue - - if isinstance(resolved, CustomLogger): - task = asyncio.create_task(resolved.async_log_audit_log_event(payload)) - task.add_done_callback(_audit_log_task_done_callback) - except Exception as e: - verbose_proxy_logger.error( - "Failed dispatching audit log to callback: %s", e - ) async def create_object_audit_log( @@ -136,22 +40,20 @@ async def create_object_audit_log( """ from litellm.secret_managers.main import get_secret_bool - _store_audit_logs: Optional[bool] = litellm.store_audit_logs or get_secret_bool( + store_audit_logs = litellm.store_audit_logs or get_secret_bool( "LITELLM_STORE_AUDIT_LOGS" ) - if _store_audit_logs is not True: + if store_audit_logs is not True: return - _changed_by = ( - litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name - ) - await create_audit_log_for_update( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=_changed_by, + changed_by=litellm_changed_by + or user_api_key_dict.user_id + or litellm_proxy_admin_name, changed_by_api_key=user_api_key_dict.api_key, table_name=table_name, object_id=object_id, @@ -168,10 +70,10 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): """ from litellm.secret_managers.main import get_secret_bool - _store_audit_logs: Optional[bool] = litellm.store_audit_logs or get_secret_bool( + store_audit_logs = litellm.store_audit_logs or get_secret_bool( "LITELLM_STORE_AUDIT_LOGS" ) - if _store_audit_logs is not True: + if store_audit_logs is not True: return from litellm.proxy.proxy_server import premium_user, prisma_client @@ -179,6 +81,9 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): if premium_user is not True: return + if prisma_client is None: + raise Exception("prisma_client is None, no DB connected") + verbose_proxy_logger.debug("creating audit log for %s", request_data) if isinstance(request_data.updated_values, dict): @@ -187,15 +92,6 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): if isinstance(request_data.before_value, dict): request_data.before_value = json.dumps(request_data.before_value) - # Dispatch to external audit log callbacks regardless of DB availability - await _dispatch_audit_log_to_callbacks(request_data) - - if prisma_client is None: - verbose_proxy_logger.error( - "prisma_client is None, cannot write audit log to DB" - ) - return - _request_data = request_data.model_dump(exclude_none=True) try: @@ -207,3 +103,5 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): except Exception as e: # [Non-Blocking Exception. Do not allow blocking LLM API call] verbose_proxy_logger.error(f"Failed Creating audit log {e}") + + return diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 74c7f9bca5f3..f3f0ba56cb91 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -72,7 +72,7 @@ def test_get_api_key_with_custom_litellm_key_header( def test_team_metadata_with_tags_flows_through_jwt_auth(): """ Test that team_metadata (specifically tags) flows through JWT authentication. - + This is a regression test for the issue where JWT auth was not populating team_metadata, causing team-level tags to be missing in litellm_pre_call_utils.py """ @@ -87,7 +87,7 @@ def test_team_metadata_with_tags_flows_through_jwt_auth(): rpm_limit=100, models=["gpt-4", "gpt-3.5-turbo"], ) - + # Simulate constructing UserAPIKeyAuth like we do in JWT auth # This is the pattern from user_api_key_auth.py lines 552-587 user_api_key_auth = UserAPIKeyAuth( @@ -100,16 +100,14 @@ def test_team_metadata_with_tags_flows_through_jwt_auth(): user_role="internal_user", user_id="test-user", ) - + # Verify team_metadata is set - assert ( - user_api_key_auth.team_metadata is not None - ), "team_metadata should be populated" + assert user_api_key_auth.team_metadata is not None, "team_metadata should be populated" assert user_api_key_auth.team_metadata == team_object.metadata, ( f"team_metadata not correctly mapped. " f"Expected: {team_object.metadata}, Got: {user_api_key_auth.team_metadata}" ) - + # Specifically verify tags are present assert "tags" in user_api_key_auth.team_metadata, "tags should be in team_metadata" assert user_api_key_auth.team_metadata["tags"] == ["production", "high-priority"], ( @@ -120,7 +118,7 @@ def test_team_metadata_with_tags_flows_through_jwt_auth(): def test_route_checks_is_llm_api_route(): """Test RouteChecks.is_llm_api_route() correctly identifies LLM API routes including passthrough endpoints""" - + # Test OpenAI routes openai_routes = [ "/v1/chat/completions", @@ -144,22 +142,18 @@ def test_route_checks_is_llm_api_route(): "/v1/realtime", "/realtime", ] - + for route in openai_routes: - assert RouteChecks.is_llm_api_route( - route=route - ), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" # Test Anthropic routes anthropic_routes = [ "/v1/messages", "/v1/messages/count_tokens", ] - + for route in anthropic_routes: - assert RouteChecks.is_llm_api_route( - route=route - ), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" # Test passthrough routes (this is the key improvement over the old route checking) passthrough_routes = [ @@ -177,11 +171,9 @@ def test_route_checks_is_llm_api_route(): "/vllm/v1/chat/completions", "/mistral/v1/chat/completions", ] - + for route in passthrough_routes: - assert RouteChecks.is_llm_api_route( - route=route - ), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" # Test MCP routes mcp_routes = [ @@ -189,11 +181,9 @@ def test_route_checks_is_llm_api_route(): "/mcp/", "/mcp/test", ] - + for route in mcp_routes: - assert RouteChecks.is_llm_api_route( - route=route - ), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" # Test LiteLLM native RAG routes rag_routes = [ @@ -203,9 +193,7 @@ def test_route_checks_is_llm_api_route(): "/v1/rag/query", ] for route in rag_routes: - assert RouteChecks.is_llm_api_route( - route=route - ), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" # Test routes with placeholders placeholder_routes = [ @@ -218,11 +206,9 @@ def test_route_checks_is_llm_api_route(): "/v1/batches/batch_123", "/batches/batch_123", ] - + for route in placeholder_routes: - assert RouteChecks.is_llm_api_route( - route=route - ), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" # Test Azure OpenAI routes azure_routes = [ @@ -231,11 +217,9 @@ def test_route_checks_is_llm_api_route(): "/engines/gpt-4/chat/completions", "/engines/gpt-3.5-turbo/completions", ] - + for route in azure_routes: - assert RouteChecks.is_llm_api_route( - route=route - ), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" # Test non-LLM routes (should return False) non_llm_routes = [ @@ -252,11 +236,9 @@ def test_route_checks_is_llm_api_route(): "/debug", "/test", ] - + for route in non_llm_routes: - assert not RouteChecks.is_llm_api_route( - route=route - ), f"Route {route} should NOT be identified as LLM API route" + assert not RouteChecks.is_llm_api_route(route=route), f"Route {route} should NOT be identified as LLM API route" # Test invalid inputs invalid_inputs = [ @@ -266,11 +248,9 @@ def test_route_checks_is_llm_api_route(): {}, "", ] - + for invalid_input in invalid_inputs: - assert not RouteChecks.is_llm_api_route( - route=invalid_input - ), f"Invalid input {invalid_input} should return False" + assert not RouteChecks.is_llm_api_route(route=invalid_input), f"Invalid input {invalid_input} should return False" @pytest.mark.asyncio @@ -279,7 +259,7 @@ async def test_proxy_admin_expired_key_from_cache(): Test that PROXY_ADMIN keys retrieved from cache are checked for expiration before being returned. This prevents expired keys from bypassing expiration checks when retrieved from cache (which normally happens at lines 1014-1036). - + Regression test for issue where PROXY_ADMIN keys from cache skipped expiration check. """ from datetime import datetime, timedelta, timezone @@ -300,42 +280,39 @@ async def test_proxy_admin_expired_key_from_cache(): api_key = "sk-test-proxy-admin-key" hashed_key = hash_token(api_key) expired_time = datetime.now(timezone.utc) - timedelta(hours=1) # Expired 1 hour ago - + expired_token = UserAPIKeyAuth( api_key=api_key, user_role=LitellmUserRoles.PROXY_ADMIN, expires=expired_time, token=hashed_key, ) - + # Mock cache to return the expired token mock_cache = AsyncMock() mock_cache.async_get_cache = AsyncMock(return_value=expired_token) mock_cache.delete_cache = MagicMock() - + # Mock proxy_logging_obj mock_proxy_logging_obj = MagicMock() mock_proxy_logging_obj.internal_usage_cache = MagicMock() mock_proxy_logging_obj.internal_usage_cache.dual_cache = AsyncMock() - mock_proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache = ( - AsyncMock() - ) + mock_proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock() # Mock post_call_failure_hook as async function returning None (no transformation) mock_proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) - + # Mock prisma_client mock_prisma_client = MagicMock() - + # Mock get_key_object to return expired token from cache with patch( "litellm.proxy.auth.user_api_key_auth.get_key_object", new_callable=AsyncMock, - ) as mock_get_key_object, patch( - "litellm.proxy.auth.user_api_key_auth._delete_cache_key_object", - new_callable=AsyncMock, - ) as mock_delete_cache: + ) as mock_get_key_object, \ + patch("litellm.proxy.auth.user_api_key_auth._delete_cache_key_object", new_callable=AsyncMock) as mock_delete_cache: + mock_get_key_object.return_value = expired_token - + # Set attributes on proxy_server module (these are imported inside _user_api_key_auth_builder) import litellm.proxy.proxy_server as _proxy_server_mod @@ -354,12 +331,14 @@ async def test_proxy_admin_expired_key_from_cache(): "litellm_proxy_admin_name": "admin", } _original_values = { - attr: getattr(_proxy_server_mod, attr, None) for attr in _attrs_to_set + attr: getattr(_proxy_server_mod, attr, None) + for attr in _attrs_to_set } try: for attr, val in _attrs_to_set.items(): setattr(_proxy_server_mod, attr, val) + # Create a mock request request = Request(scope={"type": "http"}) request._url = URL(url="/chat/completions") @@ -379,41 +358,38 @@ async def test_proxy_admin_expired_key_from_cache(): ) # Verify that ProxyException was raised with expired_key type - assert hasattr( - exc_info.value, "type" - ), "Exception should have 'type' attribute" - assert ( - exc_info.value.type == ProxyErrorTypes.expired_key - ), f"Expected expired_key error type, got {exc_info.value.type}" - assert "Expired Key" in str( - exc_info.value.message - ), f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" + assert hasattr(exc_info.value, "type"), "Exception should have 'type' attribute" + assert exc_info.value.type == ProxyErrorTypes.expired_key, ( + f"Expected expired_key error type, got {exc_info.value.type}" + ) + assert "Expired Key" in str(exc_info.value.message), ( + f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" + ) # Verify that the param field does NOT leak the full API key (Issue #18731) # The param should be abbreviated like "sk-...XXXX" not the full plaintext key - assert ( - exc_info.value.param is not None - ), "Exception should have 'param' attribute" + assert exc_info.value.param is not None, "Exception should have 'param' attribute" assert exc_info.value.param != api_key, ( f"SECURITY: Full API key should NOT be in param field! " f"Got: {exc_info.value.param}, Expected abbreviated format like 'sk-...XXXX'" ) - assert exc_info.value.param.startswith( - "sk-..." - ), f"Param should be abbreviated to 'sk-...XXXX' format. Got: {exc_info.value.param}" + assert exc_info.value.param.startswith("sk-..."), ( + f"Param should be abbreviated to 'sk-...XXXX' format. Got: {exc_info.value.param}" + ) # Verify that cache deletion was called mock_delete_cache.assert_called_once() call_args = mock_delete_cache.call_args - assert ( - call_args[1]["hashed_token"] == hashed_key - ), "Cache deletion should be called with the hashed key" + assert call_args[1]["hashed_token"] == hashed_key, ( + "Cache deletion should be called with the hashed key" + ) finally: # Restore all module-level attributes so subsequent tests are not affected for attr, val in _original_values.items(): setattr(_proxy_server_mod, attr, val) + @pytest.mark.asyncio async def test_return_user_api_key_auth_obj_user_spend_and_budget(): """ @@ -424,7 +400,7 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import _return_user_api_key_auth_obj - + user_obj = type( "LiteLLM_UserTable", (), @@ -437,7 +413,7 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): "user_role": "internal_user", }, ) - + api_key = "sk-test-key" valid_token_dict = { "user_id": "test-user", @@ -445,10 +421,10 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): } route = "/chat/completions" start_time = datetime.now() - + mock_service_logger = MagicMock() mock_service_logger.async_service_success_hook = AsyncMock() - + with patch( "litellm.proxy.auth.user_api_key_auth.user_api_key_service_logger_obj", new=mock_service_logger, @@ -462,7 +438,7 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): start_time=start_time, user_role=None, ) - + assert isinstance(result, UserAPIKeyAuth) assert result.user_spend == 250.0 assert result.user_max_budget == 1000.0 @@ -494,7 +470,9 @@ def test_proxy_admin_jwt_auth_includes_identity_fields(): user_role=LitellmUserRoles.PROXY_ADMIN, user_id="user-abc", team_id="team-123", - team_alias=(team_object.team_alias if team_object is not None else None), + team_alias=( + team_object.team_alias if team_object is not None else None + ), team_metadata=team_object.metadata if team_object is not None else None, org_id="org-456", end_user_id="end-user-789", @@ -525,7 +503,9 @@ def test_proxy_admin_jwt_auth_handles_no_team_object(): user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-user", team_id=None, - team_alias=(team_object.team_alias if team_object is not None else None), + team_alias=( + team_object.team_alias if team_object is not None else None + ), team_metadata=team_object.metadata if team_object is not None else None, org_id=None, end_user_id=None, @@ -554,10 +534,7 @@ class TestJWTOAuth2Coexistence: def test_is_jwt_detects_jwt_tokens(self): """JWT tokens have 3 dot-separated parts.""" assert JWTHandler.is_jwt("header.payload.signature") is True - assert ( - JWTHandler.is_jwt("eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.sig123") - is True - ) + assert JWTHandler.is_jwt("eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.sig123") is True def test_is_jwt_rejects_opaque_tokens(self): """Opaque OAuth2 tokens do not have 3 dot-separated parts.""" @@ -566,10 +543,6 @@ def test_is_jwt_rejects_opaque_tokens(self): assert JWTHandler.is_jwt("Bearer token") is False assert JWTHandler.is_jwt("two.parts") is False - def test_is_jwt_returns_false_for_none(self): - """None token (missing Authorization header) should not be treated as JWT.""" - assert JWTHandler.is_jwt(None) is False - @pytest.mark.asyncio async def test_both_enabled_opaque_token_uses_oauth2(self): """ @@ -594,20 +567,13 @@ async def test_both_enabled_opaque_token_uses_oauth2(self): mock_request.headers = {"authorization": f"Bearer {opaque_token}"} mock_request.query_params = {} - with patch( - "litellm.proxy.proxy_server.general_settings", general_settings - ), patch("litellm.proxy.proxy_server.premium_user", True), patch( - "litellm.proxy.proxy_server.master_key", "sk-master" - ), patch( - "litellm.proxy.proxy_server.prisma_client", None - ), patch( - "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", - new_callable=AsyncMock, - return_value=mock_oauth2_response, - ) as mock_oauth2, patch( - "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", - new_callable=AsyncMock, - ) as mock_jwt_auth: + with patch("litellm.proxy.proxy_server.general_settings", general_settings), \ + patch("litellm.proxy.proxy_server.premium_user", True), \ + patch("litellm.proxy.proxy_server.master_key", "sk-master"), \ + patch("litellm.proxy.proxy_server.prisma_client", None), \ + patch("litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", new_callable=AsyncMock, return_value=mock_oauth2_response) as mock_oauth2, \ + patch("litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", new_callable=AsyncMock) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( prisma_client=None, user_api_key_cache=DualCache(), @@ -658,20 +624,13 @@ async def test_both_enabled_jwt_token_skips_oauth2(self): mock_request.headers = {"authorization": f"Bearer {jwt_token}"} mock_request.query_params = {} - with patch( - "litellm.proxy.proxy_server.general_settings", general_settings - ), patch("litellm.proxy.proxy_server.premium_user", True), patch( - "litellm.proxy.proxy_server.master_key", "sk-master" - ), patch( - "litellm.proxy.proxy_server.prisma_client", None - ), patch( - "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", - new_callable=AsyncMock, - ) as mock_oauth2, patch( - "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", - new_callable=AsyncMock, - return_value=mock_jwt_result, - ) as mock_jwt_auth: + with patch("litellm.proxy.proxy_server.general_settings", general_settings), \ + patch("litellm.proxy.proxy_server.premium_user", True), \ + patch("litellm.proxy.proxy_server.master_key", "sk-master"), \ + patch("litellm.proxy.proxy_server.prisma_client", None), \ + patch("litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", new_callable=AsyncMock) as mock_oauth2, \ + patch("litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", new_callable=AsyncMock, return_value=mock_jwt_result) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( prisma_client=None, user_api_key_cache=DualCache(), @@ -712,17 +671,12 @@ async def test_only_oauth2_enabled_handles_all_tokens(self): mock_request.headers = {"authorization": f"Bearer {jwt_like_token}"} mock_request.query_params = {} - with patch( - "litellm.proxy.proxy_server.general_settings", general_settings - ), patch("litellm.proxy.proxy_server.premium_user", True), patch( - "litellm.proxy.proxy_server.master_key", "sk-master" - ), patch( - "litellm.proxy.proxy_server.prisma_client", None - ), patch( - "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", - new_callable=AsyncMock, - return_value=mock_oauth2_response, - ) as mock_oauth2: + with patch("litellm.proxy.proxy_server.general_settings", general_settings), \ + patch("litellm.proxy.proxy_server.premium_user", True), \ + patch("litellm.proxy.proxy_server.master_key", "sk-master"), \ + patch("litellm.proxy.proxy_server.prisma_client", None), \ + patch("litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", new_callable=AsyncMock, return_value=mock_oauth2_response) as mock_oauth2: + result = await user_api_key_auth( request=mock_request, api_key=f"Bearer {jwt_like_token}", @@ -731,131 +685,3 @@ async def test_only_oauth2_enabled_handles_all_tokens(self): # OAuth2 should handle it since JWT auth is disabled mock_oauth2.assert_called_once_with(token=jwt_like_token) assert result.user_id == "oauth2-user" - - -@pytest.mark.asyncio -async def test_user_api_key_auth_builder_no_blocking_calls(): - """ - _user_api_key_auth_builder must never call any synchronous DualCache method - (set_cache, get_cache, batch_get_cache, increment_cache, delete_cache) on - the hot auth path — those methods call Redis synchronously and block the - event loop. Only async_* variants are allowed. - """ - from starlette.datastructures import URL - from starlette.requests import Request - - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.auth.user_api_key_auth import _user_api_key_auth_builder - - _blocking_methods = [ - "set_cache", - "get_cache", - "batch_get_cache", - "increment_cache", - "delete_cache", - ] - - api_key = "sk-test-no-blocking-cache" - valid_token = UserAPIKeyAuth( - api_key=api_key, - token=api_key, - user_role=LitellmUserRoles.INTERNAL_USER, - team_id="team-abc", - ) - - mock_cache = AsyncMock() - mock_cache.async_get_cache = AsyncMock(return_value=valid_token) - mock_cache.async_set_cache = AsyncMock(return_value=None) - # Wire sync methods on the instance as plain MagicMocks (no side_effect) so - # calls are recorded but not raised — the function's broad except Exception - # would swallow a raised error. We assert not_called() after the run instead. - for _m in _blocking_methods: - setattr(mock_cache, _m, MagicMock()) - - mock_proxy_logging_obj = MagicMock() - mock_proxy_logging_obj.internal_usage_cache = MagicMock() - mock_proxy_logging_obj.internal_usage_cache.dual_cache = AsyncMock() - mock_proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache = ( - AsyncMock() - ) - mock_proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) - - import litellm.proxy.proxy_server as _proxy_server_mod - - _attrs = { - "prisma_client": MagicMock(), - "user_api_key_cache": mock_cache, - "proxy_logging_obj": mock_proxy_logging_obj, - "master_key": "sk-master-key", - "general_settings": {}, - "llm_model_list": [], - "llm_router": None, - "open_telemetry_logger": None, - "model_max_budget_limiter": MagicMock(), - "user_custom_auth": None, - "jwt_handler": None, - "litellm_proxy_admin_name": "admin", - } - _originals = {k: getattr(_proxy_server_mod, k, None) for k in _attrs} - - try: - for k, v in _attrs.items(): - setattr(_proxy_server_mod, k, v) - - request = Request(scope={"type": "http"}) - request._url = URL(url="/chat/completions") - - import contextlib - - from litellm.caching.dual_cache import DualCache - - blocking_patches = [ - patch.object( - DualCache, - m, - MagicMock( - side_effect=AssertionError( - f"Blocking DualCache.{m}() called on async hot path — use async_{m}() instead" - ) - ), - ) - for m in _blocking_methods - ] - - with contextlib.ExitStack() as stack: - for p in blocking_patches: - stack.enter_context(p) - stack.enter_context( - patch( - "litellm.proxy.auth.user_api_key_auth.get_key_object", - new_callable=AsyncMock, - return_value=valid_token, - ) - ) - stack.enter_context( - patch( - "litellm.proxy.auth.user_api_key_auth.get_team_object", - new_callable=AsyncMock, - return_value=None, - ) - ) - await _user_api_key_auth_builder( - request=request, - api_key=f"Bearer {api_key}", - azure_api_key_header="", - anthropic_api_key_header=None, - google_ai_studio_api_key_header=None, - azure_apim_header=None, - request_data={}, - ) - - for _m in _blocking_methods: - mock = getattr(mock_cache, _m) - assert mock.call_count == 0, ( - f"Blocking DualCache.{_m}() was called {mock.call_count} time(s) " - f"on the async hot path — use async_{_m}() instead" - ) - - finally: - for k, v in _originals.items(): - setattr(_proxy_server_mod, k, v) From eff9e70ed5f34a0f2ff4b9a15f4e22a0a6f4ed15 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Tue, 31 Mar 2026 22:46:42 +0300 Subject: [PATCH 24/28] Revert "revert: remove unrelated changes from HttpOnly cookie fix branch" This reverts commit 0684a1e27521ada35cf8a4afbdff7aeaa87ff41d. --- litellm/integrations/prometheus.py | 420 +++++++++++++++++- .../bedrock/chat/converse_transformation.py | 68 ++- .../chat/guardrail_translation/handler.py | 43 +- litellm/proxy/auth/auth_checks.py | 302 ++++++++----- .../proxy/common_utils/reset_budget_job.py | 86 +++- .../proxy/management_helpers/audit_logs.py | 128 +++++- .../proxy/auth/test_user_api_key_auth.py | 342 ++++++++++---- 7 files changed, 1125 insertions(+), 264 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 357e0229fc6a..5d1ff8329301 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -65,6 +65,17 @@ def _get_cached_end_user_id_for_cost_tracking(): class PrometheusLogger(CustomLogger): # Class variables or attributes + + @staticmethod + def get_instance() -> Optional["PrometheusLogger"]: + """Find the PrometheusLogger instance from litellm.callbacks, if registered.""" + import litellm + + for cb in litellm.callbacks: + if isinstance(cb, PrometheusLogger): + return cb + return None + def __init__( # noqa: PLR0915 self, **kwargs, @@ -180,6 +191,31 @@ def __init__( # noqa: PLR0915 ), ) + # Remaining Budget for Org + self.litellm_remaining_org_budget_metric = self._gauge_factory( + "litellm_remaining_org_budget_metric", + "Remaining budget for org", + labelnames=self.get_labels_for_metric( + "litellm_remaining_org_budget_metric" + ), + ) + + # Max Budget for Org + self.litellm_org_max_budget_metric = self._gauge_factory( + "litellm_org_max_budget_metric", + "Maximum budget set for org", + labelnames=self.get_labels_for_metric("litellm_org_max_budget_metric"), + ) + + # Org Budget Reset At + self.litellm_org_budget_remaining_hours_metric = self._gauge_factory( + "litellm_org_budget_remaining_hours_metric", + "Remaining hours for org budget to be reset", + labelnames=self.get_labels_for_metric( + "litellm_org_budget_remaining_hours_metric" + ), + ) + # Remaining Budget for API Key self.litellm_remaining_api_key_budget_metric = self._gauge_factory( "litellm_remaining_api_key_budget_metric", @@ -440,6 +476,76 @@ def __init__( # noqa: PLR0915 labelnames=[], ) + ######################################## + # Managed Batch Metrics + ######################################## + self.litellm_managed_batch_created_total = self._counter_factory( + name="litellm_managed_batch_created_total", + documentation="Total number of managed batches created", + labelnames=[ + "model", + "api_provider", + "user", + "user_email", + "api_key_alias", + ], + ) + + self.litellm_managed_file_size_bytes = self._gauge_factory( + "litellm_managed_file_size_bytes", + "Size of the most recent managed batch file in bytes (last-seen value per label combination)", + labelnames=["purpose", "file_type", "model", "api_provider", "user"], + ) + + self.litellm_managed_batch_duration_seconds = self._histogram_factory( + "litellm_managed_batch_duration_seconds", + "Duration of completed managed batches in seconds (completed_at - created_at)", + labelnames=["model", "api_provider"], + buckets=BATCH_DURATION_BUCKETS, + ) + + self.litellm_managed_file_created_total = self._counter_factory( + name="litellm_managed_file_created_total", + documentation="Total number of managed files created", + labelnames=[ + "model", + "api_provider", + "user", + "user_email", + "api_key_alias", + ], + ) + + self.litellm_managed_file_deleted_total = self._counter_factory( + name="litellm_managed_file_deleted_total", + documentation="Total number of managed file deletions (success or blocked)", + labelnames=["result"], + ) + + self.litellm_check_batch_cost_jobs_polled = self._gauge_factory( + "litellm_check_batch_cost_jobs_polled", + "Number of unprocessed batches found by the last CheckBatchCost poll", + labelnames=[], + ) + + self.litellm_check_batch_cost_jobs_processed_total = self._counter_factory( + name="litellm_check_batch_cost_jobs_processed_total", + documentation="Total number of batches successfully cost-tracked by CheckBatchCost", + labelnames=["model", "api_provider"], + ) + + self.litellm_check_batch_cost_errors_total = self._counter_factory( + name="litellm_check_batch_cost_errors_total", + documentation="Total number of errors in CheckBatchCost by error type", + labelnames=["error_type"], + ) + + self.litellm_check_batch_cost_last_run_timestamp = self._gauge_factory( + "litellm_check_batch_cost_last_run_timestamp", + "Unix timestamp of the last CheckBatchCost job run", + labelnames=[], + ) + except Exception as e: print_verbose(f"Got exception on init prometheus client {str(e)}") raise e @@ -922,6 +1028,9 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti user_api_team_alias = standard_logging_payload["metadata"][ "user_api_key_team_alias" ] + user_api_key_org_id = standard_logging_payload["metadata"].get( + "user_api_key_org_id" + ) output_tokens = standard_logging_payload["completion_tokens"] tokens_used = standard_logging_payload["total_tokens"] response_cost = standard_logging_payload["response_cost"] @@ -931,10 +1040,14 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[ "metadata" ].get("user_api_key_auth_metadata") + spend_logs_metadata: Optional[dict] = standard_logging_payload["metadata"].get( + "spend_logs_metadata" + ) combined_metadata: Dict[str, Any] = { **(_requester_metadata if _requester_metadata else {}), **(user_api_key_auth_metadata if user_api_key_auth_metadata else {}), + **(spend_logs_metadata if spend_logs_metadata else {}), } if standard_logging_payload is not None and isinstance( standard_logging_payload, dict @@ -974,9 +1087,11 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti ), client_ip=standard_logging_payload["metadata"].get("requester_ip_address"), user_agent=standard_logging_payload["metadata"].get("user_agent"), - stream=str(standard_logging_payload.get("stream")) - if litellm.prometheus_emit_stream_label - else None, + stream=( + str(standard_logging_payload.get("stream")) + if litellm.prometheus_emit_stream_label + else None + ), ) if ( @@ -1026,6 +1141,7 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti litellm_params=litellm_params, response_cost=response_cost, user_id=user_id, + user_api_key_org_id=user_api_key_org_id, ) # set proxy virtual key rpm/tpm metrics @@ -1181,6 +1297,7 @@ async def _increment_remaining_budget_metrics( litellm_params: dict, response_cost: float, user_id: Optional[str] = None, + user_api_key_org_id: Optional[str] = None, ): _metadata = litellm_params.get("metadata") or {} _team_spend = _metadata.get("user_api_key_team_spend", None) @@ -1213,12 +1330,16 @@ async def _increment_remaining_budget_metrics( user_max_budget=_user_max_budget, response_cost=response_cost, ), + self._set_org_budget_metrics_after_api_request( + org_id=user_api_key_org_id, + response_cost=response_cost, + ), return_exceptions=True, ) for i, r in enumerate(results): if isinstance(r, Exception): verbose_logger.debug( - f"[Non-Blocking] Prometheus: Budget metric lookup {['key', 'team', 'user'][i]} failed: {r}" + f"[Non-Blocking] Prometheus: Budget metric lookup {['key', 'team', 'user', 'org'][i]} failed: {r}" ) def _increment_top_level_request_and_spend_metrics( @@ -1407,6 +1528,9 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti user_api_team_alias = standard_logging_payload["metadata"][ "user_api_key_team_alias" ] + user_api_key_org_id = standard_logging_payload["metadata"].get( + "user_api_key_org_id" + ) try: self.litellm_llm_api_failed_requests_metric.labels( @@ -1422,6 +1546,10 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti ), ).inc() self.set_llm_deployment_failure_metrics(kwargs) + await self._set_org_budget_metrics_after_api_request( + org_id=user_api_key_org_id, + response_cost=0, + ) except Exception as e: verbose_logger.exception( "prometheus Layer Error(): Exception occured - {}".format(str(e)) @@ -1629,9 +1757,11 @@ async def async_post_call_failure_hook( client_ip=_metadata.get("requester_ip_address"), user_agent=_metadata.get("user_agent"), model_id=model_id, - stream=str(request_data.get("stream")) - if litellm.prometheus_emit_stream_label - else None, + stream=( + str(request_data.get("stream")) + if litellm.prometheus_emit_stream_label + else None + ), ) _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric( @@ -2154,6 +2284,127 @@ def _record_guardrail_metrics( except Exception as e: verbose_logger.debug(f"Error recording guardrail metrics: {str(e)}") + ######################################## + # Managed Batch Metric Recording Methods + ######################################## + + def record_managed_batch_created( + self, + model: Optional[str], + api_provider: Optional[str], + user: Optional[str], + user_email: Optional[str], + api_key_alias: Optional[str], + ): + try: + self.litellm_managed_batch_created_total.labels( + model=model, + api_provider=api_provider, + user=user, + user_email=user_email, + api_key_alias=api_key_alias, + ).inc() + except Exception as e: + verbose_logger.warning(f"Error recording batch created metric: {e}") + + def record_managed_file_size( + self, + size_bytes: int, + purpose: str, + file_type: str, + model: Optional[str] = None, + api_provider: Optional[str] = None, + user: Optional[str] = None, + ): + """Record the size of a managed file. Uses a gauge (last-seen value per label combination).""" + try: + self.litellm_managed_file_size_bytes.labels( + purpose=purpose, + file_type=file_type, + model=model or "", + api_provider=api_provider or "", + user=user or "", + ).set(size_bytes) + except Exception as e: + verbose_logger.warning(f"Error recording file size metric: {e}") + + def record_managed_batch_duration( + self, + duration_seconds: float, + model: Optional[str] = None, + api_provider: Optional[str] = None, + ): + try: + self.litellm_managed_batch_duration_seconds.labels( + model=model or "", + api_provider=api_provider or "", + ).observe(duration_seconds) + except Exception as e: + verbose_logger.warning(f"Error recording batch duration metric: {e}") + + def record_managed_file_created( + self, + model: Optional[str], + api_provider: Optional[str], + user: Optional[str], + user_email: Optional[str], + api_key_alias: Optional[str], + ): + try: + self.litellm_managed_file_created_total.labels( + model=model, + api_provider=api_provider, + user=user, + user_email=user_email, + api_key_alias=api_key_alias, + ).inc() + except Exception as e: + verbose_logger.warning(f"Error recording file created metric: {e}") + + def record_managed_file_deleted(self, result: str): + """Record a managed file deletion attempt. result is 'success' or 'blocked'.""" + try: + self.litellm_managed_file_deleted_total.labels(result=result).inc() + except Exception as e: + verbose_logger.warning(f"Error recording file deleted metric: {e}") + + def record_check_batch_cost_run( + self, + jobs_polled: int, + processed_models: Optional[List[Tuple[Optional[str], Optional[str]]]] = None, + ): + """ + Record CheckBatchCost polling metrics. + + Args: + jobs_polled: Number of unprocessed batches found + processed_models: List of (model, api_provider) tuples for processed jobs + """ + import time + + try: + self.litellm_check_batch_cost_last_run_timestamp.set(time.time()) + self.litellm_check_batch_cost_jobs_polled.set(jobs_polled) + + if processed_models: + for model, api_provider in processed_models: + self.litellm_check_batch_cost_jobs_processed_total.labels( + model=model or "", + api_provider=api_provider or "", + ).inc() + except Exception as e: + verbose_logger.warning(f"Error recording check batch cost metrics: {e}") + + def record_check_batch_cost_error(self, error_type: str): + try: + self.litellm_check_batch_cost_errors_total.labels( + error_type=error_type, + ).inc() + except Exception as e: + verbose_logger.warning( + f"Error recording check batch cost error metric: {e}" + ) + @staticmethod def _get_exception_class_name(exception: Exception) -> str: exception_class_name = "" @@ -2530,6 +2781,35 @@ async def fetch_users( data_type="users", ) + async def _initialize_org_budget_metrics(self): + """ + Initialize org budget metrics by reusing the generic pagination logic. + """ + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + verbose_logger.debug( + "Prometheus: skipping org metrics initialization, DB not initialized" + ) + return + + async def fetch_orgs(page_size: int, page: int) -> Tuple[list, Optional[int]]: + skip = (page - 1) * page_size + orgs = await prisma_client.db.litellm_organizationtable.find_many( + skip=skip, + take=page_size, + order={"created_at": "desc"}, + include={"litellm_budget_table": True}, + ) + total_count = await prisma_client.db.litellm_organizationtable.count() + return orgs, total_count + + await self._initialize_budget_metrics( + data_fetch_function=fetch_orgs, + set_metrics_function=self._set_org_list_budget_metrics, + data_type="orgs", + ) + async def initialize_remaining_budget_metrics(self): """ Handler for initializing remaining budget metrics for all teams to avoid metric discrepancies. @@ -2564,10 +2844,11 @@ async def _initialize_remaining_budget_metrics(self): """ Helper to initialize remaining budget metrics for all teams, API keys, and users. """ - verbose_logger.debug("Emitting key, team, user budget metrics....") + verbose_logger.debug("Emitting key, team, user, org budget metrics....") await self._initialize_team_budget_metrics() await self._initialize_api_key_budget_metrics() await self._initialize_user_budget_metrics() + await self._initialize_org_budget_metrics() await self._initialize_user_and_team_count_metrics() async def _initialize_user_and_team_count_metrics(self): @@ -2623,6 +2904,22 @@ async def _set_user_list_budget_metrics(self, users: List[LiteLLM_UserTable]): for user in users: self._set_user_budget_metrics(user) + async def _set_org_list_budget_metrics(self, orgs: list): + """Helper function to set budget metrics for a list of orgs""" + for org in orgs: + budget_table = getattr(org, "litellm_budget_table", None) + self._set_org_budget_metrics( + org_id=org.organization_id or "", + org_alias=org.organization_alias or "", + spend=org.spend or 0.0, + max_budget=budget_table.max_budget if budget_table else None, + budget_reset_at=( + getattr(budget_table, "budget_reset_at", None) + if budget_table + else None + ), + ) + async def _set_team_budget_metrics_after_api_request( self, user_api_team: Optional[str], @@ -2744,6 +3041,113 @@ def _set_team_budget_metrics( ) ) + async def _set_org_budget_metrics_after_api_request( + self, + org_id: Optional[str], + response_cost: float, + ): + """ + Set org budget metrics after an LLM API request + + - Fetches org info via cache (get_org_object) + - Sets org budget metrics + """ + if not org_id: + return + + from litellm.proxy.auth.auth_checks import get_org_object + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + if prisma_client is None: + return + + try: + org_info = await get_org_object( + org_id=org_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + include_budget_table=True, + ) + except Exception as e: + verbose_logger.debug( + f"[Non-Blocking] Prometheus: Error getting org info: {str(e)}" + ) + return + + if org_info is None: + return + + org_alias = org_info.organization_alias or "" + _total_org_spend = (org_info.spend or 0.0) + response_cost + budget_table = org_info.litellm_budget_table + max_budget = budget_table.max_budget if budget_table else None + budget_reset_at = ( + getattr(budget_table, "budget_reset_at", None) if budget_table else None + ) + + self._set_org_budget_metrics( + org_id=org_id, + org_alias=org_alias, + spend=_total_org_spend, + max_budget=max_budget, + budget_reset_at=budget_reset_at, + ) + + def _set_org_budget_metrics( + self, + org_id: str, + org_alias: str, + spend: float, + max_budget: Optional[float], + budget_reset_at: Optional[datetime], + ): + """ + Set org budget metrics for a single org + + - Remaining Budget + - Max Budget + - Budget Reset At + """ + enum_values = UserAPIKeyLabelValues( + org_id=org_id, + org_alias=org_alias, + ) + + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_remaining_org_budget_metric" + ), + enum_values=enum_values, + ) + self.litellm_remaining_org_budget_metric.labels(**_labels).set( + self._safe_get_remaining_budget( + max_budget=max_budget, + spend=spend, + ) + ) + + if max_budget is not None: + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_org_max_budget_metric" + ), + enum_values=enum_values, + ) + self.litellm_org_max_budget_metric.labels(**_labels).set(max_budget) + + if budget_reset_at is not None: + _labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + metric_name="litellm_org_budget_remaining_hours_metric" + ), + enum_values=enum_values, + ) + self.litellm_org_budget_remaining_hours_metric.labels(**_labels).set( + self._get_remaining_hours_for_budget_reset( + budget_reset_at=budget_reset_at + ) + ) + def _set_key_budget_metrics(self, user_api_key_dict: UserAPIKeyAuth): """ Set virtual key budget metrics diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 229457a73b48..5cfb00d69b61 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -91,34 +91,6 @@ "compact-2026-01-12", # The compact beta feature is not currently supported on the Converse and ConverseStream APIs ] -# Models that support Bedrock's native structured outputs API (outputConfig.textFormat) -# Uses substring matching against the Bedrock model ID -# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html -BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS = { - # Anthropic Claude 4.5+ - "claude-haiku-4-5", - "claude-sonnet-4-5", - "claude-opus-4-5", - "claude-opus-4-6", - # Qwen3 - "qwen3", - # DeepSeek - "deepseek-v3.1", - # Gemma 3 - "gemma-3", - # MiniMax - "minimax-m2", - # Mistral (magistral-small excluded: broken constrained decoding on Bedrock) - "ministral", - "mistral-large-3", - "voxtral", - # Moonshot - "kimi-k2", - # NVIDIA - "nemotron-nano", - # OpenAI (gpt-oss excluded: broken constrained decoding, works via tool-call fallback) -} - class AmazonConverseConfig(BaseConfig): """ @@ -493,8 +465,7 @@ def _clamp_thinking_budget_tokens(optional_params: dict) -> None: budget = thinking.get("budget_tokens") if isinstance(budget, int) and budget < BEDROCK_MIN_THINKING_BUDGET_TOKENS: verbose_logger.debug( - "Bedrock requires thinking.budget_tokens >= %d, got %d. " - "Clamping to minimum.", + "Bedrock requires thinking.budget_tokens >= %d, got %d. Clamping to minimum.", BEDROCK_MIN_THINKING_BUDGET_TOKENS, budget, ) @@ -763,10 +734,20 @@ def _create_json_tool_call_for_response_format( return _tool @staticmethod - def _supports_native_structured_outputs(model: str) -> bool: - """Check if the Bedrock model supports native structured outputs (outputConfig.textFormat).""" - return any( - substring in model for substring in BEDROCK_NATIVE_STRUCTURED_OUTPUT_MODELS + def _supports_native_structured_outputs( + model: str, custom_llm_provider: Optional[str] = None + ) -> bool: + """Check if the Bedrock model supports native structured outputs (outputConfig.textFormat). + + Delegates to the standard ``supports_native_structured_output`` utility + which looks up the flag in ``litellm.model_cost`` via + ``_get_model_info_helper``. + Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html + """ + from litellm.utils import supports_native_structured_output + + return supports_native_structured_output( + model=model, custom_llm_provider=custom_llm_provider ) @staticmethod @@ -913,7 +894,9 @@ def map_openai_params( ) if param == "tool_choice": _tool_choice_value = self.map_tool_choice_values( - model=model, tool_choice=value, drop_params=drop_params # type: ignore + model=model, + tool_choice=value, + drop_params=drop_params, # type: ignore ) if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value @@ -1006,7 +989,10 @@ def _translate_response_format_param( if "type" in value and value["type"] == "text": return optional_params - if self._supports_native_structured_outputs(model) and json_schema is not None: + if ( + self._supports_native_structured_outputs(model, self.custom_llm_provider) + and json_schema is not None + ): # Use Bedrock's native structured outputs API (outputConfig.textFormat) # No synthetic tool injection, no fake_stream needed. # Requires an explicit schema — json_object with no schema falls through @@ -1446,6 +1432,16 @@ def _transform_request_helper( original_tools, model, headers, additional_request_params ) + # Append cachePoint to tools if cache_control_injection_points has tool_config + cache_injection_points = additional_request_params.pop( + "cache_control_injection_points", None + ) + if cache_injection_points and len(bedrock_tools) > 0: + for point in cache_injection_points: + if point.get("location") == "tool_config": + bedrock_tools.append({"cachePoint": {"type": "default"}}) + break + bedrock_tool_config: Optional[ToolConfigBlock] = None if len(bedrock_tools) > 0: tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index bab4c3b5eb75..f854cdb13d00 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -260,6 +260,7 @@ async def process_output_response( guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: Optional[Any] = None, user_api_key_dict: Optional[Any] = None, + request_data: Optional[dict] = None, ) -> Any: """ Process output response by applying guardrails to text content. @@ -308,15 +309,21 @@ async def process_output_response( # Step 2: Apply guardrail to all texts and tool calls in batch if texts_to_check or tool_calls_to_check: - # Create a request_data dict with response info and user API key metadata - request_data: dict = {"response": response} + # Use the real request_data if provided (proxy path), otherwise + # create a standalone dict (SDK / direct-call path). + if request_data is None: + request_data = {"response": response} + else: + if "response" not in request_data: + request_data["response"] = response # Add user API key metadata with prefixed keys - user_metadata = self.transform_user_api_key_dict_to_metadata( - user_api_key_dict - ) - if user_metadata: - request_data["litellm_metadata"] = user_metadata + if "litellm_metadata" not in request_data: + user_metadata = self.transform_user_api_key_dict_to_metadata( + user_api_key_dict + ) + if user_metadata: + request_data["litellm_metadata"] = user_metadata inputs = GenericGuardrailAPIInputs(texts=texts_to_check) if images_to_check: @@ -364,6 +371,7 @@ async def process_output_streaming_response( guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: Optional[Any] = None, user_api_key_dict: Optional[Any] = None, + request_data: Optional[dict] = None, ) -> List["ModelResponseStream"]: """ Process output streaming responses by applying guardrails to text content. @@ -402,6 +410,7 @@ async def process_output_streaming_response( guardrail_to_apply=guardrail_to_apply, litellm_logging_obj=litellm_logging_obj, user_api_key_dict=user_api_key_dict, + request_data=request_data, ) return responses_so_far @@ -436,15 +445,21 @@ async def process_output_streaming_response( # Step 3: Apply guardrail to all combined texts in batch if texts_to_check: - # Create a request_data dict with response info and user API key metadata - request_data: dict = {"responses": responses_so_far} + # Use the real request_data if provided (proxy path), otherwise + # create a standalone dict (SDK / direct-call path). + if request_data is None: + request_data = {"responses": responses_so_far} + else: + if "responses" not in request_data: + request_data["responses"] = responses_so_far # Add user API key metadata with prefixed keys - user_metadata = self.transform_user_api_key_dict_to_metadata( - user_api_key_dict - ) - if user_metadata: - request_data["litellm_metadata"] = user_metadata + if "litellm_metadata" not in request_data: + user_metadata = self.transform_user_api_key_dict_to_metadata( + user_api_key_dict + ) + if user_metadata: + request_data["litellm_metadata"] = user_metadata inputs = GenericGuardrailAPIInputs(texts=texts_to_check) if images_to_check: diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index b4c123b361b9..ed600b26e832 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -8,6 +8,7 @@ 2. If user is in budget 3. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget """ + import asyncio import re import time @@ -29,6 +30,7 @@ DEFAULT_MAX_RECURSE_DEPTH, EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE, ) +from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.proxy._types import ( RBAC_ROLES, @@ -38,6 +40,7 @@ LiteLLM_EndUserTable, Litellm_EntityType, LiteLLM_JWTAuth, + LiteLLM_ManagedVectorStoresTable, LiteLLM_ObjectPermissionTable, LiteLLM_OrganizationMembershipTable, LiteLLM_OrganizationTable, @@ -402,23 +405,26 @@ async def common_checks( # noqa: PLR0915 # 1. If team is blocked if team_object is not None and team_object.blocked is True: raise Exception( - f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if your admin." + f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if you're an admin." ) # 2. If team can call model if _model and team_object: - if not await can_team_access_model( - model=_model, - team_object=team_object, - llm_router=llm_router, - team_model_aliases=valid_token.team_model_aliases if valid_token else None, - ): - raise ProxyException( - message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", - type=ProxyErrorTypes.team_model_access_denied, - param="model", - code=status.HTTP_401_UNAUTHORIZED, - ) + with tracer.trace("litellm.proxy.auth.common_checks.can_team_access_model"): + if not await can_team_access_model( + model=_model, + team_object=team_object, + llm_router=llm_router, + team_model_aliases=( + valid_token.team_model_aliases if valid_token else None + ), + ): + raise ProxyException( + message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", + type=ProxyErrorTypes.team_model_access_denied, + param="model", + code=status.HTTP_401_UNAUTHORIZED, + ) # Require trace id for agent keys when agent has require_trace_id_on_calls_by_agent if valid_token is not None and valid_token.agent_id: @@ -443,54 +449,62 @@ async def common_checks( # noqa: PLR0915 ## 2.1 If user can call model (if personal key) if _model and team_object is None and user_object is not None: - await can_user_call_model( - model=_model, - llm_router=llm_router, - user_object=user_object, - ) + with tracer.trace("litellm.proxy.auth.common_checks.can_user_call_model"): + await can_user_call_model( + model=_model, + llm_router=llm_router, + user_object=user_object, + ) # 1.1 - 2.2 - 3.0.2 - 3.0.3: Project checks (blocked, model access, budget) - await _run_project_checks( - project_object=project_object, - _model=_model, - llm_router=llm_router, - skip_budget_checks=skip_budget_checks, - valid_token=valid_token, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.common_checks.run_project_checks"): + await _run_project_checks( + project_object=project_object, + _model=_model, + llm_router=llm_router, + skip_budget_checks=skip_budget_checks, + valid_token=valid_token, + proxy_logging_obj=proxy_logging_obj, + ) # If this is a free model, skip all budget checks if not skip_budget_checks: # 3. If team is in budget - await _team_max_budget_check( - team_object=team_object, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.team_max_budget_check"): + await _team_max_budget_check( + team_object=team_object, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 3.0.5. If team is over soft budget (alert only, doesn't block) - await _team_soft_budget_check( - team_object=team_object, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.team_soft_budget_check"): + await _team_soft_budget_check( + team_object=team_object, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 3.1. If organization is in budget - await _organization_max_budget_check( - valid_token=valid_token, - team_object=team_object, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace( + "litellm.proxy.auth.common_checks.organization_max_budget_check" + ): + await _organization_max_budget_check( + valid_token=valid_token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) - await _tag_max_budget_check( - request_body=request_body, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.tag_max_budget_check"): + await _tag_max_budget_check( + request_body=request_body, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) # 4. If user is in budget ## 4.1 check personal budget, if personal key @@ -508,14 +522,15 @@ async def common_checks( # noqa: PLR0915 ) ## 4.2 check team member budget, if team key - await _check_team_member_budget( - team_object=team_object, - user_object=user_object, - valid_token=valid_token, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + with tracer.trace("litellm.proxy.auth.common_checks.check_team_member_budget"): + await _check_team_member_budget( + team_object=team_object, + user_object=user_object, + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) # 5. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget if ( @@ -554,19 +569,21 @@ async def common_checks( # noqa: PLR0915 ) # 11. [OPTIONAL] Vector store checks - is the object allowed to access the vector store - await vector_store_access_check( - request_body=request_body, - team_object=team_object, - valid_token=valid_token, - ) + with tracer.trace("litellm.proxy.auth.common_checks.vector_store_access_check"): + await vector_store_access_check( + request_body=request_body, + team_object=team_object, + valid_token=valid_token, + ) # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path) - await check_tools_allowlist( - request_body=request_body, - valid_token=valid_token, - team_object=team_object, - route=route, - ) + with tracer.trace("litellm.proxy.auth.common_checks.check_tools_allowlist"): + await check_tools_allowlist( + request_body=request_body, + valid_token=valid_token, + team_object=team_object, + route=route, + ) return True @@ -2279,6 +2296,71 @@ async def get_object_permission( return None +@log_db_metrics +async def get_managed_vector_store_rows_by_uuids( + uuids: List[str], + prisma_client: Optional[PrismaClient], + user_api_key_cache: DualCache, + parent_otel_span: Optional[Span] = None, + proxy_logging_obj: Optional[ProxyLogging] = None, +) -> List[LiteLLM_ManagedVectorStoresTable]: + """ + Fetch managed vector store rows by their internal UUIDs. + + Follows the get_team_object / get_key_object / get_object_permission pattern: + cache-first lookup (in-memory / Redis), DB fallback only on cache miss. + Critical-path DB access must go through this helper to avoid raw Prisma + calls on the hot request path. + """ + if not uuids or prisma_client is None: + return [] + + result: List[LiteLLM_ManagedVectorStoresTable] = [] + cache_misses: List[str] = [] + + for uuid in uuids: + key = "managed_vector_store_id:{}".format(uuid) + cached = await user_api_key_cache.async_get_cache(key=key) + if cached is not None: + if isinstance(cached, dict): + result.append(LiteLLM_ManagedVectorStoresTable(**cached)) + elif isinstance(cached, LiteLLM_ManagedVectorStoresTable): + result.append(cached) + else: + cache_misses.append(uuid) + else: + cache_misses.append(uuid) + + if not cache_misses: + return result + + rows = await prisma_client.db.litellm_managedvectorstorestable.find_many( + where={"vector_store_id": {"in": cache_misses}}, + take=len(cache_misses), + ) + + for row in rows: + row_dict = ( + row.model_dump() + if hasattr(row, "model_dump") + else (row.dict() if hasattr(row, "dict") else None) + ) + if not isinstance(row_dict, dict) or not row_dict: + row_dict = dict(row) if hasattr(row, "__dict__") else {} + if not row_dict: + continue + cached_obj = LiteLLM_ManagedVectorStoresTable(**row_dict) + key = "managed_vector_store_id:{}".format(cached_obj.vector_store_id) + await user_api_key_cache.async_set_cache( + key=key, + value=row_dict, + ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, + ) + result.append(cached_obj) + + return result + + @log_db_metrics async def get_org_object( org_id: str, @@ -2795,7 +2877,15 @@ async def _virtual_key_max_budget_check( Triggers a budget alert if the token is over it's max budget. """ - if valid_token.spend is not None and valid_token.max_budget is not None: + if valid_token.max_budget is not None: + from litellm.proxy.proxy_server import get_current_spend + + # Read spend from cross-pod counter (Redis-first) or cached object (fallback) + spend = await get_current_spend( + counter_key=f"spend:key:{valid_token.token}", + fallback_spend=valid_token.spend or 0.0, + ) + #################################### # collect information for alerting # #################################### @@ -2807,7 +2897,7 @@ async def _virtual_key_max_budget_check( call_info = CallInfo( token=valid_token.token, - spend=valid_token.spend, + spend=spend, max_budget=valid_token.max_budget, soft_budget=valid_token.soft_budget, user_id=valid_token.user_id, @@ -2828,9 +2918,9 @@ async def _virtual_key_max_budget_check( # collect information for alerting # #################################### - if valid_token.spend >= valid_token.max_budget: + if spend >= valid_token.max_budget: raise litellm.BudgetExceededError( - current_cost=valid_token.spend, + current_cost=spend, max_budget=valid_token.max_budget, ) @@ -2961,6 +3051,14 @@ async def _check_team_member_budget( team_member_budget = team_membership.litellm_budget_table.max_budget team_member_spend = team_membership.spend or 0.0 + # Read from cross-pod counter (Redis-first) if available + from litellm.proxy.proxy_server import get_current_spend + + team_member_spend = await get_current_spend( + counter_key=f"spend:team_member:{valid_token.user_id}:{team_object.team_id}", + fallback_spend=team_member_spend, + ) + if team_member_spend >= team_member_budget: raise litellm.BudgetExceededError( current_cost=team_member_spend, @@ -2981,36 +3079,40 @@ async def _team_max_budget_check( BudgetExceededError if the team is over it's max budget. Triggers a budget alert if the team is over it's max budget. """ - if ( - team_object is not None - and team_object.max_budget is not None - and team_object.spend is not None - and team_object.spend > team_object.max_budget - ): - if valid_token: - call_info = CallInfo( - token=valid_token.token, - spend=team_object.spend, - max_budget=team_object.max_budget, - user_id=valid_token.user_id, - team_id=valid_token.team_id, - team_alias=valid_token.team_alias, - organization_id=valid_token.org_id, - event_group=Litellm_EntityType.TEAM, - ) - asyncio.create_task( - proxy_logging_obj.budget_alerts( - type="team_budget", - user_info=call_info, - ) - ) + if team_object is not None and team_object.max_budget is not None: + from litellm.proxy.proxy_server import get_current_spend - raise litellm.BudgetExceededError( - current_cost=team_object.spend, - max_budget=team_object.max_budget, - message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {team_object.spend}, Max budget: {team_object.max_budget}", + # Read spend from cross-pod counter (Redis-first) or cached object (fallback) + spend = await get_current_spend( + counter_key=f"spend:team:{team_object.team_id}", + fallback_spend=team_object.spend or 0.0, ) + if spend > team_object.max_budget: + if valid_token: + call_info = CallInfo( + token=valid_token.token, + spend=spend, + max_budget=team_object.max_budget, + user_id=valid_token.user_id, + team_id=valid_token.team_id, + team_alias=valid_token.team_alias, + organization_id=valid_token.org_id, + event_group=Litellm_EntityType.TEAM, + ) + asyncio.create_task( + proxy_logging_obj.budget_alerts( + type="team_budget", + user_info=call_info, + ) + ) + + raise litellm.BudgetExceededError( + current_cost=spend, + max_budget=team_object.max_budget, + message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}", + ) + async def _team_soft_budget_check( team_object: Optional[LiteLLM_TeamTable], diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 674214b19e53..bcfaed243981 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -54,16 +54,47 @@ async def reset_budget_for_litellm_team_members( """ Resets the budget for all LiteLLM Team Members if their budget has expired """ + budget_ids = [ + budget.budget_id + for budget in budgets_to_reset + if budget.budget_id is not None + ] + + # Reset spend counters for affected team members. + # Reset Redis directly so a transient failure doesn't leave stale + # counters that get_current_spend would read as authoritative. + try: + from litellm.proxy.proxy_server import spend_counter_cache + + memberships = await self.prisma_client.db.litellm_teammembership.find_many( + where={"budget_id": {"in": budget_ids}} + ) + for m in memberships: + counter_key = f"spend:team_member:{m.user_id}:{m.team_id}" + # Always reset in-memory + spend_counter_cache.in_memory_cache.set_cache( + key=counter_key, value=0.0 + ) + # Explicitly reset Redis with warning on failure + if spend_counter_cache.redis_cache is not None: + try: + await spend_counter_cache.redis_cache.async_set_cache( + key=counter_key, value=0.0 + ) + except Exception as redis_err: + verbose_proxy_logger.warning( + "Failed to reset team member spend counter in Redis %s: %s. " + "Budget may be over-enforced until counter expires.", + counter_key, + redis_err, + ) + except Exception as e: + verbose_proxy_logger.warning( + "Failed to reset team member spend counters: %s", e + ) + return await self.prisma_client.db.litellm_teammembership.update_many( - where={ - "budget_id": { - "in": [ - budget.budget_id - for budget in budgets_to_reset - if budget.budget_id is not None - ] - } - }, + where={"budget_id": {"in": budget_ids}}, data={ "spend": 0, }, @@ -531,6 +562,43 @@ async def _reset_budget_common( """ try: item.spend = 0.0 + + # Reset the cross-pod spend counter. + # Reset Redis directly (not via DualCache) so a Redis failure + # doesn't silently leave a stale counter that get_current_spend + # would read as authoritative, permanently blocking the user. + from litellm.proxy.proxy_server import spend_counter_cache + + counter_key = None + if item_type == "key" and hasattr(item, "token") and item.token is not None: + counter_key = f"spend:key:{item.token}" + elif ( + item_type == "team" + and hasattr(item, "team_id") + and item.team_id is not None + ): + counter_key = f"spend:team:{item.team_id}" + + if counter_key is not None: + # Always reset in-memory (local fallback) + spend_counter_cache.in_memory_cache.set_cache( + key=counter_key, value=0.0 + ) + # Explicitly reset Redis with warning on failure + if spend_counter_cache.redis_cache is not None: + try: + await spend_counter_cache.redis_cache.async_set_cache( + key=counter_key, value=0.0 + ) + except Exception as redis_err: + verbose_proxy_logger.warning( + "Failed to reset spend counter in Redis for %s key=%s: %s. " + "Budget may be over-enforced until counter expires.", + item_type, + counter_key, + redis_err, + ) + if hasattr(item, "budget_duration") and item.budget_duration is not None: # Get standardized reset time based on budget duration from litellm.proxy.common_utils.timezone_utils import ( diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index ea082f468aee..7599e11bdef0 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -2,12 +2,15 @@ Functions to create audit logs for LiteLLM Proxy """ +import asyncio import json -from litellm._uuid import uuid from datetime import datetime, timezone +from typing import Dict import litellm from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid +from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ( AUDIT_ACTIONS, LiteLLM_AuditLogs, @@ -15,6 +18,99 @@ Optional, UserAPIKeyAuth, ) +from litellm.types.utils import StandardAuditLogPayload + +_audit_log_callback_cache: Dict[str, CustomLogger] = {} + + +def _resolve_audit_log_callback(name: str) -> Optional[CustomLogger]: + """Resolve a string callback name to a CustomLogger instance, with caching.""" + if name in _audit_log_callback_cache: + return _audit_log_callback_cache[name] + + from litellm.litellm_core_utils.litellm_logging import ( + _init_custom_logger_compatible_class, + ) + + instance = _init_custom_logger_compatible_class( + logging_integration=name, # type: ignore + internal_usage_cache=None, + llm_router=None, + ) + + if instance is not None: + _audit_log_callback_cache[name] = instance + return instance + + +def _build_audit_log_payload( + request_data: LiteLLM_AuditLogs, +) -> StandardAuditLogPayload: + """Convert LiteLLM_AuditLogs to StandardAuditLogPayload for callback dispatch.""" + updated_at = "" + if request_data.updated_at is not None: + updated_at = request_data.updated_at.isoformat() + + table_name_str: str = ( + request_data.table_name.value + if isinstance(request_data.table_name, LitellmTableNames) + else str(request_data.table_name) + ) + + return StandardAuditLogPayload( + id=request_data.id, + updated_at=updated_at, + changed_by=request_data.changed_by or "", + changed_by_api_key=request_data.changed_by_api_key or "", + action=request_data.action, + table_name=table_name_str, + object_id=request_data.object_id, + before_value=request_data.before_value, + updated_values=request_data.updated_values, + ) + + +def _audit_log_task_done_callback(task: asyncio.Task) -> None: + """Log exceptions from audit log callback tasks so they don't slip through silently.""" + try: + exc = task.exception() + except asyncio.CancelledError: + return + if exc is not None: + verbose_proxy_logger.error( + "Audit log callback task failed: %s", exc, exc_info=exc + ) + + +async def _dispatch_audit_log_to_callbacks( + request_data: LiteLLM_AuditLogs, +) -> None: + """Dispatch audit log to all registered audit_log_callbacks.""" + if not litellm.audit_log_callbacks: + return + + payload = _build_audit_log_payload(request_data) + + for callback in litellm.audit_log_callbacks: + try: + resolved: Optional[CustomLogger] = ( + callback if isinstance(callback, CustomLogger) else None + ) + if isinstance(callback, str): + resolved = _resolve_audit_log_callback(callback) + if resolved is None: + verbose_proxy_logger.warning( + "Could not resolve audit log callback: %s", callback + ) + continue + + if isinstance(resolved, CustomLogger): + task = asyncio.create_task(resolved.async_log_audit_log_event(payload)) + task.add_done_callback(_audit_log_task_done_callback) + except Exception as e: + verbose_proxy_logger.error( + "Failed dispatching audit log to callback: %s", e + ) async def create_object_audit_log( @@ -40,20 +136,22 @@ async def create_object_audit_log( """ from litellm.secret_managers.main import get_secret_bool - store_audit_logs = litellm.store_audit_logs or get_secret_bool( + _store_audit_logs: Optional[bool] = litellm.store_audit_logs or get_secret_bool( "LITELLM_STORE_AUDIT_LOGS" ) - if store_audit_logs is not True: + if _store_audit_logs is not True: return + _changed_by = ( + litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name + ) + await create_audit_log_for_update( request_data=LiteLLM_AuditLogs( id=str(uuid.uuid4()), updated_at=datetime.now(timezone.utc), - changed_by=litellm_changed_by - or user_api_key_dict.user_id - or litellm_proxy_admin_name, + changed_by=_changed_by, changed_by_api_key=user_api_key_dict.api_key, table_name=table_name, object_id=object_id, @@ -70,10 +168,10 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): """ from litellm.secret_managers.main import get_secret_bool - store_audit_logs = litellm.store_audit_logs or get_secret_bool( + _store_audit_logs: Optional[bool] = litellm.store_audit_logs or get_secret_bool( "LITELLM_STORE_AUDIT_LOGS" ) - if store_audit_logs is not True: + if _store_audit_logs is not True: return from litellm.proxy.proxy_server import premium_user, prisma_client @@ -81,9 +179,6 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): if premium_user is not True: return - if prisma_client is None: - raise Exception("prisma_client is None, no DB connected") - verbose_proxy_logger.debug("creating audit log for %s", request_data) if isinstance(request_data.updated_values, dict): @@ -92,6 +187,15 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): if isinstance(request_data.before_value, dict): request_data.before_value = json.dumps(request_data.before_value) + # Dispatch to external audit log callbacks regardless of DB availability + await _dispatch_audit_log_to_callbacks(request_data) + + if prisma_client is None: + verbose_proxy_logger.error( + "prisma_client is None, cannot write audit log to DB" + ) + return + _request_data = request_data.model_dump(exclude_none=True) try: @@ -103,5 +207,3 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): except Exception as e: # [Non-Blocking Exception. Do not allow blocking LLM API call] verbose_proxy_logger.error(f"Failed Creating audit log {e}") - - return diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index f3f0ba56cb91..74c7f9bca5f3 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -72,7 +72,7 @@ def test_get_api_key_with_custom_litellm_key_header( def test_team_metadata_with_tags_flows_through_jwt_auth(): """ Test that team_metadata (specifically tags) flows through JWT authentication. - + This is a regression test for the issue where JWT auth was not populating team_metadata, causing team-level tags to be missing in litellm_pre_call_utils.py """ @@ -87,7 +87,7 @@ def test_team_metadata_with_tags_flows_through_jwt_auth(): rpm_limit=100, models=["gpt-4", "gpt-3.5-turbo"], ) - + # Simulate constructing UserAPIKeyAuth like we do in JWT auth # This is the pattern from user_api_key_auth.py lines 552-587 user_api_key_auth = UserAPIKeyAuth( @@ -100,14 +100,16 @@ def test_team_metadata_with_tags_flows_through_jwt_auth(): user_role="internal_user", user_id="test-user", ) - + # Verify team_metadata is set - assert user_api_key_auth.team_metadata is not None, "team_metadata should be populated" + assert ( + user_api_key_auth.team_metadata is not None + ), "team_metadata should be populated" assert user_api_key_auth.team_metadata == team_object.metadata, ( f"team_metadata not correctly mapped. " f"Expected: {team_object.metadata}, Got: {user_api_key_auth.team_metadata}" ) - + # Specifically verify tags are present assert "tags" in user_api_key_auth.team_metadata, "tags should be in team_metadata" assert user_api_key_auth.team_metadata["tags"] == ["production", "high-priority"], ( @@ -118,7 +120,7 @@ def test_team_metadata_with_tags_flows_through_jwt_auth(): def test_route_checks_is_llm_api_route(): """Test RouteChecks.is_llm_api_route() correctly identifies LLM API routes including passthrough endpoints""" - + # Test OpenAI routes openai_routes = [ "/v1/chat/completions", @@ -142,18 +144,22 @@ def test_route_checks_is_llm_api_route(): "/v1/realtime", "/realtime", ] - + for route in openai_routes: - assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route( + route=route + ), f"Route {route} should be identified as LLM API route" # Test Anthropic routes anthropic_routes = [ "/v1/messages", "/v1/messages/count_tokens", ] - + for route in anthropic_routes: - assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route( + route=route + ), f"Route {route} should be identified as LLM API route" # Test passthrough routes (this is the key improvement over the old route checking) passthrough_routes = [ @@ -171,9 +177,11 @@ def test_route_checks_is_llm_api_route(): "/vllm/v1/chat/completions", "/mistral/v1/chat/completions", ] - + for route in passthrough_routes: - assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route( + route=route + ), f"Route {route} should be identified as LLM API route" # Test MCP routes mcp_routes = [ @@ -181,9 +189,11 @@ def test_route_checks_is_llm_api_route(): "/mcp/", "/mcp/test", ] - + for route in mcp_routes: - assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route( + route=route + ), f"Route {route} should be identified as LLM API route" # Test LiteLLM native RAG routes rag_routes = [ @@ -193,7 +203,9 @@ def test_route_checks_is_llm_api_route(): "/v1/rag/query", ] for route in rag_routes: - assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route( + route=route + ), f"Route {route} should be identified as LLM API route" # Test routes with placeholders placeholder_routes = [ @@ -206,9 +218,11 @@ def test_route_checks_is_llm_api_route(): "/v1/batches/batch_123", "/batches/batch_123", ] - + for route in placeholder_routes: - assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route( + route=route + ), f"Route {route} should be identified as LLM API route" # Test Azure OpenAI routes azure_routes = [ @@ -217,9 +231,11 @@ def test_route_checks_is_llm_api_route(): "/engines/gpt-4/chat/completions", "/engines/gpt-3.5-turbo/completions", ] - + for route in azure_routes: - assert RouteChecks.is_llm_api_route(route=route), f"Route {route} should be identified as LLM API route" + assert RouteChecks.is_llm_api_route( + route=route + ), f"Route {route} should be identified as LLM API route" # Test non-LLM routes (should return False) non_llm_routes = [ @@ -236,9 +252,11 @@ def test_route_checks_is_llm_api_route(): "/debug", "/test", ] - + for route in non_llm_routes: - assert not RouteChecks.is_llm_api_route(route=route), f"Route {route} should NOT be identified as LLM API route" + assert not RouteChecks.is_llm_api_route( + route=route + ), f"Route {route} should NOT be identified as LLM API route" # Test invalid inputs invalid_inputs = [ @@ -248,9 +266,11 @@ def test_route_checks_is_llm_api_route(): {}, "", ] - + for invalid_input in invalid_inputs: - assert not RouteChecks.is_llm_api_route(route=invalid_input), f"Invalid input {invalid_input} should return False" + assert not RouteChecks.is_llm_api_route( + route=invalid_input + ), f"Invalid input {invalid_input} should return False" @pytest.mark.asyncio @@ -259,7 +279,7 @@ async def test_proxy_admin_expired_key_from_cache(): Test that PROXY_ADMIN keys retrieved from cache are checked for expiration before being returned. This prevents expired keys from bypassing expiration checks when retrieved from cache (which normally happens at lines 1014-1036). - + Regression test for issue where PROXY_ADMIN keys from cache skipped expiration check. """ from datetime import datetime, timedelta, timezone @@ -280,39 +300,42 @@ async def test_proxy_admin_expired_key_from_cache(): api_key = "sk-test-proxy-admin-key" hashed_key = hash_token(api_key) expired_time = datetime.now(timezone.utc) - timedelta(hours=1) # Expired 1 hour ago - + expired_token = UserAPIKeyAuth( api_key=api_key, user_role=LitellmUserRoles.PROXY_ADMIN, expires=expired_time, token=hashed_key, ) - + # Mock cache to return the expired token mock_cache = AsyncMock() mock_cache.async_get_cache = AsyncMock(return_value=expired_token) mock_cache.delete_cache = MagicMock() - + # Mock proxy_logging_obj mock_proxy_logging_obj = MagicMock() mock_proxy_logging_obj.internal_usage_cache = MagicMock() mock_proxy_logging_obj.internal_usage_cache.dual_cache = AsyncMock() - mock_proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock() + mock_proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache = ( + AsyncMock() + ) # Mock post_call_failure_hook as async function returning None (no transformation) mock_proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) - + # Mock prisma_client mock_prisma_client = MagicMock() - + # Mock get_key_object to return expired token from cache with patch( "litellm.proxy.auth.user_api_key_auth.get_key_object", new_callable=AsyncMock, - ) as mock_get_key_object, \ - patch("litellm.proxy.auth.user_api_key_auth._delete_cache_key_object", new_callable=AsyncMock) as mock_delete_cache: - + ) as mock_get_key_object, patch( + "litellm.proxy.auth.user_api_key_auth._delete_cache_key_object", + new_callable=AsyncMock, + ) as mock_delete_cache: mock_get_key_object.return_value = expired_token - + # Set attributes on proxy_server module (these are imported inside _user_api_key_auth_builder) import litellm.proxy.proxy_server as _proxy_server_mod @@ -331,14 +354,12 @@ async def test_proxy_admin_expired_key_from_cache(): "litellm_proxy_admin_name": "admin", } _original_values = { - attr: getattr(_proxy_server_mod, attr, None) - for attr in _attrs_to_set + attr: getattr(_proxy_server_mod, attr, None) for attr in _attrs_to_set } try: for attr, val in _attrs_to_set.items(): setattr(_proxy_server_mod, attr, val) - # Create a mock request request = Request(scope={"type": "http"}) request._url = URL(url="/chat/completions") @@ -358,38 +379,41 @@ async def test_proxy_admin_expired_key_from_cache(): ) # Verify that ProxyException was raised with expired_key type - assert hasattr(exc_info.value, "type"), "Exception should have 'type' attribute" - assert exc_info.value.type == ProxyErrorTypes.expired_key, ( - f"Expected expired_key error type, got {exc_info.value.type}" - ) - assert "Expired Key" in str(exc_info.value.message), ( - f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" - ) + assert hasattr( + exc_info.value, "type" + ), "Exception should have 'type' attribute" + assert ( + exc_info.value.type == ProxyErrorTypes.expired_key + ), f"Expected expired_key error type, got {exc_info.value.type}" + assert "Expired Key" in str( + exc_info.value.message + ), f"Exception message should mention 'Expired Key', got: {exc_info.value.message}" # Verify that the param field does NOT leak the full API key (Issue #18731) # The param should be abbreviated like "sk-...XXXX" not the full plaintext key - assert exc_info.value.param is not None, "Exception should have 'param' attribute" + assert ( + exc_info.value.param is not None + ), "Exception should have 'param' attribute" assert exc_info.value.param != api_key, ( f"SECURITY: Full API key should NOT be in param field! " f"Got: {exc_info.value.param}, Expected abbreviated format like 'sk-...XXXX'" ) - assert exc_info.value.param.startswith("sk-..."), ( - f"Param should be abbreviated to 'sk-...XXXX' format. Got: {exc_info.value.param}" - ) + assert exc_info.value.param.startswith( + "sk-..." + ), f"Param should be abbreviated to 'sk-...XXXX' format. Got: {exc_info.value.param}" # Verify that cache deletion was called mock_delete_cache.assert_called_once() call_args = mock_delete_cache.call_args - assert call_args[1]["hashed_token"] == hashed_key, ( - "Cache deletion should be called with the hashed key" - ) + assert ( + call_args[1]["hashed_token"] == hashed_key + ), "Cache deletion should be called with the hashed key" finally: # Restore all module-level attributes so subsequent tests are not affected for attr, val in _original_values.items(): setattr(_proxy_server_mod, attr, val) - @pytest.mark.asyncio async def test_return_user_api_key_auth_obj_user_spend_and_budget(): """ @@ -400,7 +424,7 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import _return_user_api_key_auth_obj - + user_obj = type( "LiteLLM_UserTable", (), @@ -413,7 +437,7 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): "user_role": "internal_user", }, ) - + api_key = "sk-test-key" valid_token_dict = { "user_id": "test-user", @@ -421,10 +445,10 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): } route = "/chat/completions" start_time = datetime.now() - + mock_service_logger = MagicMock() mock_service_logger.async_service_success_hook = AsyncMock() - + with patch( "litellm.proxy.auth.user_api_key_auth.user_api_key_service_logger_obj", new=mock_service_logger, @@ -438,7 +462,7 @@ async def test_return_user_api_key_auth_obj_user_spend_and_budget(): start_time=start_time, user_role=None, ) - + assert isinstance(result, UserAPIKeyAuth) assert result.user_spend == 250.0 assert result.user_max_budget == 1000.0 @@ -470,9 +494,7 @@ def test_proxy_admin_jwt_auth_includes_identity_fields(): user_role=LitellmUserRoles.PROXY_ADMIN, user_id="user-abc", team_id="team-123", - team_alias=( - team_object.team_alias if team_object is not None else None - ), + team_alias=(team_object.team_alias if team_object is not None else None), team_metadata=team_object.metadata if team_object is not None else None, org_id="org-456", end_user_id="end-user-789", @@ -503,9 +525,7 @@ def test_proxy_admin_jwt_auth_handles_no_team_object(): user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-user", team_id=None, - team_alias=( - team_object.team_alias if team_object is not None else None - ), + team_alias=(team_object.team_alias if team_object is not None else None), team_metadata=team_object.metadata if team_object is not None else None, org_id=None, end_user_id=None, @@ -534,7 +554,10 @@ class TestJWTOAuth2Coexistence: def test_is_jwt_detects_jwt_tokens(self): """JWT tokens have 3 dot-separated parts.""" assert JWTHandler.is_jwt("header.payload.signature") is True - assert JWTHandler.is_jwt("eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.sig123") is True + assert ( + JWTHandler.is_jwt("eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMSJ9.sig123") + is True + ) def test_is_jwt_rejects_opaque_tokens(self): """Opaque OAuth2 tokens do not have 3 dot-separated parts.""" @@ -543,6 +566,10 @@ def test_is_jwt_rejects_opaque_tokens(self): assert JWTHandler.is_jwt("Bearer token") is False assert JWTHandler.is_jwt("two.parts") is False + def test_is_jwt_returns_false_for_none(self): + """None token (missing Authorization header) should not be treated as JWT.""" + assert JWTHandler.is_jwt(None) is False + @pytest.mark.asyncio async def test_both_enabled_opaque_token_uses_oauth2(self): """ @@ -567,13 +594,20 @@ async def test_both_enabled_opaque_token_uses_oauth2(self): mock_request.headers = {"authorization": f"Bearer {opaque_token}"} mock_request.query_params = {} - with patch("litellm.proxy.proxy_server.general_settings", general_settings), \ - patch("litellm.proxy.proxy_server.premium_user", True), \ - patch("litellm.proxy.proxy_server.master_key", "sk-master"), \ - patch("litellm.proxy.proxy_server.prisma_client", None), \ - patch("litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", new_callable=AsyncMock, return_value=mock_oauth2_response) as mock_oauth2, \ - patch("litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", new_callable=AsyncMock) as mock_jwt_auth: - + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: litellm.proxy.proxy_server.jwt_handler.update_environment( prisma_client=None, user_api_key_cache=DualCache(), @@ -624,13 +658,20 @@ async def test_both_enabled_jwt_token_skips_oauth2(self): mock_request.headers = {"authorization": f"Bearer {jwt_token}"} mock_request.query_params = {} - with patch("litellm.proxy.proxy_server.general_settings", general_settings), \ - patch("litellm.proxy.proxy_server.premium_user", True), \ - patch("litellm.proxy.proxy_server.master_key", "sk-master"), \ - patch("litellm.proxy.proxy_server.prisma_client", None), \ - patch("litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", new_callable=AsyncMock) as mock_oauth2, \ - patch("litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", new_callable=AsyncMock, return_value=mock_jwt_result) as mock_jwt_auth: - + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=mock_jwt_result, + ) as mock_jwt_auth: litellm.proxy.proxy_server.jwt_handler.update_environment( prisma_client=None, user_api_key_cache=DualCache(), @@ -671,12 +712,17 @@ async def test_only_oauth2_enabled_handles_all_tokens(self): mock_request.headers = {"authorization": f"Bearer {jwt_like_token}"} mock_request.query_params = {} - with patch("litellm.proxy.proxy_server.general_settings", general_settings), \ - patch("litellm.proxy.proxy_server.premium_user", True), \ - patch("litellm.proxy.proxy_server.master_key", "sk-master"), \ - patch("litellm.proxy.proxy_server.prisma_client", None), \ - patch("litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", new_callable=AsyncMock, return_value=mock_oauth2_response) as mock_oauth2: - + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2: result = await user_api_key_auth( request=mock_request, api_key=f"Bearer {jwt_like_token}", @@ -685,3 +731,131 @@ async def test_only_oauth2_enabled_handles_all_tokens(self): # OAuth2 should handle it since JWT auth is disabled mock_oauth2.assert_called_once_with(token=jwt_like_token) assert result.user_id == "oauth2-user" + + +@pytest.mark.asyncio +async def test_user_api_key_auth_builder_no_blocking_calls(): + """ + _user_api_key_auth_builder must never call any synchronous DualCache method + (set_cache, get_cache, batch_get_cache, increment_cache, delete_cache) on + the hot auth path — those methods call Redis synchronously and block the + event loop. Only async_* variants are allowed. + """ + from starlette.datastructures import URL + from starlette.requests import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import _user_api_key_auth_builder + + _blocking_methods = [ + "set_cache", + "get_cache", + "batch_get_cache", + "increment_cache", + "delete_cache", + ] + + api_key = "sk-test-no-blocking-cache" + valid_token = UserAPIKeyAuth( + api_key=api_key, + token=api_key, + user_role=LitellmUserRoles.INTERNAL_USER, + team_id="team-abc", + ) + + mock_cache = AsyncMock() + mock_cache.async_get_cache = AsyncMock(return_value=valid_token) + mock_cache.async_set_cache = AsyncMock(return_value=None) + # Wire sync methods on the instance as plain MagicMocks (no side_effect) so + # calls are recorded but not raised — the function's broad except Exception + # would swallow a raised error. We assert not_called() after the run instead. + for _m in _blocking_methods: + setattr(mock_cache, _m, MagicMock()) + + mock_proxy_logging_obj = MagicMock() + mock_proxy_logging_obj.internal_usage_cache = MagicMock() + mock_proxy_logging_obj.internal_usage_cache.dual_cache = AsyncMock() + mock_proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache = ( + AsyncMock() + ) + mock_proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + + import litellm.proxy.proxy_server as _proxy_server_mod + + _attrs = { + "prisma_client": MagicMock(), + "user_api_key_cache": mock_cache, + "proxy_logging_obj": mock_proxy_logging_obj, + "master_key": "sk-master-key", + "general_settings": {}, + "llm_model_list": [], + "llm_router": None, + "open_telemetry_logger": None, + "model_max_budget_limiter": MagicMock(), + "user_custom_auth": None, + "jwt_handler": None, + "litellm_proxy_admin_name": "admin", + } + _originals = {k: getattr(_proxy_server_mod, k, None) for k in _attrs} + + try: + for k, v in _attrs.items(): + setattr(_proxy_server_mod, k, v) + + request = Request(scope={"type": "http"}) + request._url = URL(url="/chat/completions") + + import contextlib + + from litellm.caching.dual_cache import DualCache + + blocking_patches = [ + patch.object( + DualCache, + m, + MagicMock( + side_effect=AssertionError( + f"Blocking DualCache.{m}() called on async hot path — use async_{m}() instead" + ) + ), + ) + for m in _blocking_methods + ] + + with contextlib.ExitStack() as stack: + for p in blocking_patches: + stack.enter_context(p) + stack.enter_context( + patch( + "litellm.proxy.auth.user_api_key_auth.get_key_object", + new_callable=AsyncMock, + return_value=valid_token, + ) + ) + stack.enter_context( + patch( + "litellm.proxy.auth.user_api_key_auth.get_team_object", + new_callable=AsyncMock, + return_value=None, + ) + ) + await _user_api_key_auth_builder( + request=request, + api_key=f"Bearer {api_key}", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, + ) + + for _m in _blocking_methods: + mock = getattr(mock_cache, _m) + assert mock.call_count == 0, ( + f"Blocking DualCache.{_m}() was called {mock.call_count} time(s) " + f"on the async hot path — use async_{_m}() instead" + ) + + finally: + for k, v in _originals.items(): + setattr(_proxy_server_mod, k, v) From 940789ff3113f9dafb135fd924dfdcadda46a1cf Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Tue, 31 Mar 2026 22:46:42 +0300 Subject: [PATCH 25/28] Revert "fix: use storeLoginToken in exchangeLoginCode, add credentials include" This reverts commit 866405f443eb2004ee56ed2f52c9047593d8cc6e. --- litellm/proxy/proxy_server.py | 2 +- ui/litellm-dashboard/src/components/networking.tsx | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index de57e8cd969e..5ef94783a8af 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11486,7 +11486,7 @@ async def login_v3_exchange(request: Request): }, status_code=status.HTTP_200_OK, ) - json_response.set_cookie(key="token", value=cached_data["token"], httponly=True) + json_response.set_cookie(key="token", value=cached_data["token"]) return json_response except ProxyException: raise diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index f3adb0fb1791..b3e75f35287c 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -9166,7 +9166,6 @@ export const exchangeLoginCode = async (code: string, workerBaseUrl?: string | n const response = await fetch(`${base}/v3/login/exchange`, { method: "POST", body: JSON.stringify({ code }), - credentials: "include", headers: { "Content-Type": "application/json" }, }); @@ -9177,7 +9176,7 @@ export const exchangeLoginCode = async (code: string, workerBaseUrl?: string | n const data = await response.json(); if (data.token) { - storeLoginToken(data.token); + document.cookie = `token=${data.token}; path=/; SameSite=Lax`; } return data.token; }; From dcbe924d289ec74c43d32b483f8b0bad017c0102 Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Tue, 31 Mar 2026 22:46:42 +0300 Subject: [PATCH 26/28] Revert "fix: keep token in login JSON body for UI storeLoginToken flow" This reverts commit 086c41640c5749399f11aba298321d1d16e10a92. --- litellm/proxy/proxy_server.py | 2 +- tests/test_litellm/proxy/test_proxy_server.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5ef94783a8af..e4b2b184bc09 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11315,7 +11315,7 @@ async def login_v2(request: Request): # noqa: PLR0915 litellm_dashboard_ui += "?login=success" json_response = JSONResponse( - content={"redirect_url": litellm_dashboard_ui, "token": jwt_token}, + content={"redirect_url": litellm_dashboard_ui}, status_code=status.HTTP_200_OK, ) json_response.set_cookie(key="token", value=jwt_token, httponly=True) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 8fecb2f61682..11bae9009a24 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -104,10 +104,7 @@ def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch): ) assert response.status_code == 200 - assert response.json() == { - "redirect_url": "http://testserver/ui/?login=success", - "token": "signed-token", - } + assert response.json() == {"redirect_url": "http://testserver/ui/?login=success"} assert response.cookies.get("token") == "signed-token" mock_authenticate_user.assert_awaited_once_with( From ecd8cc52a53c87aec73f56370e5f9572f8e02aff Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Tue, 31 Mar 2026 22:46:42 +0300 Subject: [PATCH 27/28] Revert "fix: reformat with Black 23.x to match CI" This reverts commit b2c3334c888a9dbb60fd94cce12f43e8f3aa7e82. --- litellm/integrations/prometheus.py | 18 +++--- .../bedrock/chat/converse_transformation.py | 34 +++++----- .../chat/guardrail_translation/handler.py | 6 +- litellm/proxy/management_endpoints/ui_sso.py | 54 ++++++++-------- litellm/proxy/proxy_server.py | 62 +++++++++---------- 5 files changed, 83 insertions(+), 91 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 5d1ff8329301..d022af689d3a 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2085,9 +2085,9 @@ def set_llm_deployment_success_metrics( ): try: verbose_logger.debug("setting remaining tokens requests metric") - standard_logging_payload: Optional[ - StandardLoggingPayload - ] = request_kwargs.get("standard_logging_object") + standard_logging_payload: Optional[StandardLoggingPayload] = ( + request_kwargs.get("standard_logging_object") + ) if standard_logging_payload is None: return @@ -2720,9 +2720,7 @@ async def _initialize_api_key_budget_metrics(self): ) return - async def fetch_keys( - page_size: int, page: int - ) -> Tuple[ + async def fetch_keys(page_size: int, page: int) -> Tuple[ List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]], Optional[int], ]: @@ -3399,10 +3397,10 @@ def initialize_budget_metrics_cron_job(scheduler: AsyncIOScheduler): from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES from litellm.integrations.custom_logger import CustomLogger - prometheus_loggers: List[ - CustomLogger - ] = litellm.logging_callback_manager.get_custom_loggers_for_type( - callback_type=PrometheusLogger + prometheus_loggers: List[CustomLogger] = ( + litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=PrometheusLogger + ) ) # we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers)) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5cfb00d69b61..e01f29948107 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1744,9 +1744,7 @@ def apply_tool_call_transformation_if_needed( return message, returned_finish_reason - def _translate_message_content( - self, content_blocks: List[ContentBlock] - ) -> Tuple[ + def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[ str, List[ChatCompletionToolCallChunk], Optional[List[BedrockConverseReasoningContentBlock]], @@ -1763,9 +1761,9 @@ def _translate_message_content( """ content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[ - List[BedrockConverseReasoningContentBlock] - ] = None + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( + None + ) citationsContentBlocks: Optional[List[CitationsContentBlock]] = None for idx, content in enumerate(content_blocks): """ @@ -1976,9 +1974,9 @@ def _transform_response( # noqa: PLR0915 chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[ - List[BedrockConverseReasoningContentBlock] - ] = None + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( + None + ) citationsContentBlocks: Optional[List[CitationsContentBlock]] = None if message is not None: @@ -1997,17 +1995,17 @@ def _transform_response( # noqa: PLR0915 provider_specific_fields["citationsContent"] = citationsContentBlocks if provider_specific_fields: - chat_completion_message[ - "provider_specific_fields" - ] = provider_specific_fields + chat_completion_message["provider_specific_fields"] = ( + provider_specific_fields + ) if reasoningContentBlocks is not None: - chat_completion_message[ - "reasoning_content" - ] = self._transform_reasoning_content(reasoningContentBlocks) - chat_completion_message[ - "thinking_blocks" - ] = self._transform_thinking_blocks(reasoningContentBlocks) + chat_completion_message["reasoning_content"] = ( + self._transform_reasoning_content(reasoningContentBlocks) + ) + chat_completion_message["thinking_blocks"] = ( + self._transform_thinking_blocks(reasoningContentBlocks) + ) chat_completion_message["content"] = content_str filtered_tools = self._filter_json_mode_tools( json_mode=json_mode, diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index f854cdb13d00..0ce9d57a65f7 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -86,9 +86,9 @@ async def process_input_messages( if tool_calls_to_check: inputs["tool_calls"] = tool_calls_to_check # type: ignore if messages: - inputs[ - "structured_messages" - ] = messages # pass the openai /chat/completions messages to the guardrail, as-is + inputs["structured_messages"] = ( + messages # pass the openai /chat/completions messages to the guardrail, as-is + ) # Pass tools (function definitions) to the guardrail tools = data.get("tools") if tools: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 7ab065ce3931..02d7b9baf4a4 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -722,9 +722,9 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: import ast try: - generic_user_role_mappings_data: Dict[ - LitellmUserRoles, List[str] - ] = ast.literal_eval(generic_role_mappings) + generic_user_role_mappings_data: Dict[LitellmUserRoles, List[str]] = ( + ast.literal_eval(generic_role_mappings) + ) if isinstance(generic_user_role_mappings_data, dict): from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings @@ -881,9 +881,9 @@ def response_convertor(response, client): verbose_proxy_logger.debug("calling generic_sso.verify_and_process") additional_generic_sso_headers_dict = _parse_generic_sso_headers() - code_verifier: Optional[ - str - ] = None # assigned inside try; initialized for type tracking + code_verifier: Optional[str] = ( + None # assigned inside try; initialized for type tracking + ) access_token_payload: Optional[dict] = None # decoded JWT access token claims try: @@ -1263,9 +1263,9 @@ def apply_user_info_values_to_sso_user_defined_values( else: # SSO didn't provide a valid role, fall back to DB role or default if user_info is None or user_info.user_role is None: - user_defined_values[ - "user_role" - ] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value + user_defined_values["user_role"] = ( + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value + ) verbose_proxy_logger.debug( "No SSO or DB role found, using default: INTERNAL_USER_VIEW_ONLY" ) @@ -1373,16 +1373,14 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: ) elif generic_client_id is not None: - ( - result, - received_response, - access_token_payload, - ) = await get_generic_sso_response( - request=request, - jwt_handler=jwt_handler, - generic_client_id=generic_client_id, - redirect_url=redirect_url, - sso_jwt_handler=sso_jwt_handler, + result, received_response, access_token_payload = ( + await get_generic_sso_response( + request=request, + jwt_handler=jwt_handler, + generic_client_id=generic_client_id, + redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, + ) ) if result is None: @@ -1705,9 +1703,9 @@ async def insert_sso_user( if user_defined_values.get("max_budget") is None: user_defined_values["max_budget"] = litellm.max_internal_user_budget if user_defined_values.get("budget_duration") is None: - user_defined_values[ - "budget_duration" - ] = litellm.internal_user_budget_duration + user_defined_values["budget_duration"] = ( + litellm.internal_user_budget_duration + ) if user_defined_values["user_role"] is None: user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -3350,9 +3348,9 @@ async def get_microsoft_callback_response( # if user is trying to get the raw sso response for debugging, return the raw sso response if return_raw_sso_response: - original_msft_result[ - MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY - ] = user_team_ids + original_msft_result[MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY] = ( + user_team_ids + ) original_msft_result["app_roles"] = app_roles return original_msft_result or {} @@ -3471,9 +3469,9 @@ async def get_user_groups_from_graph_api( # Fetch user membership from Microsoft Graph API all_group_ids = [] - next_link: Optional[ - str - ] = MicrosoftSSOHandler.graph_api_user_groups_endpoint + next_link: Optional[str] = ( + MicrosoftSSOHandler.graph_api_user_groups_endpoint + ) auth_headers = {"Authorization": f"Bearer {access_token}"} page_count = 0 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e4b2b184bc09..317b8f761792 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -639,9 +639,9 @@ def generate_feedback_box(): server_root_path = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional[ - "EnterpriseLicenseData" -] = _license_check.airgapped_license_data +premium_user_data: Optional["EnterpriseLicenseData"] = ( + _license_check.airgapped_license_data +) global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -1524,9 +1524,9 @@ async def root_redirect(): config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional[ - "ClientSession" -] = None # Global shared session for connection reuse +shared_aiohttp_session: Optional["ClientSession"] = ( + None # Global shared session for connection reuse +) user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1537,13 +1537,13 @@ async def root_redirect(): dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[ - RedisCache -] = None # redis cache used for tracking spend, tpm/rpm limits +redis_usage_cache: Optional[RedisCache] = ( + None # redis cache used for tracking spend, tpm/rpm limits +) polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False -native_background_mode: List[ - str -] = [] # Models that should use native provider background mode instead of polling +native_background_mode: List[str] = ( + [] +) # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None user_custom_key_generate = None @@ -2027,9 +2027,9 @@ async def _update_team_cache(): _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[ - LiteLLM_TeamTable - ] = await user_api_key_cache.async_get_cache(key=_id) + existing_spend_obj: Optional[LiteLLM_TeamTable] = ( + await user_api_key_cache.async_get_cache(key=_id) + ) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -2150,11 +2150,9 @@ def run_ollama_serve(): with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - verbose_proxy_logger.debug( - f""" + verbose_proxy_logger.debug(f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` - """ - ) + """) def _get_process_rss_mb() -> Optional[float]: @@ -5202,10 +5200,10 @@ async def _init_guardrails_in_db(self, prisma_client: PrismaClient): ) try: - guardrails_in_db: List[ - Guardrail - ] = await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client + guardrails_in_db: List[Guardrail] = ( + await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client + ) ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -5587,9 +5585,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ[ - "AZURE_API_VERSION" - ] = api_version # set this for azure - litellm can read this from the env + os.environ["AZURE_API_VERSION"] = ( + api_version # set this for azure - litellm can read this from the env + ) if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -5926,9 +5924,9 @@ def _validate_redis_transaction_buffer_config( """ from litellm.secret_managers.main import str_to_bool - _use_redis_transaction_buffer: Optional[ - Union[bool, str] - ] = general_settings.get("use_redis_transaction_buffer", False) + _use_redis_transaction_buffer: Optional[Union[bool, str]] = ( + general_settings.get("use_redis_transaction_buffer", False) + ) if isinstance(_use_redis_transaction_buffer, str): _use_redis_transaction_buffer = str_to_bool(_use_redis_transaction_buffer) @@ -12523,9 +12521,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[ - idx - ].field_description = sub_field_info.description + nested_fields[idx].field_description = ( + sub_field_info.description + ) idx += 1 _stored_in_db = None From 7f14d41c0283fa50c6f25269a83551a04d132bfa Mon Sep 17 00:00:00 2001 From: Hendrik Jaks Date: Tue, 31 Mar 2026 22:46:42 +0300 Subject: [PATCH 28/28] Revert "fix CI: formatting, test params, remove token from login JSON" This reverts commit 2905d47bd4013b81fbaa01cf2c7402d51360f1e6. --- litellm/integrations/prometheus.py | 46 ++- .../bedrock/chat/converse_transformation.py | 315 +++++------------- litellm/proxy/auth/auth_checks.py | 12 +- .../proxy/common_utils/reset_budget_job.py | 12 +- litellm/proxy/management_endpoints/ui_sso.py | 62 ++-- litellm/proxy/proxy_server.py | 77 +++-- .../proxy/auth/test_user_api_key_auth.py | 8 +- 7 files changed, 188 insertions(+), 344 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index d022af689d3a..294b9d464f58 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -1087,11 +1087,9 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti ), client_ip=standard_logging_payload["metadata"].get("requester_ip_address"), user_agent=standard_logging_payload["metadata"].get("user_agent"), - stream=( - str(standard_logging_payload.get("stream")) - if litellm.prometheus_emit_stream_label - else None - ), + stream=str(standard_logging_payload.get("stream")) + if litellm.prometheus_emit_stream_label + else None, ) if ( @@ -1757,11 +1755,9 @@ async def async_post_call_failure_hook( client_ip=_metadata.get("requester_ip_address"), user_agent=_metadata.get("user_agent"), model_id=model_id, - stream=( - str(request_data.get("stream")) - if litellm.prometheus_emit_stream_label - else None - ), + stream=str(request_data.get("stream")) + if litellm.prometheus_emit_stream_label + else None, ) _labels = prometheus_label_factory( supported_enum_labels=self.get_labels_for_metric( @@ -2085,9 +2081,9 @@ def set_llm_deployment_success_metrics( ): try: verbose_logger.debug("setting remaining tokens requests metric") - standard_logging_payload: Optional[StandardLoggingPayload] = ( - request_kwargs.get("standard_logging_object") - ) + standard_logging_payload: Optional[ + StandardLoggingPayload + ] = request_kwargs.get("standard_logging_object") if standard_logging_payload is None: return @@ -2720,7 +2716,9 @@ async def _initialize_api_key_budget_metrics(self): ) return - async def fetch_keys(page_size: int, page: int) -> Tuple[ + async def fetch_keys( + page_size: int, page: int + ) -> Tuple[ List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]], Optional[int], ]: @@ -2791,7 +2789,9 @@ async def _initialize_org_budget_metrics(self): ) return - async def fetch_orgs(page_size: int, page: int) -> Tuple[list, Optional[int]]: + async def fetch_orgs( + page_size: int, page: int + ) -> Tuple[list, Optional[int]]: skip = (page - 1) * page_size orgs = await prisma_client.db.litellm_organizationtable.find_many( skip=skip, @@ -2911,11 +2911,9 @@ async def _set_org_list_budget_metrics(self, orgs: list): org_alias=org.organization_alias or "", spend=org.spend or 0.0, max_budget=budget_table.max_budget if budget_table else None, - budget_reset_at=( - getattr(budget_table, "budget_reset_at", None) - if budget_table - else None - ), + budget_reset_at=getattr(budget_table, "budget_reset_at", None) + if budget_table + else None, ) async def _set_team_budget_metrics_after_api_request( @@ -3397,10 +3395,10 @@ def initialize_budget_metrics_cron_job(scheduler: AsyncIOScheduler): from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES from litellm.integrations.custom_logger import CustomLogger - prometheus_loggers: List[CustomLogger] = ( - litellm.logging_callback_manager.get_custom_loggers_for_type( - callback_type=PrometheusLogger - ) + prometheus_loggers: List[ + CustomLogger + ] = litellm.logging_callback_manager.get_custom_loggers_for_type( + callback_type=PrometheusLogger ) # we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers)) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index e01f29948107..90b501a9cbb0 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -160,8 +160,7 @@ def _convert_consecutive_user_messages_to_guarded_text( if isinstance(content, list): has_guarded_text = any( - isinstance(item, dict) and item.get("type") == "guarded_text" - for item in content + isinstance(item, dict) and item.get("type") == "guarded_text" for item in content ) if has_guarded_text: continue # Skip this message if it already has guarded_text @@ -322,13 +321,9 @@ def _is_nova_2_model(self, model: str) -> bool: # Check if the model is a Nova 2 model (matches nova-2-lite, nova-2-pro, etc.) # Also check for nova-2/ spec prefix for imported models - return model_without_region.startswith( - "amazon.nova-2-" - ) or model_without_region.startswith("nova-2/") + return model_without_region.startswith("amazon.nova-2-") or model_without_region.startswith("nova-2/") - def _map_web_search_options( - self, web_search_options: dict, model: str - ) -> Optional[BedrockToolBlock]: + def _map_web_search_options(self, web_search_options: dict, model: str) -> Optional[BedrockToolBlock]: """ Map web_search_options to Nova grounding systemTool. @@ -357,9 +352,7 @@ def _map_web_search_options( # (unlike Anthropic), so we just enable grounding with no options return BedrockToolBlock(systemTool={"name": "nova_grounding"}) - def _transform_reasoning_effort_to_reasoning_config( - self, reasoning_effort: str - ) -> dict: + def _transform_reasoning_effort_to_reasoning_config(self, reasoning_effort: str) -> dict: """ Transform reasoning_effort parameter to Nova 2 reasoningConfig structure. @@ -404,9 +397,7 @@ def _transform_reasoning_effort_to_reasoning_config( } } - def _handle_reasoning_effort_parameter( - self, model: str, reasoning_effort: str, optional_params: dict - ) -> None: + def _handle_reasoning_effort_parameter(self, model: str, reasoning_effort: str, optional_params: dict) -> None: """ Handle the reasoning_effort parameter based on the model type. @@ -443,9 +434,7 @@ def _handle_reasoning_effort_parameter( optional_params["reasoning_effort"] = reasoning_effort elif self._is_nova_2_model(model): # Nova 2 models: transform to reasoningConfig - reasoning_config = self._transform_reasoning_effort_to_reasoning_config( - reasoning_effort - ) + reasoning_config = self._transform_reasoning_effort_to_reasoning_config(reasoning_effort) optional_params.update(reasoning_config) else: # Anthropic and other models: convert to thinking parameter @@ -489,9 +478,7 @@ def get_supported_openai_params(self, model: str) -> List[str]: "parallel_tool_calls", ] - if ( - "arn" in model - ): # we can't infer the model from the arn, so just add all params + if "arn" in model: # we can't infer the model from the arn, so just add all params supported_params.append("tools") supported_params.append("tool_choice") supported_params.append("thinking") @@ -513,9 +500,7 @@ def get_supported_openai_params(self, model: str) -> List[str]: or base_model.startswith("meta.llama3-3") or base_model.startswith("meta.llama4") or base_model.startswith("amazon.nova") - or supports_function_calling( - model=model, custom_llm_provider=self.custom_llm_provider - ) + or supports_function_calling(model=model, custom_llm_provider=self.custom_llm_provider) ): supported_params.append("tools") @@ -525,9 +510,7 @@ def get_supported_openai_params(self, model: str) -> List[str]: if litellm.utils.supports_tool_choice( model=model, custom_llm_provider=self.custom_llm_provider - ) or litellm.utils.supports_tool_choice( - model=base_model, custom_llm_provider=self.custom_llm_provider - ): + ) or litellm.utils.supports_tool_choice(model=base_model, custom_llm_provider=self.custom_llm_provider): # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html supported_params.append("tool_choice") @@ -546,9 +529,7 @@ def get_supported_openai_params(self, model: str) -> List[str]: model=model, custom_llm_provider=self.custom_llm_provider, ) - or supports_reasoning( - model=base_model, custom_llm_provider=self.custom_llm_provider - ) + or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) ): supported_params.append("thinking") supported_params.append("reasoning_effort") @@ -573,9 +554,7 @@ def map_tool_choice_values( return ToolChoiceValuesBlock(auto={}) elif isinstance(tool_choice, dict): # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html - specific_tool = SpecificToolChoiceBlock( - name=tool_choice.get("function", {}).get("name", "") - ) + specific_tool = SpecificToolChoiceBlock(name=tool_choice.get("function", {}).get("name", "")) return ToolChoiceValuesBlock(tool=specific_tool) else: raise litellm.utils.UnsupportedParamsError( @@ -595,15 +574,9 @@ def get_supported_video_types(self) -> List[str]: return ["mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp"] def get_all_supported_content_types(self) -> List[str]: - return ( - self.get_supported_image_types() - + self.get_supported_document_types() - + self.get_supported_video_types() - ) + return self.get_supported_image_types() + self.get_supported_document_types() + self.get_supported_video_types() - def is_computer_use_tool_used( - self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str - ) -> bool: + def is_computer_use_tool_used(self, tools: Optional[List[OpenAIChatCompletionToolParam]], model: str) -> bool: """Check if computer use tools are being used in the request.""" if tools is None: return False @@ -616,9 +589,7 @@ def is_computer_use_tool_used( return True return False - def _transform_computer_use_tools( - self, computer_use_tools: List[OpenAIChatCompletionToolParam] - ) -> List[dict]: + def _transform_computer_use_tools(self, computer_use_tools: List[OpenAIChatCompletionToolParam]) -> List[dict]: """Transform computer use tools to Bedrock format.""" transformed_tools: List[dict] = [] @@ -660,9 +631,7 @@ def _transform_computer_use_tools( def _separate_computer_use_tools( self, tools: List[OpenAIChatCompletionToolParam], model: str - ) -> Tuple[ - List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam] - ]: + ) -> Tuple[List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]]: """ Separate computer use tools from regular function tools. @@ -771,25 +740,18 @@ def _add_additional_properties_to_schema(schema: dict) -> dict: # Recurse into nested schemas if "properties" in result and isinstance(result["properties"], dict): result["properties"] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) - for k, v in result["properties"].items() + k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result["properties"].items() } if "items" in result and isinstance(result["items"], dict): - result["items"] = AmazonConverseConfig._add_additional_properties_to_schema( - result["items"] - ) + result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"]) for defs_key in ("$defs", "definitions"): if defs_key in result and isinstance(result[defs_key], dict): result[defs_key] = { - k: AmazonConverseConfig._add_additional_properties_to_schema(v) - for k, v in result[defs_key].items() + k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items() } for key in ("anyOf", "allOf", "oneOf"): if key in result and isinstance(result[key], list): - result[key] = [ - AmazonConverseConfig._add_additional_properties_to_schema(item) - for item in result[key] - ] + result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]] return result @@ -819,9 +781,7 @@ def _create_output_config_for_response_format( } """ if json_schema is not None: - json_schema = AmazonConverseConfig._add_additional_properties_to_schema( - json_schema - ) + json_schema = AmazonConverseConfig._add_additional_properties_to_schema(json_schema) schema_str = json.dumps(json_schema) if json_schema is not None else "{}" json_schema_def: JsonSchemaDefinition = {"schema": schema_str} if name is not None: @@ -843,14 +803,9 @@ def _apply_tool_call_transformation( non_default_params: dict, optional_params: dict, ): - optional_params = self._add_tools_to_optional_params( - optional_params=optional_params, tools=tools - ) + optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=tools) - if ( - "meta.llama3-3-70b-instruct-v1:0" in model - and non_default_params.get("stream", False) is True - ): + if "meta.llama3-3-70b-instruct-v1:0" in model and non_default_params.get("stream", False) is True: optional_params["fake_stream"] = True def map_openai_params( @@ -989,10 +944,7 @@ def _translate_response_format_param( if "type" in value and value["type"] == "text": return optional_params - if ( - self._supports_native_structured_outputs(model, self.custom_llm_provider) - and json_schema is not None - ): + if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None: # Use Bedrock's native structured outputs API (outputConfig.textFormat) # No synthetic tool injection, no fake_stream needed. # Requires an explicit schema — json_object with no schema falls through @@ -1010,14 +962,10 @@ def _translate_response_format_param( json_schema=json_schema, description=description, ) - optional_params = self._add_tools_to_optional_params( - optional_params=optional_params, tools=[_tool] - ) + optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) if ( - litellm.utils.supports_tool_choice( - model=model, custom_llm_provider=self.custom_llm_provider - ) + litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) and not is_thinking_enabled ): optional_params["tool_choice"] = ToolChoiceValuesBlock( @@ -1029,9 +977,7 @@ def _translate_response_format_param( optional_params["json_mode"] = True return optional_params - def update_optional_params_with_thinking_tokens( - self, non_default_params: dict, optional_params: dict - ): + def update_optional_params_with_thinking_tokens(self, non_default_params: dict, optional_params: dict): """ Handles scenario where max tokens is not specified. For anthropic models (anthropic api/bedrock/vertex ai), this requires having the max tokens being set and being greater than the thinking token budget. @@ -1049,13 +995,9 @@ def update_optional_params_with_thinking_tokens( is_thinking_enabled = self.is_thinking_enabled(optional_params) is_max_tokens_in_request = self.is_max_tokens_in_request(non_default_params) if is_thinking_enabled and not is_max_tokens_in_request: - thinking_token_budget = cast(dict, optional_params["thinking"]).get( - "budget_tokens", None - ) + thinking_token_budget = cast(dict, optional_params["thinking"]).get("budget_tokens", None) if thinking_token_budget is not None: - optional_params["maxTokens"] = ( - thinking_token_budget + DEFAULT_MAX_TOKENS - ) + optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS @overload def _get_cache_point_block( @@ -1121,23 +1063,15 @@ def _transform_system_message( if message["role"] == "system": system_prompt_indices.append(idx) if isinstance(message["content"], str) and message["content"]: - system_content_blocks.append( - SystemContentBlock(text=message["content"]) - ) - cache_block = self._get_cache_point_block( - message, block_type="system", model=model - ) + system_content_blocks.append(SystemContentBlock(text=message["content"])) + cache_block = self._get_cache_point_block(message, block_type="system", model=model) if cache_block: system_content_blocks.append(cache_block) elif isinstance(message["content"], list): for m in message["content"]: if m.get("type") == "text" and m.get("text"): - system_content_blocks.append( - SystemContentBlock(text=m["text"]) - ) - cache_block = self._get_cache_point_block( - m, block_type="system", model=model - ) + system_content_blocks.append(SystemContentBlock(text=m["text"])) + cache_block = self._get_cache_point_block(m, block_type="system", model=model) if cache_block: system_content_blocks.append(cache_block) if len(system_prompt_indices) > 0: @@ -1175,16 +1109,10 @@ def _prepare_request_params( # Exceptions should not be stored in optional_params (this is a defensive fix) cleaned_params = filter_exceptions_from_params(optional_params) inference_params = safe_deep_copy(cleaned_params) - supported_converse_params = list( - AmazonConverseConfig.__annotations__.keys() - ) + ["top_k"] + supported_converse_params = list(AmazonConverseConfig.__annotations__.keys()) + ["top_k"] supported_tool_call_params = ["tools", "tool_choice"] supported_config_params = list(self.get_config_blocks().keys()) - total_supported_params = ( - supported_converse_params - + supported_tool_call_params - + supported_config_params - ) + total_supported_params = supported_converse_params + supported_tool_call_params + supported_config_params inference_params.pop("json_mode", None) # used for handling json_schema # Anthropic-only key. Bedrock expects `outputConfig` (camelCase) and # will reject `output_config` if it leaks through pass-through routes. @@ -1195,25 +1123,15 @@ def _prepare_request_params( if request_metadata is not None: self._validate_request_metadata(request_metadata) - output_config: Optional[OutputConfigBlock] = inference_params.pop( - "outputConfig", None - ) - inference_params.pop( - "output_config", None - ) # Bedrock Converse doesn't support it + output_config: Optional[OutputConfigBlock] = inference_params.pop("outputConfig", None) + inference_params.pop("output_config", None) # Bedrock Converse doesn't support it # keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params' - additional_request_params = { - k: v for k, v in inference_params.items() if k not in total_supported_params - } - inference_params = { - k: v for k, v in inference_params.items() if k in total_supported_params - } + additional_request_params = {k: v for k, v in inference_params.items() if k not in total_supported_params} + inference_params = {k: v for k, v in inference_params.items() if k in total_supported_params} # Handle parallel_tool_calls configuration - parallel_tool_use_config = additional_request_params.pop( - "_parallel_tool_use_config", None - ) + parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): for key, value in parallel_tool_use_config.items(): if ( @@ -1228,9 +1146,7 @@ def _prepare_request_params( additional_request_params.pop("parallel_tool_calls", None) # Only set the topK value in for models that support it - additional_request_params.update( - self._handle_top_k_value(model, inference_params) - ) + additional_request_params.update(self._handle_top_k_value(model, inference_params)) # Filter out internal/MCP-related parameters that shouldn't be sent to the API # These are LiteLLM internal parameters, not API parameters @@ -1239,9 +1155,7 @@ def _prepare_request_params( # Filter out non-serializable objects (exceptions, callables, logging objects, etc.) # from additional_request_params to prevent JSON serialization errors # This filters: Exception objects, callable objects (functions), Logging objects, etc. - additional_request_params = filter_exceptions_from_params( - additional_request_params - ) + additional_request_params = filter_exceptions_from_params(additional_request_params) return ( inference_params, @@ -1288,9 +1202,7 @@ def _process_tools_and_beta( # Only separate tools if computer use tools are actually present if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model): # Separate computer use tools from regular function tools - computer_use_tools, regular_tools = self._separate_computer_use_tools( - filtered_tools, model - ) + computer_use_tools, regular_tools = self._separate_computer_use_tools(filtered_tools, model) # Process regular function tools using existing logic bedrock_tools = _bedrock_tools_pt(regular_tools) @@ -1351,9 +1263,7 @@ def _process_tools_and_beta( anthropic_beta_list.append(computer_use_header) # Transform computer use tools to proper Bedrock format - transformed_computer_tools = self._transform_computer_use_tools( - computer_use_tools - ) + transformed_computer_tools = self._transform_computer_use_tools(computer_use_tools) additional_request_params["tools"] = transformed_computer_tools else: # No computer use tools, process all tools as regular tools @@ -1382,15 +1292,9 @@ def _transform_request_helper( """ Bedrock doesn't support tool calling without `tools=` param specified. """ - if ( - "tools" not in optional_params - and messages is not None - and has_tool_call_blocks(messages) - ): + if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages): if litellm.modify_params: - optional_params["tools"] = add_dummy_tool( - custom_llm_provider="bedrock_converse" - ) + optional_params["tools"] = add_dummy_tool(custom_llm_provider="bedrock_converse") else: raise litellm.UnsupportedParamsError( message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.", @@ -1444,9 +1348,7 @@ def _transform_request_helper( bedrock_tool_config: Optional[ToolConfigBlock] = None if len(bedrock_tools) > 0: - tool_choice_values: ToolChoiceValuesBlock = inference_params.pop( - "tool_choice", None - ) + tool_choice_values: ToolChoiceValuesBlock = inference_params.pop("tool_choice", None) bedrock_tool_config = ToolConfigBlock( tools=bedrock_tools, ) @@ -1456,9 +1358,7 @@ def _transform_request_helper( data: CommonRequestObject = { "additionalModelRequestFields": additional_request_params, "system": system_content_blocks, - "inferenceConfig": self._transform_inference_params( - inference_params=inference_params - ), + "inferenceConfig": self._transform_inference_params(inference_params=inference_params), } # Handle all config blocks @@ -1488,14 +1388,10 @@ async def _async_transform_request( litellm_params: dict, headers: Optional[dict] = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message( - messages, model=model - ) + messages, system_content_blocks = self._transform_system_message(messages, model=model) # Convert last user message to guarded_text if guardrailConfig is present - messages = self._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) ## TRANSFORMATION ## _data: CommonRequestObject = self._transform_request_helper( @@ -1506,13 +1402,11 @@ async def _async_transform_request( headers=headers, ) - bedrock_messages = ( - await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( - messages=messages, - model=model, - llm_provider="bedrock_converse", - user_continue_message=litellm_params.pop("user_continue_message", None), - ) + bedrock_messages = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model=model, + llm_provider="bedrock_converse", + user_continue_message=litellm_params.pop("user_continue_message", None), ) data: RequestObject = {"messages": bedrock_messages, **_data} @@ -1546,14 +1440,10 @@ def _transform_request( litellm_params: dict, headers: Optional[dict] = None, ) -> RequestObject: - messages, system_content_blocks = self._transform_system_message( - messages, model=model - ) + messages, system_content_blocks = self._transform_system_message(messages, model=model) # Convert last user message to guarded_text if guardrailConfig is present - messages = self._convert_consecutive_user_messages_to_guarded_text( - messages, optional_params - ) + messages = self._convert_consecutive_user_messages_to_guarded_text(messages, optional_params) _data: CommonRequestObject = self._transform_request_helper( model=model, @@ -1602,9 +1492,7 @@ def transform_response( encoding=encoding, ) - def _transform_reasoning_content( - self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock] - ) -> str: + def _transform_reasoning_content(self, reasoning_content_blocks: List[BedrockConverseReasoningContentBlock]) -> str: """ Extract the reasoning text from the reasoning content blocks @@ -1620,9 +1508,7 @@ def _transform_thinking_blocks( self, thinking_blocks: List[BedrockConverseReasoningContentBlock] ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]: """Return a consistent format for thinking blocks between Anthropic and Bedrock.""" - thinking_blocks_list: List[ - Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] - ] = [] + thinking_blocks_list: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] = [] for block in thinking_blocks: if "reasoningText" in block: _thinking_block = ChatCompletionThinkingBlock(type="thinking") @@ -1658,21 +1544,11 @@ def _transform_usage( cache_creation_input_tokens = usage["cacheWriteInputTokens"] input_tokens += cache_creation_input_tokens - prompt_tokens_details = PromptTokensDetailsWrapper( - cached_tokens=cache_read_input_tokens - ) - reasoning_tokens = ( - token_counter(text=reasoning_content, count_response_tokens=True) - if reasoning_content - else 0 - ) + prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=cache_read_input_tokens) + reasoning_tokens = token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0 completion_tokens_details = CompletionTokensDetailsWrapper( reasoning_tokens=reasoning_tokens, - text_tokens=( - output_tokens - reasoning_tokens - if reasoning_tokens > 0 - else output_tokens - ), + text_tokens=(output_tokens - reasoning_tokens if reasoning_tokens > 0 else output_tokens), ) openai_usage = Usage( prompt_tokens=input_tokens, @@ -1687,9 +1563,7 @@ def _transform_usage( def get_tool_call_names( self, - tools: Optional[ - Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]] - ] = None, + tools: Optional[Union[List[ToolBlock], List[OpenAIChatCompletionToolParam]]] = None, ) -> List[str]: if tools is None: return [] @@ -1728,13 +1602,8 @@ def apply_tool_call_transformation_if_needed( try: tool_call_names = self.get_tool_call_names(tools) json_content = json.loads(message.content) - if ( - json_content.get("type") == "function" - and json_content.get("name") in tool_call_names - ): - tool_calls = [ - ChatCompletionMessageToolCall(function=Function(**json_content)) - ] + if json_content.get("type") == "function" and json_content.get("name") in tool_call_names: + tool_calls = [ChatCompletionMessageToolCall(function=Function(**json_content))] message.tool_calls = tool_calls message.content = None @@ -1744,7 +1613,9 @@ def apply_tool_call_transformation_if_needed( return message, returned_finish_reason - def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[ + def _translate_message_content( + self, content_blocks: List[ContentBlock] + ) -> Tuple[ str, List[ChatCompletionToolCallChunk], Optional[List[BedrockConverseReasoningContentBlock]], @@ -1761,9 +1632,7 @@ def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tupl """ content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( - None - ) + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None citationsContentBlocks: Optional[List[CitationsContentBlock]] = None for idx, content in enumerate(content_blocks): """ @@ -1780,9 +1649,7 @@ def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tupl if "toolUse" in content: ## check tool name was formatted by litellm _response_tool_name = content["toolUse"]["name"] - response_tool_name = get_bedrock_tool_name( - response_tool_name=_response_tool_name - ) + response_tool_name = get_bedrock_tool_name(response_tool_name=_response_tool_name) _function_chunk = ChatCompletionToolCallFunctionChunk( name=response_tool_name, arguments=json.dumps(content["toolUse"]["input"]), @@ -1833,11 +1700,7 @@ def _unwrap_bedrock_properties(json_str: str) -> str: """ try: response_data = json.loads(json_str) - if ( - isinstance(response_data, dict) - and "properties" in response_data - and len(response_data) == 1 - ): + if isinstance(response_data, dict) and "properties" in response_data and len(response_data) == 1: response_data = response_data["properties"] return json.dumps(response_data) except json.JSONDecodeError: @@ -1861,11 +1724,7 @@ def _filter_json_mode_tools( if not json_mode or not tools: return tools if tools else None - json_tool_indices = [ - i - for i, t in enumerate(tools) - if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME - ] + json_tool_indices = [i for i, t in enumerate(tools) if t["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME] if not json_tool_indices: # No json_tool_call found, return tools unchanged @@ -1873,14 +1732,10 @@ def _filter_json_mode_tools( if len(json_tool_indices) == len(tools): # All tools are json_tool_call — convert first one to content - verbose_logger.debug( - "Processing JSON tool call response for response_format" - ) + verbose_logger.debug("Processing JSON tool call response for response_format") json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments") if json_mode_content_str is not None: - json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties( - json_mode_content_str - ) + json_mode_content_str = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_content_str) chat_completion_message["content"] = json_mode_content_str return None @@ -1890,13 +1745,9 @@ def _filter_json_mode_tools( first_idx = json_tool_indices[0] json_mode_args = tools[first_idx]["function"].get("arguments") if json_mode_args is not None: - json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties( - json_mode_args - ) + json_mode_args = AmazonConverseConfig._unwrap_bedrock_properties(json_mode_args) existing = chat_completion_message.get("content") or "" - chat_completion_message["content"] = ( - existing + json_mode_args if existing else json_mode_args - ) + chat_completion_message["content"] = existing + json_mode_args if existing else json_mode_args real_tools = [t for i, t in enumerate(tools) if i not in json_tool_indices] return real_tools if real_tools else None @@ -1974,9 +1825,7 @@ def _transform_response( # noqa: PLR0915 chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} content_str = "" tools: List[ChatCompletionToolCallChunk] = [] - reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = ( - None - ) + reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = None citationsContentBlocks: Optional[List[CitationsContentBlock]] = None if message is not None: @@ -1995,17 +1844,11 @@ def _transform_response( # noqa: PLR0915 provider_specific_fields["citationsContent"] = citationsContentBlocks if provider_specific_fields: - chat_completion_message["provider_specific_fields"] = ( - provider_specific_fields - ) + chat_completion_message["provider_specific_fields"] = provider_specific_fields if reasoningContentBlocks is not None: - chat_completion_message["reasoning_content"] = ( - self._transform_reasoning_content(reasoningContentBlocks) - ) - chat_completion_message["thinking_blocks"] = ( - self._transform_thinking_blocks(reasoningContentBlocks) - ) + chat_completion_message["reasoning_content"] = self._transform_reasoning_content(reasoningContentBlocks) + chat_completion_message["thinking_blocks"] = self._transform_thinking_blocks(reasoningContentBlocks) chat_completion_message["content"] = content_str filtered_tools = self._filter_json_mode_tools( json_mode=json_mode, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index ed600b26e832..0efc3dab638f 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -8,7 +8,6 @@ 2. If user is in budget 3. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget """ - import asyncio import re import time @@ -415,9 +414,9 @@ async def common_checks( # noqa: PLR0915 model=_model, team_object=team_object, llm_router=llm_router, - team_model_aliases=( - valid_token.team_model_aliases if valid_token else None - ), + team_model_aliases=valid_token.team_model_aliases + if valid_token + else None, ): raise ProxyException( message=f"Team not allowed to access model. Team={team_object.team_id}, Model={_model}. Allowed team models = {team_object.models}", @@ -3079,7 +3078,10 @@ async def _team_max_budget_check( BudgetExceededError if the team is over it's max budget. Triggers a budget alert if the team is over it's max budget. """ - if team_object is not None and team_object.max_budget is not None: + if ( + team_object is not None + and team_object.max_budget is not None + ): from litellm.proxy.proxy_server import get_current_spend # Read spend from cross-pod counter (Redis-first) or cached object (fallback) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index bcfaed243981..5db919659eda 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -66,8 +66,10 @@ async def reset_budget_for_litellm_team_members( try: from litellm.proxy.proxy_server import spend_counter_cache - memberships = await self.prisma_client.db.litellm_teammembership.find_many( - where={"budget_id": {"in": budget_ids}} + memberships = ( + await self.prisma_client.db.litellm_teammembership.find_many( + where={"budget_id": {"in": budget_ids}} + ) ) for m in memberships: counter_key = f"spend:team_member:{m.user_id}:{m.team_id}" @@ -572,11 +574,7 @@ async def _reset_budget_common( counter_key = None if item_type == "key" and hasattr(item, "token") and item.token is not None: counter_key = f"spend:key:{item.token}" - elif ( - item_type == "team" - and hasattr(item, "team_id") - and item.team_id is not None - ): + elif item_type == "team" and hasattr(item, "team_id") and item.team_id is not None: counter_key = f"spend:team:{item.team_id}" if counter_key is not None: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 02d7b9baf4a4..a5a62813b9fd 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -722,9 +722,9 @@ async def _setup_role_mappings() -> Optional["RoleMappings"]: import ast try: - generic_user_role_mappings_data: Dict[LitellmUserRoles, List[str]] = ( - ast.literal_eval(generic_role_mappings) - ) + generic_user_role_mappings_data: Dict[ + LitellmUserRoles, List[str] + ] = ast.literal_eval(generic_role_mappings) if isinstance(generic_user_role_mappings_data, dict): from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings @@ -827,9 +827,7 @@ async def get_generic_sso_response( ], # sso specific jwt handler - used for restricted sso group access control generic_client_id: str, redirect_url: str, -) -> Tuple[ - Union[OpenID, dict], Optional[dict], Optional[dict] -]: # (result, received_response, access_token_payload) +) -> Tuple[Union[OpenID, dict], Optional[dict], Optional[dict]]: # (result, received_response, access_token_payload) # make generic sso provider from fastapi_sso.sso.base import DiscoveryDocument from fastapi_sso.sso.generic import create_provider @@ -881,9 +879,9 @@ def response_convertor(response, client): verbose_proxy_logger.debug("calling generic_sso.verify_and_process") additional_generic_sso_headers_dict = _parse_generic_sso_headers() - code_verifier: Optional[str] = ( - None # assigned inside try; initialized for type tracking - ) + code_verifier: Optional[ + str + ] = None # assigned inside try; initialized for type tracking access_token_payload: Optional[dict] = None # decoded JWT access token claims try: @@ -1233,11 +1231,9 @@ async def _sync_user_role_from_jwt_role_map( user_info.user_role = mapped_role.value await user_api_key_cache.async_set_cache( key=user_info.user_id, - value=( - user_info.model_dump() - if hasattr(user_info, "model_dump") - else dict(user_info) - ), + value=user_info.model_dump() + if hasattr(user_info, "model_dump") + else dict(user_info), ) @@ -1263,9 +1259,9 @@ def apply_user_info_values_to_sso_user_defined_values( else: # SSO didn't provide a valid role, fall back to DB role or default if user_info is None or user_info.user_role is None: - user_defined_values["user_role"] = ( - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value - ) + user_defined_values[ + "user_role" + ] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value verbose_proxy_logger.debug( "No SSO or DB role found, using default: INTERNAL_USER_VIEW_ONLY" ) @@ -1373,14 +1369,12 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa: ) elif generic_client_id is not None: - result, received_response, access_token_payload = ( - await get_generic_sso_response( - request=request, - jwt_handler=jwt_handler, - generic_client_id=generic_client_id, - redirect_url=redirect_url, - sso_jwt_handler=sso_jwt_handler, - ) + result, received_response, access_token_payload = await get_generic_sso_response( + request=request, + jwt_handler=jwt_handler, + generic_client_id=generic_client_id, + redirect_url=redirect_url, + sso_jwt_handler=sso_jwt_handler, ) if result is None: @@ -1703,9 +1697,9 @@ async def insert_sso_user( if user_defined_values.get("max_budget") is None: user_defined_values["max_budget"] = litellm.max_internal_user_budget if user_defined_values.get("budget_duration") is None: - user_defined_values["budget_duration"] = ( - litellm.internal_user_budget_duration - ) + user_defined_values[ + "budget_duration" + ] = litellm.internal_user_budget_duration if user_defined_values["user_role"] is None: user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY @@ -3348,9 +3342,9 @@ async def get_microsoft_callback_response( # if user is trying to get the raw sso response for debugging, return the raw sso response if return_raw_sso_response: - original_msft_result[MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY] = ( - user_team_ids - ) + original_msft_result[ + MicrosoftSSOHandler.GRAPH_API_RESPONSE_KEY + ] = user_team_ids original_msft_result["app_roles"] = app_roles return original_msft_result or {} @@ -3469,9 +3463,9 @@ async def get_user_groups_from_graph_api( # Fetch user membership from Microsoft Graph API all_group_ids = [] - next_link: Optional[str] = ( - MicrosoftSSOHandler.graph_api_user_groups_endpoint - ) + next_link: Optional[ + str + ] = MicrosoftSSOHandler.graph_api_user_groups_endpoint auth_headers = {"Authorization": f"Bearer {access_token}"} page_count = 0 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 317b8f761792..ab73d44accaa 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -639,9 +639,9 @@ def generate_feedback_box(): server_root_path = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() -premium_user_data: Optional["EnterpriseLicenseData"] = ( - _license_check.airgapped_license_data -) +premium_user_data: Optional[ + "EnterpriseLicenseData" +] = _license_check.airgapped_license_data global_max_parallel_request_retries_env: Optional[str] = os.getenv( "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES" ) @@ -1524,9 +1524,9 @@ async def root_redirect(): config_agents: Optional[List[AgentConfig]] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional["ClientSession"] = ( - None # Global shared session for connection reuse -) +shared_aiohttp_session: Optional[ + "ClientSession" +] = None # Global shared session for connection reuse user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1537,13 +1537,13 @@ async def root_redirect(): dual_cache=user_api_key_cache ) litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter) -redis_usage_cache: Optional[RedisCache] = ( - None # redis cache used for tracking spend, tpm/rpm limits -) +redis_usage_cache: Optional[ + RedisCache +] = None # redis cache used for tracking spend, tpm/rpm limits polling_via_cache_enabled: Union[Literal["all"], List[str], bool] = False -native_background_mode: List[str] = ( - [] -) # Models that should use native provider background mode instead of polling +native_background_mode: List[ + str +] = [] # Models that should use native provider background mode instead of polling polling_cache_ttl: int = 3600 # Default 1 hour TTL for polling cache user_custom_auth = None user_custom_key_generate = None @@ -1714,7 +1714,9 @@ async def get_current_spend(counter_key: str, fallback_spend: float) -> float: # 1. Try Redis first (cross-pod authoritative) if spend_counter_cache.redis_cache is not None: try: - val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key) + val = await spend_counter_cache.redis_cache.async_get_cache( + key=counter_key + ) if val is not None: return float(val) except Exception as e: @@ -1818,7 +1820,9 @@ async def _init_and_increment_spend_counter( key=counter_key, value=base_spend ) - await spend_counter_cache.async_increment_cache(key=counter_key, value=increment) + await spend_counter_cache.async_increment_cache( + key=counter_key, value=increment + ) async def update_cache( # noqa: PLR0915 @@ -2027,9 +2031,9 @@ async def _update_team_cache(): _id = "team_id:{}".format(team_id) try: # Fetch the existing cost for the given user - existing_spend_obj: Optional[LiteLLM_TeamTable] = ( - await user_api_key_cache.async_get_cache(key=_id) - ) + existing_spend_obj: Optional[ + LiteLLM_TeamTable + ] = await user_api_key_cache.async_get_cache(key=_id) if existing_spend_obj is None: # do nothing if team not in api key cache return @@ -2150,9 +2154,11 @@ def run_ollama_serve(): with open(os.devnull, "w") as devnull: subprocess.Popen(command, stdout=devnull, stderr=devnull) except Exception as e: - verbose_proxy_logger.debug(f""" + verbose_proxy_logger.debug( + f""" LiteLLM Warning: proxy started with `ollama` model\n`ollama serve` failed with Exception{e}. \nEnsure you run `ollama serve` - """) + """ + ) def _get_process_rss_mb() -> Optional[float]: @@ -5200,10 +5206,10 @@ async def _init_guardrails_in_db(self, prisma_client: PrismaClient): ) try: - guardrails_in_db: List[Guardrail] = ( - await GuardrailRegistry.get_all_guardrails_from_db( - prisma_client=prisma_client - ) + guardrails_in_db: List[ + Guardrail + ] = await GuardrailRegistry.get_all_guardrails_from_db( + prisma_client=prisma_client ) verbose_proxy_logger.debug( "guardrails from the DB %s", str(guardrails_in_db) @@ -5585,9 +5591,9 @@ async def initialize( # noqa: PLR0915 user_api_base = api_base dynamic_config[user_model]["api_base"] = api_base if api_version: - os.environ["AZURE_API_VERSION"] = ( - api_version # set this for azure - litellm can read this from the env - ) + os.environ[ + "AZURE_API_VERSION" + ] = api_version # set this for azure - litellm can read this from the env if max_tokens: # model-specific param dynamic_config[user_model]["max_tokens"] = max_tokens if temperature: # model-specific param @@ -5924,9 +5930,9 @@ def _validate_redis_transaction_buffer_config( """ from litellm.secret_managers.main import str_to_bool - _use_redis_transaction_buffer: Optional[Union[bool, str]] = ( - general_settings.get("use_redis_transaction_buffer", False) - ) + _use_redis_transaction_buffer: Optional[ + Union[bool, str] + ] = general_settings.get("use_redis_transaction_buffer", False) if isinstance(_use_redis_transaction_buffer, str): _use_redis_transaction_buffer = str_to_bool(_use_redis_transaction_buffer) @@ -11312,11 +11318,14 @@ async def login_v2(request: Request): # noqa: PLR0915 litellm_dashboard_ui += "/ui/" litellm_dashboard_ui += "?login=success" + # Token is included in the response body so the UI can set a JS-accessible + # cookie even when a reverse proxy (e.g. nginx-ingress) adds HttpOnly to the + # server-set cookie, which would otherwise cause an infinite login redirect. json_response = JSONResponse( - content={"redirect_url": litellm_dashboard_ui}, + content={"redirect_url": litellm_dashboard_ui, "token": jwt_token}, status_code=status.HTTP_200_OK, ) - json_response.set_cookie(key="token", value=jwt_token, httponly=True) + json_response.set_cookie(key="token", value=jwt_token) return json_response except Exception as e: verbose_proxy_logger.exception( @@ -12521,9 +12530,9 @@ async def get_config_list( hasattr(sub_field_info, "description") and sub_field_info.description is not None ): - nested_fields[idx].field_description = ( - sub_field_info.description - ) + nested_fields[ + idx + ].field_description = sub_field_info.description idx += 1 _stored_in_db = None diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 74c7f9bca5f3..ca70b4bfa8ea 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -28,7 +28,7 @@ def test_get_api_key(): assert get_api_key( custom_litellm_key_header=None, api_key=bearer_token, - azure_api_key_header=None, + AZURE_AI_API_KEY_header=None, anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -59,7 +59,7 @@ def test_get_api_key_with_custom_litellm_key_header( assert get_api_key( custom_litellm_key_header=custom_litellm_key_header, api_key=None, - azure_api_key_header=None, + AZURE_AI_API_KEY_header=None, anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -371,7 +371,7 @@ async def test_proxy_admin_expired_key_from_cache(): await _user_api_key_auth_builder( request=request, api_key=f"Bearer {api_key}", # Add Bearer prefix - azure_api_key_header="", + AZURE_AI_API_KEY_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None, @@ -842,7 +842,7 @@ async def test_user_api_key_auth_builder_no_blocking_calls(): await _user_api_key_auth_builder( request=request, api_key=f"Bearer {api_key}", - azure_api_key_header="", + AZURE_AI_API_KEY_header="", anthropic_api_key_header=None, google_ai_studio_api_key_header=None, azure_apim_header=None,