Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
bfa4753
fix(proxy): persist periodic reload schedule state so status survives…
ryan-crabbe-berri Jul 30, 2026
b7ac3e2
fix(proxy): compare reload requests against pod data age seeded at boot
ryan-crabbe-berri Jul 30, 2026
1101489
fix(proxy): scope reload persistence to the model cost map and seed t…
ryan-crabbe-berri Jul 30, 2026
2887eb3
refactor(proxy): drop the legacy force_reload backfill from the reloa…
ryan-crabbe-berri Jul 30, 2026
a7a0c22
fix(proxy): stamp reload timestamps at the precision they are stored at
ryan-crabbe-berri Jul 30, 2026
e79109c
fix(proxy): identify manual reloads by revision instead of comparing …
ryan-crabbe-berri Jul 31, 2026
9839b95
fix(proxy): seed the applied reload revision at startup
ryan-crabbe-berri Jul 31, 2026
6ec9d37
style(tests): revert incidental reformatting of test_proxy_server.py
ryan-crabbe-berri Aug 3, 2026
ecb3d58
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
ryan-crabbe-berri Aug 3, 2026
19b6570
fix(proxy): serve an outstanding reload request on a booting pod
ryan-crabbe-berri Aug 3, 2026
56e50e8
fix(proxy): accept a reload interval still encoded as JSON text
ryan-crabbe-berri Aug 3, 2026
46a4f91
fix(proxy): cancel a reload schedule without resetting the revision
ryan-crabbe-berri Aug 4, 2026
a656094
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
ryan-crabbe-berri Aug 4, 2026
d66cfab
fix(proxy): null the interval in JSON so cancelling keeps the revision
ryan-crabbe-berri Aug 4, 2026
d733385
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
ryan-crabbe-berri Aug 4, 2026
d4df611
fix(ui): match the CI-generated user_role union order in schema.d.ts
ryan-crabbe-berri Aug 4, 2026
fd10f95
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
ryan-crabbe-berri Aug 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_Config" ADD COLUMN IF NOT EXISTS "last_run_at" TIMESTAMP(3),
ADD COLUMN IF NOT EXISTS "reload_requested_at" TIMESTAMP(3);
2 changes: 2 additions & 0 deletions litellm-proxy-extras/litellm_proxy_extras/schema.prisma
Original file line number Diff line number Diff line change
Expand Up @@ -601,6 +601,8 @@ model LiteLLM_TagTable {
model LiteLLM_Config {
param_name String @id
param_value Json?
last_run_at DateTime?
reload_requested_at DateTime?
}

// View spend, model, api_key per request
Expand Down
8 changes: 8 additions & 0 deletions litellm/litellm_core_utils/get_model_cost_map.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import json
import os
from datetime import datetime, timezone
from importlib.resources import files
from typing import Dict, List, Optional

Expand Down Expand Up @@ -169,6 +170,7 @@ class ModelCostMapSourceInfo:
url: Optional[str] = None
is_env_forced: bool = False
fallback_reason: Optional[str] = None
loaded_at: "datetime | None" = None


# Module-level singleton tracking the source of the current cost map
Expand All @@ -193,6 +195,11 @@ def get_model_cost_map_source_info() -> dict:
}


def get_model_cost_map_loaded_at() -> "datetime | None":
"""When this process last loaded its cost map, stamped at the start of every load"""
return _cost_map_source_info.loaded_at


def _expand_model_aliases(model_cost: dict) -> dict:
"""
Expand ``aliases`` lists in model cost entries into top-level entries.
Expand Down Expand Up @@ -272,6 +279,7 @@ def get_model_cost_map(url: str) -> dict:
The full backup dict is only parsed when it must be *returned* as a
fallback — it is never held in memory long-term.
"""
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
Expand Down
198 changes: 198 additions & 0 deletions litellm/proxy/common_utils/periodic_reload_schedule.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
"""
Persistence for the admin-configured periodic model cost map reload schedule stored in
``LiteLLM_Config``.

Field ownership is split by writer so concurrent writers never overwrite each other:
the schedule endpoints own the ``param_value`` JSON (``interval_hours``), while the
reload job and the manual reload endpoints own the dedicated ``last_run_at`` /
``reload_requested_at`` columns. ``last_run_at`` lives in the row rather than process
memory so the Admin UI still reports the last execution after a restart and across pods.
``reload_requested_at`` fans a manual reload out to every pod without any pod having to
write the flag back down, so no pod can starve another of the request. Each pod tracks
only when its own in-memory copy was loaded (seeded at boot) and reloads whenever that
copy is older than the request or the configured interval.
"""

from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import (
TYPE_CHECKING,
Protocol,
TypedDict,
cast, # noqa: TID251 # prisma table access is untyped (PrismaWrapper.__getattr__)
)

from pydantic import BaseModel, ConfigDict, ValidationError

from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy.utils import PrismaClient, invalidate_config_param
from litellm.repositories.config_repository import ConfigRepository

if TYPE_CHECKING:
from prisma.models import LiteLLM_Config

MODEL_COST_MAP_RELOAD_PARAM_NAME = "model_cost_map_reload_config"


class _ConfigRowWrite(TypedDict, total=False):
param_name: str
param_value: str
last_run_at: datetime
reload_requested_at: datetime


class _ConfigUpsertData(TypedDict):
create: _ConfigRowWrite
update: _ConfigRowWrite


class _ConfigTable(Protocol):
async def find_unique(self, where: Mapping[str, str]) -> "LiteLLM_Config | None": ...

async def upsert(self, where: Mapping[str, str], data: _ConfigUpsertData) -> "LiteLLM_Config": ...

async def update_many(self, data: _ConfigRowWrite, where: Mapping[str, str]) -> int: ...


def _config_table(prisma_client: PrismaClient) -> _ConfigTable:
return cast(_ConfigTable, ConfigRepository(prisma_client).table) # cast-ok: prisma table is untyped (Any)


@dataclass(frozen=True, slots=True)
class ReloadSchedule:
interval_hours: int | None = None
reload_requested_at: datetime | None = None
last_run_at: datetime | None = None


class ReloadScheduleStatus(TypedDict):
scheduled: bool
interval_hours: int | None
last_run: str | None
next_run: str | None


class _IntervalConfig(BaseModel):
model_config = ConfigDict(strict=True)

interval_hours: int | None = None


def utc_now() -> datetime:
return datetime.now(timezone.utc)


def _parse_interval_hours(param_value: object) -> int | None:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good to have explicit checking (input is object -> output is verified value) instead of force casting. Love it

try:
return _IntervalConfig.model_validate(param_value).interval_hours
except ValidationError:
return None


def _as_utc(value: datetime | None) -> datetime | None:
if value is None:
Comment thread
veria-ai[bot] marked this conversation as resolved.
Resolved
return None
return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)


def parse_reload_schedule(row: "LiteLLM_Config") -> ReloadSchedule:
return ReloadSchedule(
interval_hours=_parse_interval_hours(row.param_value),
reload_requested_at=_as_utc(row.reload_requested_at),
last_run_at=_as_utc(row.last_run_at),
)


def next_run_at(schedule: ReloadSchedule) -> datetime | None:
if schedule.interval_hours is None or schedule.last_run_at is None:
return None
return schedule.last_run_at + timedelta(hours=schedule.interval_hours)


def reload_schedule_status(schedule: ReloadSchedule | None) -> ReloadScheduleStatus:
if schedule is None:
return {"scheduled": False, "interval_hours": None, "last_run": None, "next_run": None}
next_run = next_run_at(schedule)
return {
"scheduled": schedule.interval_hours is not None,
"interval_hours": schedule.interval_hours,
"last_run": schedule.last_run_at.isoformat() if schedule.last_run_at is not None else None,
"next_run": next_run.isoformat() if next_run is not None else None,
}


def pod_reload_is_due(
*,
schedule: ReloadSchedule,
pod_data_loaded_at: datetime,
current_time: datetime,
description: str,
) -> bool:
"""
Whether this pod should reload now: when its in-memory data (loaded at boot or at its
last reload) is older than a manual request or the configured interval. A schedule that
has never run anywhere fires immediately so the first run is not one interval away
"""
if schedule.reload_requested_at is not None and schedule.reload_requested_at > pod_data_loaded_at:
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
verbose_proxy_logger.info("%s reload triggered by manual reload request", description)
return True
if schedule.interval_hours is None:
return False
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
if schedule.last_run_at is None:
verbose_proxy_logger.info("%s reload triggered - schedule has never run", description)
return True

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Failed stamp causes reload loop

Medium Severity

When an interval schedule has never persisted last_run_at, pod_reload_is_due keeps returning true on every tick without looking at pod_data_loaded_at. If _check_and_reload_model_cost_map fetches and swaps pricing but record_reload_run errors, the row stays unstamped and the pod refetches on each periodic_reload_job interval.

Additional Locations (1)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit d4df611. Configure here.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think worst case of refetching is no big deal. Brief DB outages or failovers are not frequent anyways

hours_since_data_loaded = (current_time - pod_data_loaded_at).total_seconds() / 3600
if hours_since_data_loaded < schedule.interval_hours:
return False
verbose_proxy_logger.info(
"%s reload triggered by interval. Hours since data loaded: %.2f, Interval: %s",
description,
hours_since_data_loaded,
schedule.interval_hours,
)
return True


async def read_reload_schedule(prisma_client: PrismaClient, param_name: str) -> ReloadSchedule | None:
row = await _config_table(prisma_client).find_unique(where={"param_name": param_name})
if row is None:
return None
return parse_reload_schedule(row)


async def write_reload_interval(prisma_client: PrismaClient, param_name: str, interval_hours: int) -> None:
"""Admin-owned write: replaces ``param_value`` without touching the job-owned columns"""
param_value = safe_dumps({"interval_hours": interval_hours})
await _config_table(prisma_client).upsert(
where={"param_name": param_name},
data={
"create": {"param_name": param_name, "param_value": param_value},
"update": {"param_value": param_value},
},
)
await invalidate_config_param(param_name)


async def record_reload_run(prisma_client: PrismaClient, param_name: str, ran_at: datetime) -> None:
"""Job-owned write after this pod reloaded: stamps the shared last run only if the row
still exists, so a schedule deleted mid-poll is not resurrected"""
await _config_table(prisma_client).update_many(
data={"last_run_at": ran_at},
where={"param_name": param_name},
)
await invalidate_config_param(param_name)


async def record_manual_reload(prisma_client: PrismaClient, param_name: str, ran_at: datetime) -> None:
"""After a manual in-pod reload: stamps the shared last run and the request every other
pod compares against on its next poll"""
await _config_table(prisma_client).upsert(
where={"param_name": param_name},
data={
"create": {"param_name": param_name, "last_run_at": ran_at, "reload_requested_at": ran_at},
"update": {"last_run_at": ran_at, "reload_requested_at": ran_at},
},
)
await invalidate_config_param(param_name)
Loading
Loading