diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce23f394..22193036 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -118,6 +118,12 @@ jobs: path: ~/.cache/huggingface/datasets key: ${{ runner.os }}-datasets-${{ hashFiles('packages/gsm8k') }}-${{ hashFiles('packages/hotpotqa') }}-${{ hashFiles('packages/labbench') }}-${{ hashFiles('packages/lfrqa') }}-${{ hashFiles('packages/notebook') }} restore-keys: ${{ runner.os }}-datasets- + # As of litellm version 1.80.11 from https://github.com/BerriAI/litellm/pull/18070, + # tiktoken encodings are lazily loaded. This breaks our VCR cassette caching + # in fresh CI environments, so for backwards compatibility, + # we download tiktoken encodings before running tests + - name: Pre-download tiktoken encoding + run: uv run python -c "import tiktoken; tiktoken.get_encoding('cl100k_base')" - run: uv run pytest -n 16 --dist=loadfile # auto only launches 8 workers in CI, despite runners have 16 cores env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/pyproject.toml b/pyproject.toml index 3f578787..851b2b2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dev = [ "fhaviary[image,llm,server,typing,xml]", "ipython>=8", # Pin to keep recent "jupyter>=1.0.0", # For running notebooks - "litellm>=1.71", # Lower pin for aiohttp transport adoption + "litellm>=1.80.11", # Lower pin for lazy loading of tiktoken encoding "mypy>=1.19", # Pin for better TypeAlias assignment checking "numpy>=1", # Pin to keep recent "prek", diff --git a/tests/cassettes/TestParallelism.test_dummyenv_using_empty_params.yaml b/tests/cassettes/TestParallelism.test_dummyenv_using_empty_params.yaml index b282ce60..332d3c9d 100644 --- a/tests/cassettes/TestParallelism.test_dummyenv_using_empty_params.yaml +++ b/tests/cassettes/TestParallelism.test_dummyenv_using_empty_params.yaml @@ -6,7 +6,7 @@ interactions: to print.","title":"Story","type":"string"}},"required":["story"]}},{"name":"cast_float","description":"Cast the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}},{"name":"cast_int","description":"Cast the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}},{"name":"get_random_int","description":"Get - a random integer in 1 to 10.","parameters":{"type":"object","properties":{},"required":[]}}]}],"toolConfig":{"functionCallingConfig":{"mode":"ANY"}},"generationConfig":{}}' + a random integer in 1 to 10.","parameters":{}}]}],"toolConfig":{"functionCallingConfig":{"mode":"ANY"}},"generationConfig":{}}' headers: accept: - "*/*" @@ -15,25 +15,29 @@ interactions: connection: - keep-alive content-length: - - "872" + - "827" content-type: - application/json host: - generativelanguage.googleapis.com user-agent: - - litellm/1.70.4 + - litellm/1.80.11 method: POST - uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent?key=%3CFILTERED%3E + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent?key=%3CFILTERED%3E response: body: string: !!binary | - H4sIAAAAAAAC/61RXU/CMBR9369Y+swITMbAN4OakIgQXIyJMeTCLqOxa2fbacjCf7f7gg5f3cPS - 3nPuPb3nFI7rkh3wmMagUZFb991UXLeo/iUmuEauDdCWTDEDqS/c+iuss6Hsc77TVPAZMNZpbnAO - KZo6SVBvpNEX6YYamd41D2RSKhWnDmDfLuePSzeRglXzUxEjI0391BLInnKqDmsEJXhJe4mWq7M4 - ge/kSSSZFNtS2xv1/elwMroZB/7AH4d+OEVvEDqteCVLcgUJLlCDMRLOGxMzJM10JD6Rz0ReGRkO - ayHL9w4eNLAWGli3c9z7M1XdG03K7DispMz6wKg+ljtGD2+RZbCZ33lU65FjWXn9xH8SC7paTpNM - HdYrSkXrVBJMTU7esB94ewbqUA0kElUmuMJ5XHKW6vEZVvnXYpsn6XS+Ogq+ntz9EOfk/ALwP/QS - 3QIAAA== + H4sIAAAAAAAC/11SUXOiMBh891cwPN1Nr7ViUbw3RT2o0gJ6tuXmphMwQBQSLwlSdfzvF1ArlAeG + 2d18+7HZQ0OS5ADgJVoCDpn8U/ojEEk6lO+CI5hDzAVxgQS4AZRftafnUPkWkjDDAUcE6yBJaofP + PAYpFLgcQf5OhT9J35Gw+fFVB2hUOB2ONeJY18k8JlkU8xmKMOAZLSfr1Bn0X6emgvPmNtd942aR + dluv484MOM+uprARG+RW9wWgNRjH+qPt7jN/hbd09atv7WBqtTM7Ta1YfXnik4k3cmnPeJsa2raj + tTiJyNSIdoY77qnb0QBymE5yLdeQ/+Co27eBgzNvHGjh/XqHfaiiYW/lqs19Nhh6mDYnJPf+IcXp + pDfhqG9E5ou2e+h6Hd7VHz+aRM1MQ7HWXSPU1z1MwsGTv1LNfTZproNRazGfWOm4P/S8ztCze/Hv + HbPbWDejXAvkSizXxP5e05IpScp0UrKEyUX+GaccIoxY7ELACC5ks/mz/XkpMsJL+CHg+y96CzIG + onKuVcyVIoghFYVaSpceSIEowjf2/e7kWSxXriVnxUkLciAKCD6bIm8oSTd8TtYQ6yQrC9jqtE++ + lcLWBcqZ54SDpEYprfPO1cFsKGxRUm1ypeQiIZAgviv+aj56nVe6KQzqe11ybFTivpSyvmNbbZzz + Pl3BAlKGTllHMBVp3ip36m2YABaXhjKFbEMwg+ay0Bh27AMPo6n9/LGyAnsPffPBIXLj2PgP5567 + zssDAAA= headers: Alt-Svc: - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 @@ -42,11 +46,11 @@ interactions: Content-Type: - application/json; charset=UTF-8 Date: - - Sat, 14 Jun 2025 18:36:43 GMT + - Mon, 05 Jan 2026 17:42:52 GMT Server: - scaffolding on HTTPServer2 Server-Timing: - - gfet4t7; dur=472 + - gfet4t7; dur=747 Transfer-Encoding: - chunked Vary: diff --git a/tests/test_envs.py b/tests/test_envs.py index 9745311a..a4354b31 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -648,7 +648,12 @@ async def test_dummyenv_using_empty_params(self, dummy_env: DummyEnv) -> None: expected_tool_call_fn = ToolCall.from_tool(tools[-1]).function dummy_env.state = dummy_env.State(messages=obs) - selector = ToolSelector("gemini/gemini-1.5-flash") + # NOTE: originally this was Gemini 1.5 Flash, + # but as of 1/5/2026 it was deprecated from Google's API. + # Per https://github.com/BerriAI/litellm/issues/7634#issuecomment-2810321829 + # this test isn't necessary anymore, but let's keep it around anyways + # as a regression test + selector = ToolSelector("gemini/gemini-2.5-flash") assert any(not t.info.get_properties() for t in tools), ( "Test requires empty properties" diff --git a/uv.lock b/uv.lock index 3375d0ba..cb9f6ddc 100644 --- a/uv.lock +++ b/uv.lock @@ -1219,7 +1219,7 @@ requires-dist = [ { name = "httpx-aiohttp" }, { name = "ipython", marker = "extra == 'dev'", specifier = ">=8" }, { name = "jupyter", marker = "extra == 'dev'", specifier = ">=1.0.0" }, - { name = "litellm", marker = "extra == 'dev'", specifier = ">=1.71" }, + { name = "litellm", marker = "extra == 'dev'", specifier = ">=1.80.11" }, { name = "litellm", marker = "extra == 'llm'", specifier = ">=1.49.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.19" }, { name = "numpy", marker = "extra == 'dev'", specifier = ">=1" }, @@ -2366,7 +2366,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.80.10" +version = "1.80.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -2384,9 +2384,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dd/44/0aaa7449e7c4aa05668ec03f1f68a01b1e476591071d9659a68db19371a2/litellm-1.80.10.tar.gz", hash = "sha256:4a4aff7558945c2f7e5c6523e67c1b5525a46b10b0e1ad6b8f847cb13b16779e", size = 12764777, upload-time = "2025-12-14T02:07:05.362Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/47/be6cd7b356418ca8bef3b843507940ce77b76ef2dfe515f2b4ba9b461ff0/litellm-1.80.11.tar.gz", hash = "sha256:c9fc63e7acb6360363238fe291bcff1488c59ff66020416d8376c0ee56414a19", size = 13189510, upload-time = "2025-12-22T12:47:29.181Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/a9/4814b6aa58f6705df2831eaadeb5bc8240684c8c9d5964245212f85049d1/litellm-1.80.10-py3-none-any.whl", hash = "sha256:9b3e561efaba0eb1291cb1555d3dcb7283cf7f3cb65aadbcdb42e2a8765898c8", size = 11264240, upload-time = "2025-12-14T02:07:02.414Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/9e637344f24f3fe0e8039cd2337389fe05e0d31f518bc3e0a5cdbe45784a/litellm-1.80.11-py3-none-any.whl", hash = "sha256:406283d66ead77dc7ff0e0b2559c80e9e497d8e7c2257efb1cb9210a20d09d54", size = 11456346, upload-time = "2025-12-22T12:47:26.469Z" }, ] [[package]]