ci: added typechecking

.github/workflows/typecheck.yml

@@ -0,0 +1,36 @@
+name: Typecheck
+
+# If a pull-request is pushed then cancel all previously running jobs related
+# to that pull-request
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+      - master
+
+env:
+  PY_COLORS: 1
+
+jobs:
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+      - name: Install poetry
+        run: pipx install poetry
+      - uses: actions/[email protected]
+        with:
+          python-version: '3.12'
+      - run: poetry install
+      - name: Run mypy
+        # for some strange reason, the config specified in pyproject.toml is not respected
+        run: poetry run mypy . --ignore-missing-imports

cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py

@@ -22,7 +22,7 @@
 # use the auto eval Factuality() evaluator
 
 print("calling evaluator")
-evaluator = Factuality()
+evaluator = Factuality()  # type: ignore
 result = evaluator(
     output=response.choices[0]["message"][
         "content"

cookbook/litellm-ollama-docker-image/test.py

@@ -1,14 +1,15 @@
 import openai
+from litellm import completion
 
-api_base = f"http://0.0.0.0:8000"
+api_base = "http://0.0.0.0:8000"
 
-openai.api_base = api_base
+openai.base_url = api_base
 openai.api_key = "temp-key"
-print(openai.api_base)
+print(openai.base_url)
 
 
-print(f"LiteLLM: response from proxy with streaming")
-response = openai.ChatCompletion.create(
+print("LiteLLM: response from proxy with streaming")
+response = completion(
     model="ollama/llama2",
     messages=[
         {
@@ -22,7 +23,7 @@
 for chunk in response:
     print(f"LiteLLM: streaming response from proxy {chunk}")
 
-response = openai.ChatCompletion.create(
+response = completion(
     model="ollama/llama2",
     messages=[
         {

pyproject.toml

@@ -5,6 +5,9 @@ description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT"
 readme = "README.md"
+packages = [
+    { include = "litellm" }
+]
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.9.7 || >3.9.7"
@@ -55,6 +58,14 @@ litellm = 'litellm:run_server'
 flake8 = "^6.1.0"
 black = "^23.12.0"
 pytest = "^7.4.3"
+mypy = "^1.8.0"
+
+types-requests = "^2.31.0.20240106"
+types-redis = "^4.6.0.20240106"
+types-tabulate = "^0.9.0.20240106"
+types-waitress = "^2.1.4.20240106"
+types-pyyaml = "^6.0.12.12"
+types-setuptools = "^69.0.0.20240115"
 
 [build-system]
 requires = ["poetry-core", "wheel"]
@@ -66,3 +77,5 @@ version_files = [
     "pyproject.toml:^version"
 ]
 
+[tool.mypy]
+ignore_missing_imports = true

tests/test_models.py

@@ -1,9 +1,8 @@
 # What this tests ?
 ## Tests /models and /model/* endpoints
 
-import pytest
-import asyncio
 import aiohttp
+import pytest
 
 
 async def generate_key(session, models=[]):
@@ -55,7 +54,7 @@ async def test_get_models():
 async def add_models(session, model_id="123"):
     url = "http://0.0.0.0:4000/model/new"
     headers = {
-        "Authorization": f"Bearer sk-1234",
+        "Authorization": "Bearer sk-1234",
         "Content-Type": "application/json",
     }
 
@@ -141,7 +140,7 @@ async def test_add_models():
 
 
 @pytest.mark.asyncio
-async def test_get_models():
+async def test_get_models_2():
     """
     Get models user has access to
     """
@@ -160,7 +159,7 @@ async def delete_model(session, model_id="123"):
     """
     url = "http://0.0.0.0:4000/model/delete"
     headers = {
-        "Authorization": f"Bearer sk-1234",
+        "Authorization": "Bearer sk-1234",
         "Content-Type": "application/json",
     }
     data = {"id": model_id}