diff --git a/.circleci/config.yml b/.circleci/config.yml index 2072578709d3..4649e90f1e9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,9 +47,9 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" + pip install openai==1.54.0 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" pip install "respx==0.21.1" pip install fastapi @@ -165,9 +165,9 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" + pip install openai==1.54.0 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" pip install "respx==0.21.1" pip install fastapi @@ -264,9 +264,9 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" + pip install openai==1.54.0 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" pip install "respx==0.21.1" pip install fastapi @@ -367,7 +367,7 @@ jobs: # Store test results - store_test_results: path: test-results - + - persist_to_workspace: root: . paths: @@ -375,10 +375,10 @@ jobs: - auth_ui_unit_tests_coverage litellm_router_testing: # Runs all tests with the "router" keyword docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: @@ -417,10 +417,10 @@ jobs: - litellm_router_coverage litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: - checkout @@ -458,9 +458,9 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.54.0 - pip install prisma==0.11.0 - pip install "detect_secrets==1.5.0" + pip install openai==1.54.0 + pip install prisma==0.11.0 + pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" pip install "respx==0.21.1" pip install fastapi @@ -514,10 +514,10 @@ jobs: - litellm_proxy_unit_tests_coverage litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword docker: - - image: cimg/python:3.11 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} working_directory: ~/project steps: @@ -616,7 +616,7 @@ jobs: command: | mv coverage.xml llm_translation_coverage.xml mv .coverage llm_translation_coverage - + # Store test results - store_test_results: path: test-results @@ -660,7 +660,7 @@ jobs: command: | mv coverage.xml batches_coverage.xml mv .coverage batches_coverage - + # Store test results - store_test_results: path: test-results @@ -704,7 +704,7 @@ jobs: command: | mv coverage.xml secret_manager_coverage.xml mv .coverage secret_manager_coverage - + # Store test results - store_test_results: path: test-results @@ -747,7 +747,7 @@ jobs: command: | mv coverage.xml pass_through_unit_tests_coverage.xml mv .coverage pass_through_unit_tests_coverage - + # Store test results - store_test_results: path: test-results @@ -789,7 +789,7 @@ jobs: command: | mv coverage.xml image_gen_coverage.xml mv .coverage image_gen_coverage - + # Store test results - store_test_results: path: test-results @@ -835,7 +835,7 @@ jobs: command: | mv coverage.xml logging_coverage.xml mv .coverage logging_coverage - + # Store test results - store_test_results: path: test-results @@ -875,7 +875,7 @@ jobs: pwd ls python -m pytest -vv tests/local_testing/test_basic_python_version.py - + installing_litellm_on_python_3_13: docker: - image: cimg/python:3.13.1 @@ -973,7 +973,7 @@ jobs: cat docker_output.log exit 1 fi - + build_and_test: machine: image: ubuntu-2204:2023.10.1 @@ -1019,9 +1019,9 @@ jobs: pip install "langfuse>=2.0.0" pip install "logfire==0.29.0" pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema + pip install prisma + pip install fastapi + pip install jsonschema pip install "httpx==0.24.1" pip install "gunicorn==21.2.0" pip install "anyio==3.7.1" @@ -1137,9 +1137,9 @@ jobs: pip install "langfuse>=2.0.0" pip install "logfire==0.29.0" pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema + pip install prisma + pip install fastapi + pip install jsonschema pip install "httpx==0.24.1" pip install "gunicorn==21.2.0" pip install "anyio==3.7.1" @@ -1256,9 +1256,9 @@ jobs: pip install "langfuse>=2.0.0" pip install "logfire==0.29.0" pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema + pip install prisma + pip install fastapi + pip install jsonschema pip install "httpx==0.24.1" pip install "gunicorn==21.2.0" pip install "anyio==3.7.1" @@ -1321,8 +1321,9 @@ jobs: pwd ls python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - # Clean up first container + no_output_timeout: + 120m + # Clean up first container - run: name: Stop and remove first container command: | @@ -1458,8 +1459,9 @@ jobs: name: Run tests command: | python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 - no_output_timeout: 120m - # Clean up first container + no_output_timeout: + 120m + # Clean up first container - run: name: Stop and remove first container command: | @@ -1506,9 +1508,9 @@ jobs: pip install mypy pip install pyarrow pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema + pip install prisma + pip install fastapi + pip install jsonschema pip install "httpx==0.24.1" pip install "anyio==3.7.1" pip install "asyncio==3.4.3" @@ -1610,7 +1612,6 @@ jobs: - codecov/upload: file: ./coverage.xml - publish_to_pypi: docker: - image: cimg/python:3.8 @@ -1637,7 +1638,6 @@ jobs: circleci step halt fi - - run: name: Checkout code command: git checkout $CIRCLE_SHA1 @@ -1726,9 +1726,9 @@ jobs: pip install mypy pip install pyarrow pip install numpydoc - pip install prisma - pip install fastapi - pip install jsonschema + pip install prisma + pip install fastapi + pip install jsonschema pip install "httpx==0.24.1" pip install "anyio==3.7.1" pip install "asyncio==3.4.3" @@ -1779,6 +1779,28 @@ jobs: - store_test_results: path: test-results + test_nonroot_image: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - run: + name: Build Docker image + command: | + docker build -t non_root_image:latest . -f ./docker/Dockerfile.non_root + - run: + name: Install Container Structure Test + command: | + curl -LO https://github.com/GoogleContainerTools/container-structure-test/releases/download/v1.19.3/container-structure-test-linux-amd64 + chmod +x container-structure-test-linux-amd64 + sudo mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test + - run: + name: Run Container Structure Test + command: | + container-structure-test test --image non_root_image:latest --config docker/tests/nonroot.yaml + test_bad_database_url: machine: image: ubuntu-2204:2023.10.1 @@ -1844,10 +1866,10 @@ workflows: - /litellm_.*/ - litellm_assistants_api_testing: filters: - branches: - only: - - main - - /litellm_.*/ + branches: + only: + - main + - /litellm_.*/ - litellm_router_testing: filters: branches: @@ -2014,4 +2036,3 @@ workflows: branches: only: - main - diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 699d48fe0b12..381e9e5433aa 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -1,50 +1,53 @@ {{- if .Values.migrationJob.enabled }} - # This job runs the prisma migrations for the LiteLLM DB. - apiVersion: batch/v1 - kind: Job - metadata: - name: {{ include "litellm.fullname" . }}-migrations - annotations: - argocd.argoproj.io/hook: PreSync - argocd.argoproj.io/hook-delete-policy: BeforeHookCreation # delete old migration on a new deploy in case the migration needs to make updates - checksum/config: {{ toYaml .Values | sha256sum }} - spec: - template: - metadata: - annotations: - {{- with .Values.migrationJob.annotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - containers: - - name: prisma-migrations - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}" - command: ["python", "litellm/proxy/prisma_migration.py"] - workingDir: "/app" - env: - {{- if .Values.db.useExisting }} - - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.usernameKey }} - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.passwordKey }} - - name: DATABASE_HOST - value: {{ .Values.db.endpoint }} - - name: DATABASE_NAME - value: {{ .Values.db.database }} - - name: DATABASE_URL - value: {{ .Values.db.url | quote }} - {{- else }} - - name: DATABASE_URL - value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} - {{- end }} - - name: DISABLE_SCHEMA_UPDATE - value: "false" # always run the migration from the Helm PreSync hook, override the value set - restartPolicy: OnFailure - backoffLimit: {{ .Values.migrationJob.backoffLimit }} +# This job runs the prisma migrations for the LiteLLM DB. +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "litellm.fullname" . }}-migrations + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation # delete old migration on a new deploy in case the migration needs to make updates + checksum/config: {{ toYaml .Values | sha256sum }} +spec: + template: + metadata: + annotations: + {{- with .Values.migrationJob.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: prisma-migrations + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + command: ["python", "litellm/proxy/prisma_migration.py"] + workingDir: "/app" + env: + {{- if .Values.db.useExisting }} + - name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.usernameKey }} + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.passwordKey }} + - name: DATABASE_HOST + value: {{ .Values.db.endpoint }} + - name: DATABASE_NAME + value: {{ .Values.db.database }} + - name: DATABASE_URL + value: {{ .Values.db.url | quote }} + {{- else }} + - name: DATABASE_URL + value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} + {{- end }} + - name: DISABLE_SCHEMA_UPDATE + value: "false" # always run the migration from the Helm PreSync hook, override the value set + restartPolicy: OnFailure + backoffLimit: {{ .Values.migrationJob.backoffLimit }} {{- end }} diff --git a/docker/tests/nonroot.yaml b/docker/tests/nonroot.yaml new file mode 100644 index 000000000000..821b1a105ae1 --- /dev/null +++ b/docker/tests/nonroot.yaml @@ -0,0 +1,18 @@ +schemaVersion: 2.0.0 + +metadataTest: + entrypoint: ["docker/prod_entrypoint.sh"] + user: "nobody" + workdir: "/app" + +fileExistenceTests: + - name: "Prisma Folder" + path: "/usr/local/lib/python3.13/site-packages/prisma/" + shouldExist: true + uid: 65534 + gid: 65534 + - name: "Prisma Schema" + path: "/usr/local/lib/python3.13/site-packages/prisma/schema.prisma" + shouldExist: true + uid: 65534 + gid: 65534 diff --git a/docs/my-website/docs/getting_started.md b/docs/my-website/docs/getting_started.md index e9b2a0db616a..eed9d338479b 100644 --- a/docs/my-website/docs/getting_started.md +++ b/docs/my-website/docs/getting_started.md @@ -86,7 +86,7 @@ LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone from litellm import completion ## set env variables for logging tools -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" +os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings os.environ["HELICONE_API_KEY"] = "your-helicone-key" os.environ["LANGFUSE_PUBLIC_KEY"] = "" os.environ["LANGFUSE_SECRET_KEY"] = "" diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index e5c3fdaa3be4..c65f0f95e6a3 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -399,10 +399,10 @@ LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone from litellm import completion ## set env variables for logging tools +os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings os.environ["HELICONE_API_KEY"] = "your-helicone-key" os.environ["LANGFUSE_PUBLIC_KEY"] = "" os.environ["LANGFUSE_SECRET_KEY"] = "" -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" os.environ["OPENAI_API_KEY"] diff --git a/docs/my-website/docs/langchain/langchain.md b/docs/my-website/docs/langchain/langchain.md index efa6b29250c9..960ac2fd48e3 100644 --- a/docs/my-website/docs/langchain/langchain.md +++ b/docs/my-website/docs/langchain/langchain.md @@ -111,5 +111,30 @@ chat.invoke(messages) +## Use Langchain ChatLiteLLM with Lunary +```python +import os +from langchain.chat_models import ChatLiteLLM +from langchain.schema import HumanMessage +import litellm + +os.environ["LUNARY_PUBLIC_KEY"] = "" # from https://app.lunary.ai/settings +os.environ['OPENAI_API_KEY']="sk-..." + +litellm.success_callback = ["lunary"] +litellm.failure_callback = ["lunary"] + +chat = ChatLiteLLM( + model="gpt-4o" + messages = [ + HumanMessage( + content="what model are you" + ) +] +chat(messages) +``` + +Get more details [here](../observability/lunary_integration.md) + ## Use LangChain ChatLiteLLM + Langfuse Checkout this section [here](../observability/langfuse_integration#use-langchain-chatlitellm--langfuse) for more details on how to integrate Langfuse with ChatLiteLLM. diff --git a/docs/my-website/docs/observability/callbacks.md b/docs/my-website/docs/observability/callbacks.md index b959e8aae7d2..69cb0d053eeb 100644 --- a/docs/my-website/docs/observability/callbacks.md +++ b/docs/my-website/docs/observability/callbacks.md @@ -7,11 +7,11 @@ liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, liteLLM supports: - [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback) +- [Lunary](https://lunary.ai/docs) - [Langfuse](https://langfuse.com/docs) - [LangSmith](https://www.langchain.com/langsmith) - [Helicone](https://docs.helicone.ai/introduction) - [Traceloop](https://traceloop.com/docs) -- [Lunary](https://lunary.ai/docs) - [Athina](https://docs.athina.ai/) - [Sentry](https://docs.sentry.io/platforms/python/) - [PostHog](https://posthog.com/docs/libraries/python) @@ -30,6 +30,7 @@ litellm.success_callback=["posthog", "helicone", "langfuse", "lunary", "athina"] litellm.failure_callback=["sentry", "lunary", "langfuse"] ## set env variables +os.environ['LUNARY_PUBLIC_KEY'] = "" os.environ['SENTRY_DSN'], os.environ['SENTRY_API_TRACE_RATE']= "" os.environ['POSTHOG_API_KEY'], os.environ['POSTHOG_API_URL'] = "api-key", "api-url" os.environ["HELICONE_API_KEY"] = "" diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index 5be4ad641343..8f55c854db88 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -59,7 +59,7 @@ os.environ["LANGSMITH_API_KEY"] = "" # LLM API Keys os.environ['OPENAI_API_KEY']="" -# set langfuse as a callback, litellm will send the data to langfuse +# set langsmith as a callback, litellm will send the data to langsmith litellm.success_callback = ["langsmith"] response = litellm.completion( diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md index 56e74132f789..8d28321c8075 100644 --- a/docs/my-website/docs/observability/lunary_integration.md +++ b/docs/my-website/docs/observability/lunary_integration.md @@ -1,72 +1,78 @@ -# Lunary - Logging and tracing LLM input/output +import Image from '@theme/IdealImage'; -:::tip +# 🌙 Lunary - GenAI Observability -This is community maintained, Please make an issue if you run into a bug -https://github.com/BerriAI/litellm +[Lunary](https://lunary.ai/) is an open-source platform providing [observability](https://lunary.ai/docs/features/observe), [prompt management](https://lunary.ai/docs/features/prompts), and [analytics](https://lunary.ai/docs/features/observe#analytics) to help team manage and improve LLM chatbots. -::: - - -[Lunary](https://lunary.ai/) is an open-source AI developer platform providing observability, prompt management, and evaluation tools for AI developers. +You can reach out to us anytime by [email](mailto:hello@lunary.ai) or directly [schedule a Demo](https://lunary.ai/schedule). -## Use Lunary to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM) -liteLLM provides `callbacks`, making it easy for you to log data depending on the status of your responses. +## Usage with LiteLLM Python SDK +### Pre-Requisites -:::info -We want to learn how we can make the callbacks better! Meet the [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or -join our [discord](https://discord.gg/wuPM9dRgDw) -::: +```shell +pip install litellm lunary +``` -### Using Callbacks +### Quick Start -First, sign up to get a public key on the [Lunary dashboard](https://lunary.ai). +First, get your Lunary public key on the [Lunary dashboard](https://app.lunary.ai/). -Use just 2 lines of code, to instantly log your responses **across all providers** with lunary: +Use just 2 lines of code, to instantly log your responses **across all providers** with Lunary: ```python litellm.success_callback = ["lunary"] litellm.failure_callback = ["lunary"] ``` -Complete code - +Complete code: ```python from litellm import completion -## set env variables -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" - +os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # from https://app.lunary.ai/) os.environ["OPENAI_API_KEY"] = "" -# set callbacks litellm.success_callback = ["lunary"] litellm.failure_callback = ["lunary"] -#openai call response = completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}], + model="gpt-4o", + messages=[{"role": "user", "content": "Hi there 👋"}], user="ishaan_litellm" ) ``` -## Templates - -You can use Lunary to manage prompt templates and use them across all your LLM providers. +### Usage with LangChain ChatLiteLLM +```python +import os +from langchain.chat_models import ChatLiteLLM +from langchain.schema import HumanMessage +import litellm + +os.environ["LUNARY_PUBLIC_KEY"] = "" # from https://app.lunary.ai/settings +os.environ['OPENAI_API_KEY']="sk-..." + +litellm.success_callback = ["lunary"] +litellm.failure_callback = ["lunary"] + +chat = ChatLiteLLM( + model="gpt-4o" + messages = [ + HumanMessage( + content="what model are you" + ) +] +chat(messages) +``` -Make sure to have `lunary` installed: -```bash -pip install lunary -``` +### Usage with Prompt Templates -Then, use the following code to pull templates into Lunary: +You can use Lunary to manage [prompt templates](https://lunary.ai/docs/features/prompts) and use them across all your LLM providers with LiteLLM. ```python from litellm import completion @@ -81,9 +87,93 @@ litellm.success_callback = ["lunary"] result = completion(**template) ``` +### Usage with custom chains +You can wrap your LLM calls inside custom chains, so that you can visualize them as traces. + +```python +import litellm +from litellm import completion +import lunary + +litellm.success_callback = ["lunary"] +litellm.failure_callback = ["lunary"] + +@lunary.chain("My custom chain name") +def my_chain(chain_input): + chain_run_id = lunary.run_manager.current_run_id + response = completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Say 1"}], + metadata={"parent_run_id": chain_run_id}, + ) + + response = completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Say 2"}], + metadata={"parent_run_id": chain_run_id}, + ) + chain_output = response.choices[0].message + return chain_output + +my_chain("Chain input") +``` + + + +## Usage with LiteLLM Proxy Server +### Step1: Install dependencies and set your environment variables +Install the dependencies +```shell +pip install litellm lunary +``` + +Get you Lunary public key from from https://app.lunary.ai/settings +```shell +export LUNARY_PUBLIC_KEY="" +``` + +### Step 2: Create a `config.yaml` and set `lunary` callbacks + +```yaml +model_list: + - model_name: "*" + litellm_params: + model: "*" +litellm_settings: + success_callback: ["lunary"] + failure_callback: ["lunary"] +``` + +### Step 3: Start the LiteLLM proxy +```shell +litellm --config config.yaml +``` + +### Step 4: Make a request + +```shell +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-d '{ + "model": "gpt-4o", + "messages": [ + { + "role": "system", + "content": "You are a helpful math tutor. Guide the user through the solution step by step." + }, + { + "role": "user", + "content": "how can I solve 8x + 7 = -23" + } + ] +}' +``` + +You can find more details about the different ways of making requests to the LiteLLM proxy on [this page](https://docs.litellm.ai/docs/proxy/user_keys) + + ## Support & Talk to Founders -- Meet the Lunary team via [email](mailto:hello@lunary.ai). - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) - Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ diff --git a/docs/my-website/docs/proxy/architecture.md b/docs/my-website/docs/proxy/architecture.md index 4cd23adb5e45..f8804e2d238b 100644 --- a/docs/my-website/docs/proxy/architecture.md +++ b/docs/my-website/docs/proxy/architecture.md @@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem'; 6. [**litellm.completion() / litellm.embedding()**:](../index#litellm-python-sdk) The litellm Python SDK is used to call the LLM in the OpenAI API format (Translation and parameter mapping) 7. **Post-Request Processing**: After the response is sent back to the client, the following **asynchronous** tasks are performed: - - [Logging to LangFuse (logging destination is configurable)](./logging) + - [Logging to Lunary, LangFuse or other logging destinations](./logging) - The [MaxParallelRequestsHandler](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py) updates the rpm/tpm usage for the - Global Server Rate Limit - Virtual Key Rate Limit diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 3b1588da8da1..ab6dcd338ac8 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -2,6 +2,7 @@ Log Proxy input, output, and exceptions using: +- Lunary - Langfuse - OpenTelemetry - GCS, s3, Azure (Blob) Buckets @@ -182,6 +183,55 @@ Found under `kwargs["standard_logging_object"]`. This is a standard payload, log [👉 **Standard Logging Payload Specification**](./logging_spec) +## Lunary +### Step1: Install dependencies and set your environment variables +Install the dependencies +```shell +pip install litellm lunary +``` + +Get you Lunary public key from from https://app.lunary.ai/settings +```shell +export LUNARY_PUBLIC_KEY="" +``` + +### Step 2: Create a `config.yaml` and set `lunary` callbacks + +```yaml +model_list: + - model_name: "*" + litellm_params: + model: "*" +litellm_settings: + success_callback: ["lunary"] + failure_callback: ["lunary"] +``` + +### Step 3: Start the LiteLLM proxy +```shell +litellm --config config.yaml +``` + +### Step 4: Make a request + +```shell +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-d '{ + "model": "gpt-4o", + "messages": [ + { + "role": "system", + "content": "You are a helpful math tutor. Guide the user through the solution step by step." + }, + { + "role": "user", + "content": "how can I solve 8x + 7 = -23" + } + ] +}' +``` + ## Langfuse We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment diff --git a/docs/my-website/docs/tutorials/instructor.md b/docs/my-website/docs/tutorials/instructor.md index aaf76811610f..d972aff9151d 100644 --- a/docs/my-website/docs/tutorials/instructor.md +++ b/docs/my-website/docs/tutorials/instructor.md @@ -1,32 +1,22 @@ # Instructor - Function Calling -Use LiteLLM Router with [jxnl's instructor library](https://github.com/jxnl/instructor) for function calling in prod. +Use LiteLLM with [jxnl's instructor library](https://github.com/jxnl/instructor) for function calling in prod. ## Usage ```python -import litellm -from litellm import Router +import os + import instructor +from litellm import completion from pydantic import BaseModel -litellm.set_verbose = True # 👈 print DEBUG LOGS +os.environ["LITELLM_LOG"] = "DEBUG" # 👈 print DEBUG LOGS -client = instructor.patch( - Router( - model_list=[ - { - "model_name": "gpt-3.5-turbo", openai model name - "litellm_params": { # params for litellm completion/embedding call - e.g.: https://github.com/BerriAI/litellm/blob/62a591f90c99120e1a51a8445f5c3752586868ea/litellm/router.py#L111 - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - } - ] - ) -) +client = instructor.from_litellm(completion) + +# import dotenv +# dotenv.load_dotenv() class UserDetail(BaseModel): @@ -35,7 +25,7 @@ class UserDetail(BaseModel): user = client.chat.completions.create( - model="gpt-3.5-turbo", + model="gpt-4o-mini", response_model=UserDetail, messages=[ {"role": "user", "content": "Extract Jason is 25 years old"}, @@ -52,25 +42,20 @@ print(f"user: {user}") ## Async Calls ```python -import litellm +import asyncio +import instructor from litellm import Router -import instructor, asyncio from pydantic import BaseModel -aclient = instructor.apatch( +aclient = instructor.patch( Router( model_list=[ { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "gpt-4o-mini"}, } ], - default_litellm_params={"acompletion": True}, # 👈 IMPORTANT - tells litellm to route to async completion function. + default_litellm_params={"acompletion": True}, # 👈 IMPORTANT - tells litellm to route to async completion function. ) ) @@ -82,7 +67,7 @@ class UserExtract(BaseModel): async def main(): model = await aclient.chat.completions.create( - model="gpt-3.5-turbo", + model="gpt-4o-mini", response_model=UserExtract, messages=[ {"role": "user", "content": "Extract jason is 25 years old"}, diff --git a/docs/my-website/img/lunary-trace.png b/docs/my-website/img/lunary-trace.png new file mode 100644 index 000000000000..509e63ad5435 Binary files /dev/null and b/docs/my-website/img/lunary-trace.png differ diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 93400a81c9c5..b11ba575f091 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -379,6 +379,7 @@ const sidebars = { type: "category", label: "Logging & Observability", items: [ + "observability/lunary_integration", "observability/mlflow", "observability/langfuse_integration", "observability/gcs_bucket_integration", @@ -402,7 +403,6 @@ const sidebars = { "observability/wandb_integration", "observability/slack_integration", "observability/athina_integration", - "observability/lunary_integration", "observability/greenscale_integration", "observability/supabase_integration", `observability/telemetry`, diff --git a/docs/my-website/src/pages/index.md b/docs/my-website/src/pages/index.md index cea3dc52b568..a13e556eab36 100644 --- a/docs/my-website/src/pages/index.md +++ b/docs/my-website/src/pages/index.md @@ -337,10 +337,10 @@ LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone from litellm import completion ## set env variables for logging tools +os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your key at https://app.lunary.ai/settings os.environ["HELICONE_API_KEY"] = "your-helicone-key" os.environ["LANGFUSE_PUBLIC_KEY"] = "" os.environ["LANGFUSE_SECRET_KEY"] = "" -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" os.environ["OPENAI_API_KEY"] diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index c56dd5a682a0..a667cbcd9d24 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -20,17 +20,15 @@ import Usage from "../components/usage"; import CacheDashboard from "@/components/cache_dashboard"; import { jwtDecode } from "jwt-decode"; import { Typography } from "antd"; -import { setGlobalLitellmHeaderName } from "../components/networking" +import { setGlobalLitellmHeaderName } from "../components/networking"; function getCookie(name: string) { - console.log("COOKIES", document.cookie) const cookieValue = document.cookie - .split('; ') - .find(row => row.startsWith(name + '=')); - return cookieValue ? cookieValue.split('=')[1] : null; + .split("; ") + .find((row) => row.startsWith(name + "=")); + return cookieValue ? cookieValue.split("=")[1] : null; } - function formatUserRole(userRole: string) { if (!userRole) { return "Undefined Role"; @@ -68,7 +66,8 @@ const CreateKeyPage = () => { const { Title, Paragraph } = Typography; const [userRole, setUserRole] = useState(""); const [premiumUser, setPremiumUser] = useState(false); - const [disabledPersonalKeyCreation, setDisabledPersonalKeyCreation] = useState(false); + const [disabledPersonalKeyCreation, setDisabledPersonalKeyCreation] = + useState(false); const [userEmail, setUserEmail] = useState(null); const [teams, setTeams] = useState(null); const [keys, setKeys] = useState(null); @@ -80,244 +79,244 @@ const CreateKeyPage = () => { const [showSSOBanner, setShowSSOBanner] = useState(true); const searchParams = useSearchParams()!; const [modelData, setModelData] = useState({ data: [] }); + const [token, setToken] = useState(null); + const userID = searchParams.get("userID"); const invitation_id = searchParams.get("invitation_id"); - const token = getCookie('token'); // Get page from URL, default to 'api-keys' if not present const [page, setPage] = useState(() => { - return searchParams.get('page') || 'api-keys'; + return searchParams.get("page") || "api-keys"; }); // Custom setPage function that updates URL const updatePage = (newPage: string) => { // Update URL without full page reload const newSearchParams = new URLSearchParams(searchParams); - newSearchParams.set('page', newPage); - + newSearchParams.set("page", newPage); + // Use Next.js router to update URL - window.history.pushState( - null, - '', - `?${newSearchParams.toString()}` - ); - + window.history.pushState(null, "", `?${newSearchParams.toString()}`); + setPage(newPage); }; const [accessToken, setAccessToken] = useState(null); useEffect(() => { - if (token) { - const decoded = jwtDecode(token) as { [key: string]: any }; - if (decoded) { - // cast decoded to dictionary - console.log("Decoded token:", decoded); + const token = getCookie("token"); + setToken(token); + }, []); - console.log("Decoded key:", decoded.key); - // set accessToken - setAccessToken(decoded.key); + useEffect(() => { + if (!token) { + return; + } - setDisabledPersonalKeyCreation(decoded.disabled_non_admin_personal_key_creation); + const decoded = jwtDecode(token) as { [key: string]: any }; + if (decoded) { + // cast decoded to dictionary + console.log("Decoded token:", decoded); - // check if userRole is defined - if (decoded.user_role) { - const formattedUserRole = formatUserRole(decoded.user_role); - console.log("Decoded user_role:", formattedUserRole); - setUserRole(formattedUserRole); - if (formattedUserRole == "Admin Viewer") { - setPage("usage"); - } - } else { - console.log("User role not defined"); - } + console.log("Decoded key:", decoded.key); + // set accessToken + setAccessToken(decoded.key); - if (decoded.user_email) { - setUserEmail(decoded.user_email); - } else { - console.log(`User Email is not set ${decoded}`); - } + setDisabledPersonalKeyCreation( + decoded.disabled_non_admin_personal_key_creation, + ); - if (decoded.login_method) { - setShowSSOBanner( - decoded.login_method == "username_password" ? true : false - ); - } else { - console.log(`User Email is not set ${decoded}`); + // check if userRole is defined + if (decoded.user_role) { + const formattedUserRole = formatUserRole(decoded.user_role); + console.log("Decoded user_role:", formattedUserRole); + setUserRole(formattedUserRole); + if (formattedUserRole == "Admin Viewer") { + setPage("usage"); } + } else { + console.log("User role not defined"); + } - if (decoded.premium_user) { - setPremiumUser(decoded.premium_user); - } + if (decoded.user_email) { + setUserEmail(decoded.user_email); + } else { + console.log(`User Email is not set ${decoded}`); + } - if (decoded.auth_header_name) { - setGlobalLitellmHeaderName(decoded.auth_header_name); - } - + if (decoded.login_method) { + setShowSSOBanner( + decoded.login_method == "username_password" ? true : false, + ); + } else { + console.log(`User Email is not set ${decoded}`); + } + + if (decoded.premium_user) { + setPremiumUser(decoded.premium_user); + } + + if (decoded.auth_header_name) { + setGlobalLitellmHeaderName(decoded.auth_header_name); } } }, [token]); return ( Loading...}> - { - invitation_id ? ( - - ) : ( -
- -
-
+ ) : ( +
+ +
+
-
+ /> +
- {page == "api-keys" ? ( - - ) : page == "models" ? ( - - ) : page == "llm-playground" ? ( - - ) : page == "users" ? ( - - ) : page == "teams" ? ( - - ) : page == "organizations" ? ( - - ) : page == "admin-panel" ? ( - - ) : page == "api_ref" ? ( - - ) : page == "settings" ? ( - - ) : page == "budgets" ? ( - - ) : page == "general-settings" ? ( - - ) : page == "model-hub" ? ( - - ) : page == "caching" ? ( - - ) : page == "pass-through-settings" ? ( - - ) : ( - - )} + {page == "api-keys" ? ( + + ) : page == "models" ? ( + + ) : page == "llm-playground" ? ( + + ) : page == "users" ? ( + + ) : page == "teams" ? ( + + ) : page == "organizations" ? ( + + ) : page == "admin-panel" ? ( + + ) : page == "api_ref" ? ( + + ) : page == "settings" ? ( + + ) : page == "budgets" ? ( + + ) : page == "general-settings" ? ( + + ) : page == "model-hub" ? ( + + ) : page == "caching" ? ( + + ) : page == "pass-through-settings" ? ( + + ) : ( + + )} +
-
- ) - } - + )} ); }; diff --git a/ui/litellm-dashboard/src/components/budgets/edit_budget_modal.tsx b/ui/litellm-dashboard/src/components/budgets/edit_budget_modal.tsx index 04a7f8ca2d28..cfd225dd6de5 100644 --- a/ui/litellm-dashboard/src/components/budgets/edit_budget_modal.tsx +++ b/ui/litellm-dashboard/src/components/budgets/edit_budget_modal.tsx @@ -143,7 +143,7 @@ const EditBudgetModal: React.FC = ({
- Edit Budget + Save
diff --git a/ui/litellm-dashboard/src/components/create_user_button.tsx b/ui/litellm-dashboard/src/components/create_user_button.tsx index ade39699b55f..380396ad2383 100644 --- a/ui/litellm-dashboard/src/components/create_user_button.tsx +++ b/ui/litellm-dashboard/src/components/create_user_button.tsx @@ -53,11 +53,9 @@ const Createuser: React.FC = ({ useState(null); const router = useRouter(); const isLocal = process.env.NODE_ENV === "development"; - if (isLocal != true) { - console.log = function() {}; - } + const [baseUrl, setBaseUrl] = useState( - isLocal ? "http://localhost:4000" : "" + isLocal ? "http://localhost:4000" : "", ); // get all models useEffect(() => { @@ -67,7 +65,7 @@ const Createuser: React.FC = ({ const modelDataResponse = await modelAvailableCall( accessToken, userID, - userRole + userRole, ); // Assuming modelDataResponse.data contains an array of model objects with a 'model_name' property const availableModels = []; @@ -81,27 +79,26 @@ const Createuser: React.FC = ({ // Assuming modelDataResponse.data contains an array of model names setUserModels(availableModels); - // get ui settings + // get ui settings const uiSettingsResponse = await getProxyUISettings(accessToken); console.log("uiSettingsResponse:", uiSettingsResponse); - + setUISettings(uiSettingsResponse); } catch (error) { console.error("Error fetching model data:", error); } }; - - fetchData(); // Call the function to fetch model data when the component mounts }, []); // Empty dependency array to run only once useEffect(() => { - if (router) { - const { protocol, host } = window.location; - const baseUrl = `${protocol}/${host}`; - setBaseUrl(baseUrl); + if (!router) { + return; } + + const base = new URL("/", window.location.href); + setBaseUrl(base.toString()); }, [router]); const handleOk = () => { setIsModalVisible(false); @@ -123,7 +120,7 @@ const Createuser: React.FC = ({ console.log("user create Response:", response); setApiuser(response["key"]); const user_id = response.data?.user_id || response.user_id; - + // only do invite link flow if sso is not enabled if (!uiSettings?.SSO_ENABLED) { invitationCreateCall(accessToken, user_id).then((data) => { @@ -132,7 +129,7 @@ const Createuser: React.FC = ({ setIsInvitationLinkModalVisible(true); }); } else { - // create an InvitationLink Object for this user for the SSO flow + // create an InvitationLink Object for this user for the SSO flow // for SSO the invite link is the proxy base url since the User just needs to login const invitationLink: InvitationLink = { id: crypto.randomUUID(), // Generate a unique ID @@ -198,7 +195,7 @@ const Createuser: React.FC = ({

- ) + ), )} diff --git a/ui/litellm-dashboard/src/components/onboarding_link.tsx b/ui/litellm-dashboard/src/components/onboarding_link.tsx index 78f0dff8accc..36aa4fc0a414 100644 --- a/ui/litellm-dashboard/src/components/onboarding_link.tsx +++ b/ui/litellm-dashboard/src/components/onboarding_link.tsx @@ -1,16 +1,8 @@ -import React, { useState, useEffect } from "react"; -import { - Button as Button2, - Modal, - Form, - Input, - Select as Select2, - InputNumber, - message, - Typography, -} from "antd"; +import React from "react"; +import { Modal, message, Typography } from "antd"; import { CopyToClipboard } from "react-copy-to-clipboard"; import { Text, Button } from "@tremor/react"; + export interface InvitationLink { id: string; user_id: string; @@ -33,12 +25,12 @@ interface OnboardingProps { invitationLinkData: InvitationLink | null; } -const OnboardingModal: React.FC = ({ +export default function OnboardingModal({ isInvitationLinkModalVisible, setIsInvitationLinkModalVisible, baseUrl, invitationLinkData, -}) => { +}: OnboardingProps) { const { Title, Paragraph } = Typography; const handleInvitationOk = () => { setIsInvitationLinkModalVisible(false); @@ -50,9 +42,9 @@ const OnboardingModal: React.FC = ({ const getInvitationUrl = () => { if (invitationLinkData?.has_user_setup_sso) { - return `${baseUrl}/ui`; + return new URL("/ui", baseUrl).toString(); } - return `${baseUrl}/ui?invitation_id=${invitationLinkData?.id}`; + return new URL(`/ui?invitation_id=${invitationLinkData?.id}`, baseUrl).toString(); }; return ( @@ -64,7 +56,6 @@ const OnboardingModal: React.FC = ({ onOk={handleInvitationOk} onCancel={handleInvitationCancel} > - {/* {JSON.stringify(invitationLinkData)} */} Copy and send the generated link to onboard this user to the proxy. @@ -75,11 +66,10 @@ const OnboardingModal: React.FC = ({
Invitation Link - {getInvitationUrl()} + {getInvitationUrl()}
-
message.success("Copied!")} @@ -89,6 +79,4 @@ const OnboardingModal: React.FC = ({
); -}; - -export default OnboardingModal; +} diff --git a/ui/litellm-dashboard/src/components/teams.tsx b/ui/litellm-dashboard/src/components/teams.tsx index db5449f46b1f..e988b65e3a70 100644 --- a/ui/litellm-dashboard/src/components/teams.tsx +++ b/ui/litellm-dashboard/src/components/teams.tsx @@ -211,7 +211,7 @@ const Team: React.FC = ({ >
- Edit Team + Save