From cff785dedd9bb5bc17f71596421e90404cccef88 Mon Sep 17 00:00:00 2001 From: Dimitry Ageev Date: Mon, 28 Oct 2024 18:37:47 +0300 Subject: [PATCH] add latest claude 3.5 sonnet with visual interfaces support (#459) --- refact_known_models/passthrough.py | 11 +++++++++++ refact_webgui/webgui/selfhost_fastapi_completions.py | 1 + 2 files changed, 12 insertions(+) diff --git a/refact_known_models/passthrough.py b/refact_known_models/passthrough.py index 10f312a5..183cc440 100644 --- a/refact_known_models/passthrough.py +++ b/refact_known_models/passthrough.py @@ -122,4 +122,15 @@ "pp1000t_generated": 600, # $0.60 / 1M tokens "filter_caps": ["chat", "tools"], }, + "claude-3-5-sonnet-20241022": { + "backend": "litellm", + "provider": "anthropic", + "tokenizer_path": "Xenova/claude-tokenizer", + "resolve_as": "claude-3-5-sonnet-20241022", + "T": 200_000, + "T_out": 4096, + "pp1000t_prompt": 3_000, # $3.00 / 1M tokens (2024 oct) + "pp1000t_generated": 15_000, # $15.00 / 1M tokens (2024 oct) + "filter_caps": ["chat", "tools"], + }, } diff --git a/refact_webgui/webgui/selfhost_fastapi_completions.py b/refact_webgui/webgui/selfhost_fastapi_completions.py index b3b83508..f9d62e34 100644 --- a/refact_webgui/webgui/selfhost_fastapi_completions.py +++ b/refact_webgui/webgui/selfhost_fastapi_completions.py @@ -228,6 +228,7 @@ def _integrations_env_setup(): def _integrations_env_setup(env_var_name: str, api_key_name: str, api_enable_name: str): os.environ[env_var_name] = integrations.get(api_key_name, "") if inference.get(api_enable_name, False) else "" + litellm.modify_params = True # NOTE: for Anthropic API _integrations_env_setup("OPENAI_API_KEY", "openai_api_key", "openai_api_enable") _integrations_env_setup("ANTHROPIC_API_KEY", "anthropic_api_key", "anthropic_api_enable")