add latest claude 3.5 sonnet with visual interfaces support (#459)

smallcloudai · Oct 28, 2024 · cff785d · cff785d
1 parent ef29a37
commit cff785d
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 0 deletions.
diff --git a/refact_known_models/passthrough.py b/refact_known_models/passthrough.py
@@ -122,4 +122,15 @@
         "pp1000t_generated": 600,  # $0.60 / 1M tokens
         "filter_caps": ["chat", "tools"],
     },
+    "claude-3-5-sonnet-20241022": {
+        "backend": "litellm",
+        "provider": "anthropic",
+        "tokenizer_path": "Xenova/claude-tokenizer",
+        "resolve_as": "claude-3-5-sonnet-20241022",
+        "T": 200_000,
+        "T_out": 4096,
+        "pp1000t_prompt": 3_000,  # $3.00 / 1M tokens (2024 oct)
+        "pp1000t_generated": 15_000,  # $15.00 / 1M tokens (2024 oct)
+        "filter_caps": ["chat", "tools"],
+    },
 }
diff --git a/refact_webgui/webgui/selfhost_fastapi_completions.py b/refact_webgui/webgui/selfhost_fastapi_completions.py
@@ -228,6 +228,7 @@ def _integrations_env_setup():
         def _integrations_env_setup(env_var_name: str, api_key_name: str, api_enable_name: str):
             os.environ[env_var_name] = integrations.get(api_key_name, "") if inference.get(api_enable_name, False) else ""
 
+        litellm.modify_params = True  # NOTE: for Anthropic API
         _integrations_env_setup("OPENAI_API_KEY", "openai_api_key", "openai_api_enable")
         _integrations_env_setup("ANTHROPIC_API_KEY", "anthropic_api_key", "anthropic_api_enable")