diff --git a/examples/anthropic/csv_scraper_graph_multi_anthropic.py b/examples/anthropic/csv_scraper_graph_multi_haiku.py similarity index 96% rename from examples/anthropic/csv_scraper_graph_multi_anthropic.py rename to examples/anthropic/csv_scraper_graph_multi_haiku.py index fcc297ab..b833af01 100644 --- a/examples/anthropic/csv_scraper_graph_multi_anthropic.py +++ b/examples/anthropic/csv_scraper_graph_multi_haiku.py @@ -26,7 +26,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000}, } diff --git a/examples/anthropic/csv_scraper_anthropic.py b/examples/anthropic/csv_scraper_haiku.py similarity index 96% rename from examples/anthropic/csv_scraper_anthropic.py rename to examples/anthropic/csv_scraper_haiku.py index 01a26a2b..2e0ebe81 100644 --- a/examples/anthropic/csv_scraper_anthropic.py +++ b/examples/anthropic/csv_scraper_haiku.py @@ -32,7 +32,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/custom_graph_anthropic.py b/examples/anthropic/custom_graph_haiku.py similarity index 97% rename from examples/anthropic/custom_graph_anthropic.py rename to examples/anthropic/custom_graph_haiku.py index 6d787484..cea14361 100644 --- a/examples/anthropic/custom_graph_anthropic.py +++ b/examples/anthropic/custom_graph_haiku.py @@ -18,7 +18,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/json_scraper_anthropic.py b/examples/anthropic/json_scraper_haiku.py similarity index 96% rename from examples/anthropic/json_scraper_anthropic.py rename to examples/anthropic/json_scraper_haiku.py index 05ee7fdf..2610b658 100644 --- a/examples/anthropic/json_scraper_anthropic.py +++ b/examples/anthropic/json_scraper_haiku.py @@ -26,7 +26,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/json_scraper_multi_anthropic.py b/examples/anthropic/json_scraper_multi_haiku.py similarity index 93% rename from examples/anthropic/json_scraper_multi_anthropic.py rename to examples/anthropic/json_scraper_multi_haiku.py index c07fc54f..0327673b 100644 --- a/examples/anthropic/json_scraper_multi_anthropic.py +++ b/examples/anthropic/json_scraper_multi_haiku.py @@ -11,7 +11,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/pdf_scraper_graph_anthropic.py b/examples/anthropic/pdf_scraper_graph_haiku.py similarity index 96% rename from examples/anthropic/pdf_scraper_graph_anthropic.py rename to examples/anthropic/pdf_scraper_graph_haiku.py index 3e4191a6..61be06b4 100644 --- a/examples/anthropic/pdf_scraper_graph_anthropic.py +++ b/examples/anthropic/pdf_scraper_graph_haiku.py @@ -14,7 +14,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/pdf_scraper_multi_anthropic.py b/examples/anthropic/pdf_scraper_multi_haiku.py similarity index 99% rename from examples/anthropic/pdf_scraper_multi_anthropic.py rename to examples/anthropic/pdf_scraper_multi_haiku.py index 0c842787..974dd2f8 100644 --- a/examples/anthropic/pdf_scraper_multi_anthropic.py +++ b/examples/anthropic/pdf_scraper_multi_haiku.py @@ -11,7 +11,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/scrape_plain_text_anthropic.py b/examples/anthropic/scrape_plain_text_haiku.py similarity index 96% rename from examples/anthropic/scrape_plain_text_anthropic.py rename to examples/anthropic/scrape_plain_text_haiku.py index 7ebf84da..d3f36638 100644 --- a/examples/anthropic/scrape_plain_text_anthropic.py +++ b/examples/anthropic/scrape_plain_text_haiku.py @@ -28,7 +28,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/script_generator_anthropic.py b/examples/anthropic/script_generator_haiku.py similarity index 95% rename from examples/anthropic/script_generator_anthropic.py rename to examples/anthropic/script_generator_haiku.py index 160987cc..889ce0b5 100644 --- a/examples/anthropic/script_generator_anthropic.py +++ b/examples/anthropic/script_generator_haiku.py @@ -16,7 +16,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/script_multi_generator_anthropic.py b/examples/anthropic/script_multi_generator_haiku.py similarity index 96% rename from examples/anthropic/script_multi_generator_anthropic.py rename to examples/anthropic/script_multi_generator_haiku.py index c4b3f09b..f7c69010 100644 --- a/examples/anthropic/script_multi_generator_anthropic.py +++ b/examples/anthropic/script_multi_generator_haiku.py @@ -16,7 +16,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, "library": "beautifulsoup" diff --git a/examples/anthropic/search_graph_anthropic.py b/examples/anthropic/search_graph_haiku.py similarity index 95% rename from examples/anthropic/search_graph_anthropic.py rename to examples/anthropic/search_graph_haiku.py index 4ae0e6b3..f90d7598 100644 --- a/examples/anthropic/search_graph_anthropic.py +++ b/examples/anthropic/search_graph_haiku.py @@ -15,7 +15,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/search_graph_schema_anthropic.py b/examples/anthropic/search_graph_schema_haiku.py similarity index 92% rename from examples/anthropic/search_graph_schema_anthropic.py rename to examples/anthropic/search_graph_schema_haiku.py index 58e1ca0f..c9e7a875 100644 --- a/examples/anthropic/search_graph_schema_anthropic.py +++ b/examples/anthropic/search_graph_schema_haiku.py @@ -27,9 +27,8 @@ class Dishes(BaseModel): graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "model": "claude-3-haiku-20240307", + "max_tokens": 4000}, } # ************************************************ diff --git a/examples/anthropic/search_link_graph_anthropic.py b/examples/anthropic/search_link_graph_haiku.py similarity index 91% rename from examples/anthropic/search_link_graph_anthropic.py rename to examples/anthropic/search_link_graph_haiku.py index 4d671817..ccfbc1d2 100644 --- a/examples/anthropic/search_link_graph_anthropic.py +++ b/examples/anthropic/search_link_graph_haiku.py @@ -29,11 +29,8 @@ # ************************************************ graph_config = { - "llm": { - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", - "max_tokens": 4000 - }, + "llm": {"model_instance": llm_model_instance}, + "embeddings": {"model_instance": embedder_model_instance} } # ************************************************ diff --git a/examples/anthropic/smart_scraper_anthropic.py b/examples/anthropic/smart_scraper_haiku.py similarity index 96% rename from examples/anthropic/smart_scraper_anthropic.py rename to examples/anthropic/smart_scraper_haiku.py index 612363c7..f0bb2a57 100644 --- a/examples/anthropic/smart_scraper_anthropic.py +++ b/examples/anthropic/smart_scraper_haiku.py @@ -19,7 +19,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/smart_scraper_multi_anthropic.py b/examples/anthropic/smart_scraper_multi_haiku.py similarity index 96% rename from examples/anthropic/smart_scraper_multi_anthropic.py rename to examples/anthropic/smart_scraper_multi_haiku.py index 6e2af361..eb2001d4 100644 --- a/examples/anthropic/smart_scraper_multi_anthropic.py +++ b/examples/anthropic/smart_scraper_multi_haiku.py @@ -17,7 +17,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/examples/anthropic/smart_scraper_schema_anthropic.py b/examples/anthropic/smart_scraper_schema_haiku.py similarity index 96% rename from examples/anthropic/smart_scraper_schema_anthropic.py rename to examples/anthropic/smart_scraper_schema_haiku.py index 0a444923..83cedd2a 100644 --- a/examples/anthropic/smart_scraper_schema_anthropic.py +++ b/examples/anthropic/smart_scraper_schema_haiku.py @@ -33,7 +33,7 @@ class Projects(BaseModel): graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000}, } diff --git a/examples/anthropic/xml_scraper_graph_multi_anthropic.py b/examples/anthropic/xml_scraper_graph_multi_haiku.py similarity index 96% rename from examples/anthropic/xml_scraper_graph_multi_anthropic.py rename to examples/anthropic/xml_scraper_graph_multi_haiku.py index 31b350c2..6b79f709 100644 --- a/examples/anthropic/xml_scraper_graph_multi_anthropic.py +++ b/examples/anthropic/xml_scraper_graph_multi_haiku.py @@ -26,7 +26,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000}, } diff --git a/examples/anthropic/xml_scraper_anthropic.py b/examples/anthropic/xml_scraper_haiku.py similarity index 96% rename from examples/anthropic/xml_scraper_anthropic.py rename to examples/anthropic/xml_scraper_haiku.py index cd60f0d6..dd64f571 100644 --- a/examples/anthropic/xml_scraper_anthropic.py +++ b/examples/anthropic/xml_scraper_haiku.py @@ -26,7 +26,7 @@ graph_config = { "llm": { "api_key": os.getenv("ANTHROPIC_API_KEY"), - "model": "anthropic/claude-3-haiku-20240307", + "model": "claude-3-haiku-20240307", "max_tokens": 4000 }, } diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index f80e430d..555e8211 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -142,7 +142,7 @@ def handle_model(model_name, provider, token_key, default_token=8192): known_models = {"chatgpt","gpt","openai", "azure_openai", "google_genai", "ollama", "oneapi", "nvidia", "groq", "google_vertexai", "bedrock", "mistralai", "hugging_face", "deepseek", "ernie", - "fireworks", "anthropic"} + "fireworks", "claude-3-"} if llm_params["model"].split("/")[0] not in known_models and llm_params["model"].split("-")[0] not in known_models: raise ValueError(f"Model '{llm_params['model']}' is not supported") @@ -172,9 +172,8 @@ def handle_model(model_name, provider, token_key, default_token=8192): token_key = model_name if "model_tokens" not in llm_params else llm_params["model_tokens"] return handle_model(model_name, "ollama", token_key) - elif "anthropic" in llm_params["model"]: - model_name = llm_params["model"].split("anthropic/")[-1] - return handle_model(model_name, "anthropic", model_name) + elif "claude-3-" in llm_params["model"]: + return handle_model(llm_params["model"], "anthropic", "claude3") elif llm_params["model"].startswith("mistral"): model_name = llm_params["model"].split("/")[-1]