mistral api integration (#558)

uptrain-ai · Feb 25, 2024 · 4af2090 · 4af2090
1 parent 8b74deb
commit 4af2090
Show file tree

Hide file tree

Showing 2 changed files with 85 additions and 2 deletions.
diff --git a/examples/open_source_evaluator_tutorial.ipynb b/examples/open_source_evaluator_tutorial.ipynb
@@ -397,12 +397,91 @@
     "print(json.dumps(results, indent=3))"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4. Evaluations via MISTRAL API KEY"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MISTRAL_API_KEY = \"*********************\""
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "import os\n",
+    "from uptrain import Settings\n",
+    "settings = Settings(model = 'mistral/mistral-tiny', mistral_api_key=MISTRAL_API_KEY)\n",
+    "eval_llm = EvalLLM(settings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m2024-02-23 16:34:13.898\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36muptrain.operators.language.llm\u001b[0m:\u001b[36mfetch_responses\u001b[0m:\u001b[36m234\u001b[0m - \u001b[33m\u001b[1mDetected a running event loop, scheduling requests in a separate thread.\u001b[0m\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1/1 [00:02<00:00,  2.47s/it]\n",
+      "\u001b[32m2024-02-23 16:34:16.403\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36muptrain.operators.language.llm\u001b[0m:\u001b[36mfetch_responses\u001b[0m:\u001b[36m234\u001b[0m - \u001b[33m\u001b[1mDetected a running event loop, scheduling requests in a separate thread.\u001b[0m\n",
+      "100%|██████████| 1/1 [00:01<00:00,  1.56s/it]\n",
+      "\u001b[32m2024-02-23 16:34:17.971\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36muptrain.operators.language.llm\u001b[0m:\u001b[36mfetch_responses\u001b[0m:\u001b[36m234\u001b[0m - \u001b[33m\u001b[1mDetected a running event loop, scheduling requests in a separate thread.\u001b[0m\n",
+      "100%|██████████| 1/1 [00:00<00:00,  1.57it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "results = eval_llm.evaluate(\n",
+    "    data=data,\n",
+    "    checks=[Evals.CONTEXT_RELEVANCE, Evals.RESPONSE_RELEVANCE]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[\n",
+      "   {\n",
+      "      \"question\": \"Which is the most popular global sport?\",\n",
+      "      \"context\": \"The popularity of sports can be measured in various ways, including TV viewership, social media presence, number of participants, and economic impact. Football is undoubtedly the world's most popular sport with major events like the FIFA World Cup and sports personalities like Ronaldo and Messi, drawing a followership of more than 4 billion people. Cricket is particularly popular in countries like India, Pakistan, Australia, and England. The ICC Cricket World Cup and Indian Premier League (IPL) have substantial viewership. The NBA has made basketball popular worldwide, especially in countries like the USA, Canada, China, and the Philippines. Major tennis tournaments like Wimbledon, the US Open, French Open, and Australian Open have large global audiences. Players like Roger Federer, Serena Williams, and Rafael Nadal have boosted the sport's popularity. Field Hockey is very popular in countries like India, Netherlands, and Australia. It has a considerable following in many parts of the world.\",\n",
+      "      \"response\": \"Football is the most popular sport with around 4 billion followers worldwide\",\n",
+      "      \"score_context_relevance\": 1.0,\n",
+      "      \"explanation_context_relevance\": \"{\\n\\\"Reasoning\\\": \\\"The given context mentions that football is the world's most popular sport based on its followership of over 4 billion people. The context also mentions other sports like cricket, basketball, tennis, and field hockey, but it does not provide any information that would challenge the statement that football is the most popular sport. Therefore, the extracted context can answer the given query completely.\\\",\\n\\\"Choice\\\": \\\"A\\\"\\n}\",\n",
+      "      \"score_response_relevance\": 1.0,\n",
+      "      \"explanation_response_relevance\": \"Response Precision: 1.0{\\n    \\\"Reasoning\\\": \\\"The given response provides the accurate answer to the question and does not include any irrelevant information.\\\",\\n    \\\"Choice\\\": \\\"A\\\"\\n}\\nResponse Recall: 1.0{\\n    \\\"Reasoning\\\": \\\"The given response adequately answers the given question by stating that football is the most popular sport with approximately 4 billion followers.\\\",\\n    \\\"Choice\\\": \\\"A\\\"\\n}\"\n",
+      "   }\n",
+      "]\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(json.dumps(results, indent=3))"
+   ]
   }
  ],
  "metadata": {

diff --git a/uptrain/framework/base.py b/uptrain/framework/base.py
@@ -31,6 +31,7 @@ class Settings(BaseSettings):
     replicate_api_token: str = Field(None, env="REPLICATE_API_TOKEN")
     anyscale_api_key: str = Field(None, env="ANYSCALE_API_KEY")
     together_api_key: str = Field(None, env="TOGETHER_API_KEY")
+    mistral_api_key: str = Field(None, env="MISTRAL_API_KEY")
 
     azure_api_key: str = Field(None, env="AZURE_API_KEY")
     azure_api_base: str = Field(None, env="AZURE_API_BASE")
@@ -88,6 +89,9 @@ def __init__(self, **data):
         if "together_api_key" in data:
             if data["together_api_key"] is not None:
                 os.environ["TOGETHER_API_KEY"] = data["together_api_key"]
+        if "mistral_api_key" in data:
+            if data["mistral_api_key"] is not None:
+                os.environ["MISTRAL_API_KEY"] = data["mistral_api_key"]
         if "embedding_model_api_token" in data:
             if data["embedding_model_api_token"] is not None:
                 os.environ["EMBEDDING_MODEL_API_TOKEN"] = data[