helicone deeper integration

Author: chitalian

docs/howtos/integrations/helicone.ipynb

@@ -53,10 +53,11 @@
     "from datasets import Dataset\n",
     "from ragas import evaluate\n",
     "from ragas.metrics import faithfulness, answer_relevancy, context_precision\n",
+    "from ragas.integrations.helicone import helicone_config # import helicone_config\n",
+    "\n",
     "\n",
     "# Set up Helicone\n",
-    "HELICONE_API_KEY = \"your_helicone_api_key_here\"  # Replace with your actual Helicone API key\n",
-    "os.environ[\"OPENAI_API_BASE\"] = f\"https://oai.helicone.ai/{HELICONE_API_KEY}/v1\"\n",
+    "helicone_config.api_key = \"your_helicone_api_key_here\"  # Replace with your actual Helicone API key\n",
     "os.environ[\"OPENAI_API_KEY\"] = \"your_openai_api_key_here\"  # Replace with your actual OpenAI API key\n",
     "\n",
     "# Verify Helicone API key is set\n",

src/ragas/integrations/helicone.py

@@ -0,0 +1,96 @@
+from dataclasses import dataclass, field
+from typing import Optional, Dict, Any
+
+
+@dataclass
+class CacheConfig:
+    ttl: int = 60 * 60 * 24 * 30  # 30 days
+    maxsize: int = 1000
+
+
+@dataclass
+class HeliconeSingleton:
+    api_key: Optional[str] = None
+    base_url: Optional[str] = "https://oai.helicone.ai"
+    cache_config: Optional[CacheConfig] = None
+    _instance: Optional['HeliconeSingleton'] = None
+
+    # New fields for configurable headers
+    target_url: Optional[str] = None
+    openai_api_base: Optional[str] = None
+    request_id: Optional[str] = None
+    model_override: Optional[str] = None
+    prompt_id: Optional[str] = None
+    user_id: Optional[str] = None
+    fallbacks: Optional[str] = None
+    rate_limit_policy: Optional[str] = None
+    session_id: Optional[str] = None
+    session_path: Optional[str] = None
+    session_name: Optional[str] = None
+    posthog_key: Optional[str] = None
+    posthog_host: Optional[str] = None
+    omit_response: Optional[bool] = None
+    omit_request: Optional[bool] = None
+    cache_enabled: Optional[bool] = None
+    retry_enabled: Optional[bool] = None
+    moderations_enabled: Optional[bool] = None
+    llm_security_enabled: Optional[bool] = None
+    stream_force_format: Optional[bool] = None
+    custom_properties: Dict[str, str] = field(default_factory=dict)
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def default_headers(self) -> Dict[str, Any]:
+        headers = {"Helicone-Auth": f"Bearer {self.api_key}"}
+
+        if self.target_url:
+            headers["Helicone-Target-URL"] = self.target_url
+        if self.openai_api_base:
+            headers["Helicone-OpenAI-Api-Base"] = self.openai_api_base
+        if self.request_id:
+            headers["Helicone-Request-Id"] = self.request_id
+        if self.model_override:
+            headers["Helicone-Model-Override"] = self.model_override
+        if self.prompt_id:
+            headers["Helicone-Prompt-Id"] = self.prompt_id
+        if self.user_id:
+            headers["Helicone-User-Id"] = self.user_id
+        if self.fallbacks:
+            headers["Helicone-Fallbacks"] = self.fallbacks
+        if self.rate_limit_policy:
+            headers["Helicone-RateLimit-Policy"] = self.rate_limit_policy
+        if self.session_id:
+            headers["Helicone-Session-Id"] = self.session_id
+        if self.session_path:
+            headers["Helicone-Session-Path"] = self.session_path
+        if self.session_name:
+            headers["Helicone-Session-Name"] = self.session_name
+        if self.posthog_key:
+            headers["Helicone-Posthog-Key"] = self.posthog_key
+        if self.posthog_host:
+            headers["Helicone-Posthog-Host"] = self.posthog_host
+
+        # Boolean headers
+        for header, value in {
+            "Helicone-Omit-Response": self.omit_response,
+            "Helicone-Omit-Request": self.omit_request,
+            "Helicone-Cache-Enabled": (self.cache_enabled and "true") or (self.cache_config.maxsize or self.cache_config.ttl and "true"),
+            "Helicone-Retry-Enabled": self.retry_enabled,
+            "Helicone-Moderations-Enabled": self.moderations_enabled,
+            "Helicone-LLM-Security-Enabled": self.llm_security_enabled,
+            "Helicone-Stream-Force-Format": self.stream_force_format
+        }.items():
+            if value is not None:
+                headers[header] = str(value).lower()
+
+        # Custom properties
+        for key, value in self.custom_properties.items():
+            headers[f"Helicone-Property-{key}"] = value
+
+        return headers
+
+
+helicone_config = HeliconeSingleton()

src/ragas/llms/base.py

@@ -15,6 +15,7 @@
 from langchain_openai.llms import AzureOpenAI, OpenAI
 from langchain_openai.llms.base import BaseOpenAI
 
+from ragas.integrations.helicone import helicone_config
 from ragas.run_config import RunConfig, add_async_retry, add_retry
 
 if t.TYPE_CHECKING:
@@ -103,7 +104,8 @@ async def generate(
             )
         else:
             loop = asyncio.get_event_loop()
-            generate_text_with_retry = add_retry(self.generate_text, self.run_config)
+            generate_text_with_retry = add_retry(
+                self.generate_text, self.run_config)
             generate_text = partial(
                 generate_text_with_retry,
                 prompt=prompt,
@@ -240,7 +242,8 @@ def check_args(
         callbacks: Callbacks,
     ) -> dict[str, t.Any]:
         if n != 1:
-            logger.warning("n values greater than 1 not support for LlamaIndex LLMs")
+            logger.warning(
+                "n values greater than 1 not support for LlamaIndex LLMs")
         if temperature != 1e-8:
             logger.info("temperature kwarg passed to LlamaIndex LLM")
         if stop is not None:
@@ -294,5 +297,17 @@ def llm_factory(
     timeout = None
     if run_config is not None:
         timeout = run_config.timeout
-    openai_model = ChatOpenAI(model=model, timeout=timeout)
+
+    default_headers = None
+    base_url = None
+    if (helicone_config.api_key is not None):
+        default_headers = helicone_config.default_headers()
+        base_url = helicone_config.base_url
+
+    openai_model = ChatOpenAI(
+        model=model,
+        timeout=timeout,
+        default_headers=default_headers,
+        base_url=base_url
+    )
     return LangchainLLMWrapper(openai_model, run_config)