diff --git a/.gitignore b/.gitignore index 4a92cbf31c7..be045dd2651 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,5 @@ litellm/proxy/google-cloud-sdk/* tests/llm_translation/log.txt venv/ tests/local_testing/log.txt + +.codegpt \ No newline at end of file diff --git a/litellm/llms/custom_httpx/aiohttp_handler.py b/litellm/llms/custom_httpx/aiohttp_handler.py index ab6872e0437..f87355097df 100644 --- a/litellm/llms/custom_httpx/aiohttp_handler.py +++ b/litellm/llms/custom_httpx/aiohttp_handler.py @@ -29,6 +29,9 @@ class BaseLLMAIOHTTPHandler: + def __init__(self): + self.client_session: Optional[aiohttp.ClientSession] = None + async def _make_common_async_call( self, async_httpx_client: AsyncHTTPHandler, @@ -52,21 +55,23 @@ async def _make_common_async_call( ) ) - async with aiohttp.ClientSession(timeout=timeout_obj) as session: - for i in range(max(max_retry_on_unprocessable_entity_error, 1)): - try: - response = await session.post( - url=api_base, - headers=headers, - json=data, - ) - if not response.ok: - response.raise_for_status() - except aiohttp.ClientResponseError as e: - raise self._handle_error(e=e, provider_config=provider_config) - except Exception as e: - raise self._handle_error(e=e, provider_config=provider_config) - break + if self.client_session is None: + self.client_session = aiohttp.ClientSession() + + for i in range(max(max_retry_on_unprocessable_entity_error, 1)): + try: + response = await self.client_session.post( + url=api_base, + headers=headers, + json=data, + ) + if not response.ok: + response.raise_for_status() + except aiohttp.ClientResponseError as e: + raise self._handle_error(e=e, provider_config=provider_config) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + break if response is None: raise provider_config.get_error_class( @@ -168,25 +173,7 @@ async def async_completion( ) _json_response = await _response.json() - # cast to httpx.Response - # Todo - use this until we migrate fully to aiohttp - response = httpx.Response( - status_code=_response.status, - headers=_response.headers, - json=_json_response, - ) - return provider_config.transform_response( - model=model, - raw_response=response, - model_response=model_response, - logging_obj=logging_obj, - api_key=api_key, - request_data=data, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - encoding=encoding, - ) + return _json_response def completion( self, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index e80b764cf96..787347e84d7 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,8 +1,8 @@ model_list: - model_name: "fake-openai-endpoint" litellm_params: - model: openai/any - api_base: https://exampleopenaiendpoint-production.up.railway.app + model: aiohttp_openai/any + api_base: https://example-openai-endpoint.onrender.com/chat/completions api_key: "ishaan" general_settings: