feat: add opensandbox sandbox provider

.github/workflows/test-unit-misc.yml

@@ -33,6 +33,7 @@ jobs:
         tests/test_litellm/images
         tests/test_litellm/interactions
         tests/test_litellm/passthrough
+        tests/test_litellm/sandbox
         tests/test_litellm/vector_stores
         tests/test_litellm/test_*.py
       workers: 2

litellm/constants.py

@@ -201,6 +201,18 @@
 
 # Provider-specific API base URLs
 XAI_API_BASE = "https://api.x.ai/v1"
+OPEN_SANDBOX_API_BASE_ENV_VAR = "OPEN_SANDBOX_API_BASE"
+OPEN_SANDBOX_API_KEY_ENV_VAR = "OPEN_SANDBOX_API_KEY"
+OPEN_SANDBOX_DEFAULT_TEMPLATE = "opensandbox/code-interpreter:v1.1.0"
+_OPEN_SANDBOX_FALLBACK_ENTRYPOINT = "/opt/code-interpreter/code-interpreter.sh"
+OPEN_SANDBOX_DEFAULT_ENTRYPOINT = (_OPEN_SANDBOX_FALLBACK_ENTRYPOINT,)
+OPEN_SANDBOX_DEFAULT_LANGUAGE = "python"
+OPEN_SANDBOX_DEFAULT_CPU_LIMIT = "1"
+OPEN_SANDBOX_DEFAULT_MEMORY_LIMIT = "2Gi"
+OPEN_SANDBOX_EXECD_PORT = 44772
+OPEN_SANDBOX_DEFAULT_TIMEOUT = 300
+OPEN_SANDBOX_READY_TIMEOUT = 30.0
+OPEN_SANDBOX_POLL_INTERVAL = 0.2
 
 DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int(
     os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024)

litellm/llms/base_llm/sandbox/transformation.py

@@ -8,10 +8,14 @@
 
 from typing import Any, Union
 
+import httpx
+
 from pydantic import Field, PrivateAttr
 
 from litellm.types.llms.base import LiteLLMPydanticObjectBase
 
+SANDBOX_MAX_OUTPUT_BYTES = 10 * 1024 * 1024
+
 
 class ContainerHandle(LiteLLMPydanticObjectBase):
     """A live sandbox container. Carries everything needed to reach it again."""
@@ -53,7 +57,7 @@ async def acreate_sandbox(
         *,
         template: str | None = None,
         timeout: int | None = None,
-        allow_internet_access: bool = True,
+        allow_internet_access: bool | None = None,
         api_key: str | None = None,
         **kwargs,
     ) -> ContainerHandle:
@@ -77,3 +81,16 @@ async def adelete_sandbox(
         **kwargs,
     ) -> bool:
         raise NotImplementedError("adelete_sandbox must be implemented by provider")
+
+    async def _read_capped_lines(self, response: httpx.Response) -> list[str]:
+        lines: list[str] = []
+        total = 0
+        async for line in response.aiter_lines():
+            total += len(line.encode("utf-8"))
+            if total > SANDBOX_MAX_OUTPUT_BYTES:
+                raise ValueError(
+                    f"Sandbox output exceeded {SANDBOX_MAX_OUTPUT_BYTES} bytes; aborting "
+                    "to avoid unbounded memory use."
+                )
+            lines.append(line)
+        return lines

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1,5 +1,6 @@
 import json
 import ssl
+from functools import lru_cache
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 from typing import (
     TYPE_CHECKING,
@@ -13,6 +14,7 @@
     Tuple,
     Union,
     cast,
+    get_type_hints,
 )
 
 import httpx  # type: ignore
@@ -26,6 +28,7 @@
 from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
 from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
+from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
@@ -101,6 +104,7 @@
     HttpxBinaryResponseContent,
     OpenAIFileObject,
     ResponseInputParam,
+    ResponsesAPIOptionalRequestParams,
     ResponsesAPIResponse,
 )
 from litellm.types.rerank import RerankResponse
@@ -135,6 +139,7 @@
     ImageResponse,
     ModelResponse,
     ProviderConfigManager,
+    async_pre_call_deployment_hook,
 )
 
 from .http_handler import get_shared_realtime_ssl_context
@@ -184,6 +189,47 @@ def _google_genai_streaming_hidden_params(
     }
 
 
+@lru_cache(maxsize=None)
+def _responses_api_optional_request_param_names() -> frozenset[str]:
+    return frozenset(get_type_hints(ResponsesAPIOptionalRequestParams).keys())
+
+
+def _custom_logger_callbacks(logging_obj: Any) -> list[Any]:
+    from litellm.integrations.custom_logger import CustomLogger
+    from litellm.litellm_core_utils.litellm_logging import (
+        get_custom_logger_compatible_class,
+    )
+
+    dynamic_success_callbacks = getattr(logging_obj, "dynamic_success_callbacks", None)
+    callbacks = list(litellm.callbacks)
+    if isinstance(dynamic_success_callbacks, (list, tuple)):
+        callbacks.extend(dynamic_success_callbacks)
+
+    custom_loggers: list[Any] = []
+    for cb in callbacks:
+        if isinstance(cb, str):
+            resolved = get_custom_logger_compatible_class(cb)  # type: ignore[arg-type]
+            if resolved is None:
+                continue
+            cb = resolved
+        if isinstance(cb, CustomLogger):
+            custom_loggers.append(cb)
+    return custom_loggers
+
+
+def _has_pre_call_deployment_hook(logging_obj: Any) -> bool:
+    from litellm.integrations.custom_logger import CustomLogger
+
+    base_func = CustomLogger.async_pre_call_deployment_hook
+    for cb in _custom_logger_callbacks(logging_obj):
+        cb_func = getattr(type(cb), "async_pre_call_deployment_hook", base_func)
+        if getattr(cb_func, "__func__", cb_func) is not getattr(
+            base_func, "__func__", base_func
+        ):
+            return True
+    return False
+
+
 class BaseLLMHTTPHandler:
     async def _make_common_async_call(
         self,
@@ -2224,12 +2270,92 @@ def anthropic_messages_handler(
             )
         raise ValueError("anthropic_messages_handler is not implemented for sync calls")
 
+    def _run_sync_responses_pre_call_deployment_hook(
+        self,
+        *,
+        model: str,
+        input: Union[str, ResponseInputParam],
+        custom_llm_provider: str,
+        response_api_optional_request_params: dict[str, Any],
+        litellm_params: GenericLiteLLMParams,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> tuple[
+        str,
+        Union[str, ResponseInputParam],
+        str,
+        dict[str, Any],
+        GenericLiteLLMParams,
+    ]:
+        if not _has_pre_call_deployment_hook(logging_obj):
+            return (
+                model,
+                input,
+                custom_llm_provider,
+                response_api_optional_request_params,
+                litellm_params,
+            )
+
+        modified_kwargs = run_async_function(
+            async_pre_call_deployment_hook,
+            {
+                **dict(litellm_params),
+                **response_api_optional_request_params,
+                "model": model,
+                "input": input,
+                "custom_llm_provider": custom_llm_provider,
+            },
+            CallTypes.responses.value,
+        )
+        if modified_kwargs is None:
+            return (
+                model,
+                input,
+                custom_llm_provider,
+                response_api_optional_request_params,
+                litellm_params,
+            )
+
+        optional_param_names = _responses_api_optional_request_param_names()
+        updated_response_params = {
+            **response_api_optional_request_params,
+            **{
+                key: value
+                for key, value in modified_kwargs.items()
+                if key in optional_param_names
+            },
+        }
+        updated_litellm_params = GenericLiteLLMParams(
+            **{
+                **dict(litellm_params),
+                **{
+                    key: value
+                    for key, value in modified_kwargs.items()
+                    if key not in optional_param_names
+                    and key not in {"model", "input", "custom_llm_provider"}
+                },
+            }
+        )
+        return (
+            str(modified_kwargs["model"]) if "model" in modified_kwargs else model,
+            cast(
+                Union[str, ResponseInputParam],
+                modified_kwargs["input"] if "input" in modified_kwargs else input,
+            ),
+            (
+                str(modified_kwargs["custom_llm_provider"])
+                if "custom_llm_provider" in modified_kwargs
+                else custom_llm_provider
+            ),
+            updated_response_params,
+            updated_litellm_params,
+        )
+
     def response_api_handler(
         self,
         model: str,
         input: Union[str, ResponseInputParam],
         responses_api_provider_config: BaseResponsesAPIConfig,
-        response_api_optional_request_params: Dict,
+        response_api_optional_request_params: dict[str, Any],
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
@@ -2276,6 +2402,21 @@ def response_api_handler(
                 shared_session=shared_session,
             )
 
+        (
+            model,
+            input,
+            custom_llm_provider,
+            response_api_optional_request_params,
+            litellm_params,
+        ) = self._run_sync_responses_pre_call_deployment_hook(
+            model=model,
+            input=input,
+            custom_llm_provider=custom_llm_provider,
+            response_api_optional_request_params=response_api_optional_request_params,
+            litellm_params=litellm_params,
+            logging_obj=logging_obj,
+        )
+
         if client is None or not isinstance(client, HTTPHandler):
             sync_httpx_client = _get_httpx_client(
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)}
@@ -2414,9 +2555,27 @@ def response_api_handler(
                 logging_obj=logging_obj,
             )
         )
-        # Responses agentic interception (e.g. code interpreter) runs the follow-up
-        # loop via the async hook, so it is async-only for now; the sync path returns
-        # the initial response unchanged.
+
+        if self._has_agentic_completion_hook(logging_obj):
+            final_response = run_async_function(
+                self._call_agentic_completion_hooks,
+                response=initial_response,
+                model=model,
+                messages=(
+                    input
+                    if isinstance(input, list)
+                    else [{"role": "user", "content": input}]
+                ),
+                anthropic_messages_provider_config=responses_api_provider_config,
+                anthropic_messages_optional_request_params=response_api_optional_request_params,
+                logging_obj=logging_obj,
+                stream=False,
+                custom_llm_provider=custom_llm_provider,
+                kwargs=dict(litellm_params),
+                api_surface="responses",
+            )
+            return final_response if final_response is not None else initial_response
+
         return initial_response
 
     async def async_response_api_handler(
@@ -4772,22 +4931,9 @@ def _has_agentic_completion_hook(logging_obj: Any) -> bool:
         agentic callback is detected too.
         """
         from litellm.integrations.custom_logger import CustomLogger
-        from litellm.litellm_core_utils.litellm_logging import (
-            get_custom_logger_compatible_class,
-        )
 
         base_func = CustomLogger.async_should_run_agentic_loop
-        callbacks = litellm.callbacks + (
-            getattr(logging_obj, "dynamic_success_callbacks", None) or []
-        )
-        for cb in callbacks:
-            if isinstance(cb, str):
-                resolved = get_custom_logger_compatible_class(cb)  # type: ignore[arg-type]
-                if resolved is None:
-                    continue
-                cb = resolved
-            if not isinstance(cb, CustomLogger):
-                continue
+        for cb in _custom_logger_callbacks(logging_obj):
             cb_func = getattr(type(cb), "async_should_run_agentic_loop", base_func)
             if getattr(cb_func, "__func__", cb_func) is not getattr(
                 base_func, "__func__", base_func

litellm/llms/e2b/sandbox/transformation.py

@@ -16,6 +16,7 @@
     BaseSandboxConfig,
     CodeExecutionResult,
     ContainerHandle,
+    SANDBOX_MAX_OUTPUT_BYTES,
 )
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -29,7 +30,7 @@
 E2B_DEFAULT_DOMAIN = "e2b.app"
 JUPYTER_PORT = 49999
 DEFAULT_SANDBOX_TIMEOUT = 300
-MAX_OUTPUT_BYTES = 10 * 1024 * 1024
+MAX_OUTPUT_BYTES = SANDBOX_MAX_OUTPUT_BYTES
 
 
 class E2BSandboxConfig(BaseSandboxConfig):
@@ -49,7 +50,7 @@ async def acreate_sandbox(
         *,
         template: str | None = None,
         timeout: int | None = None,
-        allow_internet_access: bool = True,
+        allow_internet_access: bool | None = None,
         api_key: str | None = None,
         api_base: str | None = None,
         metadata: dict | None = None,
@@ -62,7 +63,9 @@ async def acreate_sandbox(
             "templateID": template or E2B_DEFAULT_TEMPLATE,
             "timeout": timeout if timeout is not None else DEFAULT_SANDBOX_TIMEOUT,
             "secure": True,
-            "allow_internet_access": allow_internet_access,
+            "allow_internet_access": (
+                True if allow_internet_access is None else allow_internet_access
+            ),
         }
         if metadata:
             body["metadata"] = metadata
@@ -168,20 +171,6 @@ def _as_handle(container: Union[ContainerHandle, str]) -> ContainerHandle:
         handle._hidden_params = {}
         return handle
 
-    @staticmethod
-    async def _read_capped_lines(response: httpx.Response) -> list[str]:
-        lines: list[str] = []
-        total = 0
-        async for line in response.aiter_lines():
-            total += len(line.encode("utf-8"))
-            if total > MAX_OUTPUT_BYTES:
-                raise ValueError(
-                    f"Sandbox output exceeded {MAX_OUTPUT_BYTES} bytes; aborting to "
-                    "avoid unbounded memory use."
-                )
-            lines.append(line)
-        return lines
-
     @staticmethod
     def _parse_lines(lines: list[str]) -> CodeExecutionResult:
         def _try_parse(stripped: str):
@@ -192,10 +181,9 @@ def _try_parse(stripped: str):
 
         messages = tuple(
             parsed
-            for stripped in (line.strip() for line in lines)
-            if stripped
-            for parsed in (_try_parse(stripped),)
-            if parsed is not None
+            for line in lines
+            if (stripped := line.strip())
+            if (parsed := _try_parse(stripped)) is not None
         )
 
         def of_type(message_type: str):

litellm/llms/opensandbox/__init__.py

@@ -0,0 +1 @@
+