Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 45 additions & 9 deletions tests/tools/test_code_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ def _force_local_terminal(monkeypatch):
build_execute_code_schema,
EXECUTE_CODE_SCHEMA,
_TOOL_DOC_LINES,
_resolve_sandbox_tools,
)


Expand Down Expand Up @@ -116,6 +117,26 @@ def test_convenience_helpers_present(self):
self.assertIn("import json, os, socket, shlex, time", src)


class TestSandboxToolResolution(unittest.TestCase):
def test_none_preserves_legacy_full_access(self):
self.assertEqual(_resolve_sandbox_tools(None), SANDBOX_ALLOWED_TOOLS)

def test_empty_list_is_authoritative(self):
self.assertEqual(_resolve_sandbox_tools([]), frozenset())

def test_nonoverlapping_tools_do_not_expand_access(self):
self.assertEqual(
_resolve_sandbox_tools(["execute_code", "vision_analyze"]),
frozenset(),
)

def test_intersection_only_exposes_session_allowed_tools(self):
self.assertEqual(
_resolve_sandbox_tools(["execute_code", "terminal", "read_file"]),
frozenset({"terminal", "read_file"}),
)


@unittest.skipIf(sys.platform == "win32", "UDS not available on Windows")
class TestExecuteCode(unittest.TestCase):
"""Integration tests using the mock dispatcher."""
Expand Down Expand Up @@ -679,8 +700,8 @@ def test_none_enabled_tools_uses_all(self):
self.assertIn("all imports ok", result["output"])

@unittest.skipIf(sys.platform == "win32", "UDS not available on Windows")
def test_empty_enabled_tools_uses_all(self):
"""When enabled_tools is [] (empty), all sandbox tools should be available."""
def test_empty_enabled_tools_blocks_all_sandbox_tools(self):
"""An explicit empty session tool list must not widen sandbox access."""
code = (
"from hermes_tools import terminal, web_search\n"
"print('imports ok')\n"
Expand All @@ -689,13 +710,12 @@ def test_empty_enabled_tools_uses_all(self):
return_value=json.dumps({"ok": True})):
result = json.loads(execute_code(code, task_id="test-empty",
enabled_tools=[]))
self.assertEqual(result["status"], "success")
self.assertIn("imports ok", result["output"])
self.assertEqual(result["status"], "error")
self.assertIn("ImportError", result.get("error", "") + result.get("output", ""))

@unittest.skipIf(sys.platform == "win32", "UDS not available on Windows")
def test_nonoverlapping_tools_fallback(self):
"""When enabled_tools has no overlap with SANDBOX_ALLOWED_TOOLS,
should fall back to all allowed tools."""
def test_nonoverlapping_tools_do_not_fallback(self):
"""A non-overlapping explicit policy must not restore sandbox tools."""
code = (
"from hermes_tools import terminal\n"
"print('fallback ok')\n"
Expand All @@ -706,8 +726,24 @@ def test_nonoverlapping_tools_fallback(self):
code, task_id="test-nonoverlap",
enabled_tools=["vision_analyze", "browser_snapshot"],
))
self.assertEqual(result["status"], "success")
self.assertIn("fallback ok", result["output"])
self.assertEqual(result["status"], "error")
self.assertIn("ImportError", result.get("error", "") + result.get("output", ""))

@unittest.skipIf(sys.platform == "win32", "UDS not available on Windows")
def test_execute_code_only_session_cannot_regain_terminal(self):
"""Real policy case: leaving only execute_code enabled must not expose terminal."""
code = (
"from hermes_tools import terminal\n"
"print('terminal regained')\n"
)
with patch("model_tools.handle_function_call",
return_value=json.dumps({"ok": True})):
result = json.loads(execute_code(
code, task_id="test-execute-code-only",
enabled_tools=["execute_code"],
))
self.assertEqual(result["status"], "error")
self.assertIn("ImportError", result.get("error", "") + result.get("output", ""))


# ---------------------------------------------------------------------------
Expand Down
28 changes: 17 additions & 11 deletions tools/code_execution_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,20 @@ def check_sandbox_requirements() -> bool:
return SANDBOX_AVAILABLE


def _resolve_sandbox_tools(enabled_tools: Optional[List[str]]) -> frozenset[str]:
"""Resolve the exact tool set exposed inside execute_code.

Security boundary:
- ``enabled_tools is None`` means the caller did not provide session tool
context, so we preserve the historical fallback to all sandbox tools.
- Any explicit list, including an empty list or a non-overlapping list,
is treated as authoritative session policy and must not widen access.
"""
if enabled_tools is None:
return SANDBOX_ALLOWED_TOOLS
return frozenset(SANDBOX_ALLOWED_TOOLS & set(enabled_tools))


# ---------------------------------------------------------------------------
# hermes_tools.py code generator
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -698,10 +712,7 @@ def _execute_remote(
timeout = _cfg.get("timeout", DEFAULT_TIMEOUT)
max_tool_calls = _cfg.get("max_tool_calls", DEFAULT_MAX_TOOL_CALLS)

session_tools = set(enabled_tools) if enabled_tools else set()
sandbox_tools = frozenset(SANDBOX_ALLOWED_TOOLS & session_tools)
if not sandbox_tools:
sandbox_tools = SANDBOX_ALLOWED_TOOLS
sandbox_tools = _resolve_sandbox_tools(enabled_tools)

effective_task_id = task_id or "default"
env, env_type = _get_or_create_env(effective_task_id)
Expand Down Expand Up @@ -909,11 +920,7 @@ def execute_code(
max_tool_calls = _cfg.get("max_tool_calls", DEFAULT_MAX_TOOL_CALLS)

# Determine which tools the sandbox can call
session_tools = set(enabled_tools) if enabled_tools else set()
sandbox_tools = frozenset(SANDBOX_ALLOWED_TOOLS & session_tools)

if not sandbox_tools:
sandbox_tools = SANDBOX_ALLOWED_TOOLS
sandbox_tools = _resolve_sandbox_tools(enabled_tools)

# --- Set up temp directory with hermes_tools.py and script.py ---
tmpdir = tempfile.mkdtemp(prefix="hermes_sandbox_")
Expand All @@ -930,8 +937,7 @@ def execute_code(

try:
# Write the auto-generated hermes_tools module
# sandbox_tools is already the correct set (intersection with session
# tools, or SANDBOX_ALLOWED_TOOLS as fallback — see lines above).
# sandbox_tools is already the exact session-bounded allow-list.
tools_src = generate_hermes_tools_module(list(sandbox_tools))
with open(os.path.join(tmpdir, "hermes_tools.py"), "w") as f:
f.write(tools_src)
Expand Down
Loading