Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion tests/tools/test_base_environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@

from unittest.mock import MagicMock

from tools.environments.base import BaseEnvironment, _BoundedOutputCollector
from tools.environments.base import (
BaseEnvironment,
_BoundedOutputCollector,
_ThreadedProcessHandle,
)


class _TestableEnv(BaseEnvironment):
Expand Down Expand Up @@ -520,3 +524,53 @@ def test_unique_per_instance(self):
env1 = _TestableEnv()
env2 = _TestableEnv()
assert env1._cwd_marker != env2._cwd_marker


class TestThreadedProcessHandleBackendErrors:
"""A backend/transport failure must be VISIBLE to the caller.

Regression: SDK-backed environments (Modal, Daytona) run commands through
``_ThreadedProcessHandle``. When ``exec_fn`` raised (sandbox died, connection
reset, coroutine timeout) the handle recorded exit code 1 and wrote NOTHING
to stdout, so the agent saw a bare "exit 1" with empty output and could not
distinguish a dead backend from a command that legitimately failed. Modal
lanes responded by parking tasks and asking a human to "reset the execution
backend", even though the same task succeeded on a later retry.
"""

def test_backend_exception_is_written_to_stdout(self):
def boom():
raise RuntimeError("modal sandbox died: connection reset")

handle = _ThreadedProcessHandle(boom)
handle.wait(timeout=5)
output = handle.stdout.read()

assert handle.returncode == 1
# The whole point: output is no longer empty and names the real cause.
assert output != ""
assert "RuntimeError" in output
assert "modal sandbox died" in output

def test_successful_command_output_is_unchanged(self):
def ok():
return ("hello world\n", 0)

handle = _ThreadedProcessHandle(ok)
handle.wait(timeout=5)

assert handle.returncode == 0
assert handle.stdout.read() == "hello world\n"

def test_genuine_nonzero_exit_is_not_annotated(self):
"""A real command that fails must NOT be relabelled as a backend error."""
def failing_command():
return ("ls: no such file\n", 2)

handle = _ThreadedProcessHandle(failing_command)
handle.wait(timeout=5)
output = handle.stdout.read()

assert handle.returncode == 2
assert output == "ls: no such file\n"
assert "backend error" not in output
17 changes: 17 additions & 0 deletions tools/environments/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,23 @@ def _worker():
except Exception as exc:
self._error = exc
self._returncode = 1
# Surface the failure on the handle's stdout. Without this the
# agent sees a bare "exit 1" with EMPTY output and cannot tell a
# backend/transport failure (sandbox died, connection reset,
# coroutine timeout) apart from a command that legitimately
# failed. Observed cost: Modal-backed lanes repeatedly parked
# tasks with "every shell command exits 1 with no output" and
# asked a human to "reset the execution backend", when the same
# task succeeded on a later retry.
try:
os.write(
self._write_fd,
(
f"[backend error] {type(exc).__name__}: {exc}"
).encode("utf-8", errors="replace"),
)
except OSError:
pass
finally:
try:
os.close(self._write_fd)
Expand Down
Loading