Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
e58aa78
fix(modal): cancel a command without destroying the sandbox
exiao Jul 26, 2026
f11edef
fix(modal): record the command PID instead of an env marker
exiao Jul 26, 2026
774ab48
fix(modal): escalate to the process group unconditionally
exiao Jul 26, 2026
c47464d
fix(modal): wait for the command to register before giving up on cancel
exiao Jul 26, 2026
4f88a90
fix(modal): replay a cancel that arrives before the command launches
exiao Jul 26, 2026
53a5a1c
fix(modal): coordinate cancellation with exec startup
exiao Jul 26, 2026
3f64cf6
fix(modal): swallow pending cancel replay failures
exiao Jul 26, 2026
183d076
fix(modal): clean cancellation pid files on exit
exiao Jul 26, 2026
56dd8f7
fix(modal): survive a command's own EXIT trap and a spaced comm
exiao Jul 27, 2026
6e9d01b
fix(modal): move pid-file cleanup out of the command's shell
exiao Jul 27, 2026
1ee092c
fix(modal): keep pid-file cleanup off the command's timed path
exiao Jul 27, 2026
0826002
fix(modal): sweep stale pid files instead of reaping per command
exiao Jul 27, 2026
d091b48
test(modal): drive cancellation through execute() and drop source ins…
exiao Jul 27, 2026
a0c374c
fix(modal): kill descendants that detach from the process group
exiao Jul 27, 2026
d1cd150
test(modal): assert the pgid parser's output, not its shell text
exiao Jul 27, 2026
3dad56d
test(modal): replace the remaining change-detector tests with behavio…
exiao Jul 27, 2026
d831ca1
fix(modal): record the process group so cancel survives the wrapper e…
exiao Jul 27, 2026
2c4f9b3
test(modal): compare only the PID field of the pid-file record
exiao Jul 27, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
244 changes: 244 additions & 0 deletions tests/tools/test_modal_cancel.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,244 @@
"""Cancellation contract for the Modal backend.

These assert BEHAVIOR (what cancel does to the sandbox), not the text of the
kill script. The live tests at the bottom drive real Modal sandboxes and skip
unless Modal credentials and the SDK are present.
"""

import os
import shlex

import pytest

modal_env = pytest.importorskip("tools.environments.modal")


def test_cancellable_command_preserves_the_original_command():
tagged = modal_env._cancellable_command("echo hi && ls", "/tmp/.hermes-cancel/x")
assert tagged.endswith("echo hi && ls")


def test_cancellable_command_records_the_shell_pid_not_an_env_var():
"""Regression: an exported marker is clobbered by the env snapshot.

``_wrap_command`` sources the session's ``export -p`` snapshot, so any
variable this prefix exports is replaced by the bootstrap value on every
later command — cancel would then match nothing. ``$$`` is written by the
shell itself and is immune.
"""
tagged = modal_env._cancellable_command("sleep 300", "/tmp/.hermes-cancel/x")
assert "echo $$ >" in tagged
assert "export HERMES_CANCEL_TOKEN" not in tagged


def test_cancellable_command_quotes_the_pidfile():
nasty = "/tmp/a b; rm -rf /"
tagged = modal_env._cancellable_command("true", nasty)
assert shlex.quote(nasty) in tagged
assert "; rm -rf /" not in tagged.replace(shlex.quote(nasty), "")


def test_pidfile_write_cannot_break_the_command():
"""A read-only /tmp must not turn every command into a failure."""
tagged = modal_env._cancellable_command("echo hi", "/tmp/.hermes-cancel/x")
assert "|| true" in tagged.split("echo hi")[0]


class _FakeProc:
def __init__(self):
self.wait = _Aio(0)


class _Aio:
"""Mimics Modal's ``obj.method.aio(...)`` shape."""

def __init__(self, result):
self._result = result

async def aio(self, *args, **kwargs):
return self._result


class _SandboxExec:
def __init__(self, sandbox):
self._sandbox = sandbox

async def aio(self, *args, **kwargs):
self._sandbox.exec_calls.append(args)
return _FakeProc()


class _SandboxTerminate:
def __init__(self, sandbox):
self._sandbox = sandbox

async def aio(self, *args, **kwargs):
self._sandbox.terminated = True


class _RecordingSandbox:
def __init__(self):
self.exec_calls = []
self.terminated = False
self.exec = _SandboxExec(self)
self.terminate = _SandboxTerminate(self)


class _InlineWorker:
def run_coroutine(self, coro, timeout=600):
import asyncio

return asyncio.run(coro)


def _make_env(sandbox):
env = modal_env.ModalEnvironment.__new__(modal_env.ModalEnvironment)
env._sandbox = sandbox
env._worker = _InlineWorker()
return env


def test_cancel_does_not_terminate_the_sandbox():
"""The regression: cancelling one command used to destroy the session."""
sandbox = _RecordingSandbox()
env = _make_env(sandbox)

handle = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5)
handle.kill()

assert sandbox.terminated is False, "cancel must not tear down the sandbox"


def test_cancel_issues_a_kill_through_the_same_sandbox():
sandbox = _RecordingSandbox()
env = _make_env(sandbox)

handle = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5)
handle.kill()

kill_calls = [c for c in sandbox.exec_calls if modal_env._CANCEL_SCRIPT in c]
assert len(kill_calls) == 1, f"expected one cancel exec, got {sandbox.exec_calls}"
assert kill_calls[0][-1].startswith(modal_env._CANCEL_DIR)


def test_each_command_gets_a_distinct_pidfile():
"""Cancelling command A must not be able to signal command B."""
sandbox = _RecordingSandbox()
env = _make_env(sandbox)

a = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5)
b = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5)
a.kill()
b.kill()

pidfiles = [c[-1] for c in sandbox.exec_calls if modal_env._CANCEL_SCRIPT in c]
assert len(pidfiles) == 2
assert pidfiles[0] != pidfiles[1]


def test_cancel_failure_is_swallowed_and_leaves_the_sandbox_alone():
class _FailingExec:
async def aio(self, *args, **kwargs):
raise RuntimeError("transport down")

sandbox = _RecordingSandbox()
sandbox.exec = _FailingExec() # type: ignore[assignment]
env = _make_env(sandbox)

handle = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5)
handle.kill() # must not raise

assert sandbox.terminated is False


# ---------------------------------------------------------------------------
# Live E2E — real Modal sandboxes, real cancellation.
# ---------------------------------------------------------------------------

_LIVE = bool(os.environ.get("MODAL_TOKEN_ID") and os.environ.get("MODAL_TOKEN_SECRET"))
_live_only = pytest.mark.skipif(not _LIVE, reason="requires live Modal credentials")


def _live_cancel_cycle(inner_command: str, name: str):
"""Run ``inner_command``, cancel it, and report (rc, sandbox_alive, state)."""
import asyncio

modal = pytest.importorskip("modal")

async def _run():
app = await modal.App.lookup.aio("hermes-cancel-e2e", create_if_missing=True)
sandbox = await modal.Sandbox.create.aio(
"sleep", "infinity", image=modal.Image.debian_slim(), app=app, timeout=300,
)
try:
seed = await sandbox.exec.aio("bash", "-c", "echo persisted > /tmp/state")
await seed.wait.aio()

pidfile = modal_env._cancel_pidfile(name)
target = await sandbox.exec.aio(
"bash", "-c", modal_env._cancellable_command(inner_command, pidfile),
timeout=300,

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Exercise cancellation through ModalEnvironment.execute

This live cancellation helper invokes sandbox.exec.aio() with _cancellable_command directly, so it bypasses _run_bash's cancel callback, pending-cancel coordination, _ThreadedProcessHandle, and the inherited interrupt/timeout path used in production. Fresh evidence despite the earlier thread is that the current file's only live ModalEnvironment.execute() cancellation-adjacent test is the stale-file sweep at lines 603–624, which never cancels anything; a wiring regression can therefore pass every live test. Drive a real interrupt or timeout through ModalEnvironment.execute() instead.

AGENTS.md reference: AGENTS.md:L55-L55

Useful? React with 👍 / 👎.

)
await asyncio.sleep(3)

killer = await sandbox.exec.aio(
"bash", "-c", modal_env._CANCEL_SCRIPT, "--", pidfile, timeout=30,
)
kill_out = await killer.stdout.read.aio()
await killer.wait.aio()

rc = await asyncio.wait_for(target.wait.aio(), timeout=30)

after = await sandbox.exec.aio("bash", "-c", "cat /tmp/state")
state = await after.stdout.read.aio()
await after.wait.aio()
return rc, state, await sandbox.poll.aio(), kill_out
finally:
await sandbox.terminate.aio()

return asyncio.run(_run())


@_live_only
def test_live_cancel_keeps_the_sandbox_and_its_state_usable():
rc, state, poll, kill_out = _live_cancel_cycle("sleep 300", "live-sleep")

assert "cancelled=1" in kill_out
assert rc != 0, "cancelled command should report a signal exit"
assert "persisted" in state, "sandbox filesystem must survive cancellation"
assert poll is None, "sandbox must still be running after a cancel"


@_live_only
def test_live_cancel_reaches_a_shell_builtin_only_command():
"""Regression: a command that never forks has no tagged child process."""
rc, state, poll, kill_out = _live_cancel_cycle("while :; do :; done", "live-builtin")

assert "cancelled=1" in kill_out
assert rc != 0
assert poll is None


@_live_only
def test_live_cancel_survives_the_env_snapshot_being_sourced():
"""Regression: the real path sources an ``export -p`` snapshot first."""
inner = (
"echo 'export HERMES_CANCEL_TOKEN=STALE' > /tmp/snap; "
"source /tmp/snap; sleep 300"
)
rc, state, poll, kill_out = _live_cancel_cycle(inner, "live-snapshot")

assert "cancelled=1" in kill_out
assert rc != 0
assert poll is None


@_live_only
def test_live_cancel_kills_the_whole_child_tree():
rc, state, poll, kill_out = _live_cancel_cycle(
"sleep 300 & sleep 300 & wait", "live-tree"
)

assert "cancelled=1" in kill_out
assert rc != 0
assert poll is None
100 changes: 96 additions & 4 deletions tools/environments/modal.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import asyncio
import base64
import io
import itertools
import logging
import shlex
import tarfile
Expand Down Expand Up @@ -34,6 +35,71 @@
_SNAPSHOT_STORE = get_hermes_home() / "modal_snapshots.json"
_DIRECT_SNAPSHOT_NAMESPACE = "direct"

# Cancellation
# ------------
# Modal's ``ContainerProcess`` exposes only poll/wait/stdout/stderr — there is
# no kill. The obvious workaround, ``sandbox.terminate()``, cancels the command
# by destroying the entire sandbox: ``poll()`` then returns an exit code and
# every later ``exec`` raises ``NotFoundError``, so one interrupted command
# bricks the session (measured against modal==1.3.4).
#
# The sandbox itself, however, still accepts ``exec``. So we cancel a command
# the way a shell would: have the command record its own PID, then run a
# second exec that signals that process group. The sandbox, its filesystem,
# and every other running command survive.
#
# Why a PID file rather than an environment marker: the command runs under
# ``_wrap_command``, which sources the session's ``export -p`` snapshot. Any
# variable we export is therefore overwritten by the snapshot's value on every
# later command, and a shell-builtin-only command (``while :; do :; done``)
# never spawns a child carrying it at all. ``$$`` is written by the shell
# itself, so it is correct in both cases. Both verified against live Modal.
_CANCEL_DIR = "/tmp/.hermes-cancel"
_CANCEL_TIMEOUT_SECONDS = 20
_cancel_id_counter = itertools.count()

# TERM the process group first so the command can run traps, then KILL
# whatever ignored it. A missing/stale PID file is a no-op, never an error:
# by the time cancel runs the command may already have exited.
_CANCEL_SCRIPT = r"""
pidfile="$1"
[ -f "$pidfile" ] || { echo "cancelled=0"; exit 0; }
Comment thread
exiao marked this conversation as resolved.
Resolved
pid=$(cat "$pidfile" 2>/dev/null)
rm -f "$pidfile" 2>/dev/null
[ -n "$pid" ] && [ -d "/proc/$pid" ] || { echo "cancelled=0"; exit 0; }

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Retain the process group after the wrapper exits

When a command backgrounds a child that inherits stdout, such as sleep 300 &, Bash exits immediately but process.stdout.read.aio() at line 627 can keep the handle pending because the child still owns the stream. An interrupt or timeout then reaches this check after the recorded PID has disappeared, returns cancelled=0, and leaves the child running until its Modal timeout. Record the process group while the wrapper is alive so cancellation can still signal it after the wrapper exits.

AGENTS.md reference: AGENTS.md:L49-L49

Useful? React with 👍 / 👎.

pgid=$(cut -d' ' -f5 "/proc/$pid/stat" 2>/dev/null)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Parse the process group after the stat command name

When the recorded shell is replaced by an executable whose Linux comm contains whitespace, /proc/$pid/stat includes that whitespace inside its parenthesized second field, so cut -d' ' -f5 returns the PPID rather than the PGID. Cancellation then signals an unrelated group if one exists and falls back to killing only the recorded PID, allowing its descendants to survive; parse fields only after the closing ) (or query the PGID through a parser that understands /proc stat format).

AGENTS.md reference: AGENTS.md:L49-L49

Useful? React with 👍 / 👎.

if [ -n "$pgid" ] && [ "$pgid" != "0" ]; then
kill -TERM -"$pgid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null
else
kill -TERM "$pid" 2>/dev/null
fi
sleep 0.3
if [ -d "/proc/$pid" ]; then
Comment thread
exiao marked this conversation as resolved.
Resolved
kill -KILL -"$pgid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null
fi
echo "cancelled=1"
"""


def _cancel_pidfile(command_id: str) -> str:
return f"{_CANCEL_DIR}/{command_id}"


def _cancellable_command(cmd_string: str, pidfile: str) -> str:
"""Prefix ``cmd_string`` so it records its own PID for cancellation.

``$$`` is the shell's own PID, written before the command runs, so this
works for shell builtins that never fork and survives the env snapshot
that ``_wrap_command`` sources. Writing the file must never break the
command, hence the ``|| true``.
"""
quoted = shlex.quote(pidfile)
return (
f"mkdir -p {shlex.quote(_CANCEL_DIR)} 2>/dev/null; "
f"echo $$ > {quoted} 2>/dev/null || true; "
f"{cmd_string}"
Comment thread
exiao marked this conversation as resolved.
Resolved
)


def _load_snapshots() -> dict:
return _load_json_store(_SNAPSHOT_STORE)
Expand Down Expand Up @@ -165,7 +231,8 @@ class ModalEnvironment(BaseEnvironment):
"""Modal cloud execution via native Modal sandboxes.

Spawn-per-call via _ThreadedProcessHandle wrapping async SDK calls.
cancel_fn wired to sandbox.terminate for interrupt support.
cancel_fn signals the running command by token (see _CANCEL_SCRIPT) so an
interrupt or timeout never destroys the sandbox.
"""

_stdin_mode = "heredoc"
Expand Down Expand Up @@ -411,17 +478,42 @@ def _run_bash(self, cmd_string: str, *, login: bool = False,
"""Return a _ThreadedProcessHandle wrapping an async Modal sandbox exec."""
sandbox = self._sandbox
worker = self._worker
pidfile = _cancel_pidfile(f"{id(self):x}-{next(_cancel_id_counter)}")
tagged_command = _cancellable_command(cmd_string, pidfile)

def cancel():
worker.run_coroutine(sandbox.terminate.aio(), timeout=15)
"""Signal only this command, leaving the sandbox usable.

Previously this called ``sandbox.terminate()``, which cancelled the
command by destroying the session: every subsequent command in the
same environment then failed with an empty exit 1 forever. Killing
the recorded process group keeps the sandbox, its filesystem, and
any concurrently running command intact.
"""
if sandbox is None:
return
try:
async def _do_cancel():
proc = await sandbox.exec.aio(
"bash", "-c", _CANCEL_SCRIPT, "--", pidfile,
timeout=_CANCEL_TIMEOUT_SECONDS,
)
return await proc.wait.aio()

worker.run_coroutine(_do_cancel(), timeout=_CANCEL_TIMEOUT_SECONDS + 10)
except Exception as exc:
# Best-effort: the caller has already stopped waiting on this
# command. Leaving a stray process behind is strictly better
# than tearing down a live sandbox.
logger.warning("Modal: could not cancel remote command: %s", exc)

def exec_fn() -> tuple[str, int]:
async def _do():
args = ["bash"]
if login:
args.extend(["-l", "-c", cmd_string])
args.extend(["-l", "-c", tagged_command])
else:
args.extend(["-c", cmd_string])
args.extend(["-c", tagged_command])
process = await sandbox.exec.aio(*args, timeout=timeout)
stdout = await process.stdout.read.aio()
stderr = await process.stderr.read.aio()
Expand Down
Loading