Repository navigation
fix(modal): cancel a command without destroying the sandbox #160
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
e58aa78
f11edef
774ab48
c47464d
4f88a90
53a5a1c
3f64cf6
183d076
56dd8f7
6e9d01b
1ee092c
0826002
d091b48
a0c374c
d1cd150
3dad56d
d831ca1
2c4f9b3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,244 @@ | ||
| """Cancellation contract for the Modal backend. | ||
|
|
||
| These assert BEHAVIOR (what cancel does to the sandbox), not the text of the | ||
| kill script. The live tests at the bottom drive real Modal sandboxes and skip | ||
| unless Modal credentials and the SDK are present. | ||
| """ | ||
|
|
||
| import os | ||
| import shlex | ||
|
|
||
| import pytest | ||
|
|
||
| modal_env = pytest.importorskip("tools.environments.modal") | ||
|
|
||
|
|
||
| def test_cancellable_command_preserves_the_original_command(): | ||
| tagged = modal_env._cancellable_command("echo hi && ls", "/tmp/.hermes-cancel/x") | ||
| assert tagged.endswith("echo hi && ls") | ||
|
|
||
|
|
||
| def test_cancellable_command_records_the_shell_pid_not_an_env_var(): | ||
| """Regression: an exported marker is clobbered by the env snapshot. | ||
|
|
||
| ``_wrap_command`` sources the session's ``export -p`` snapshot, so any | ||
| variable this prefix exports is replaced by the bootstrap value on every | ||
| later command — cancel would then match nothing. ``$$`` is written by the | ||
| shell itself and is immune. | ||
| """ | ||
| tagged = modal_env._cancellable_command("sleep 300", "/tmp/.hermes-cancel/x") | ||
| assert "echo $$ >" in tagged | ||
| assert "export HERMES_CANCEL_TOKEN" not in tagged | ||
|
|
||
|
|
||
| def test_cancellable_command_quotes_the_pidfile(): | ||
| nasty = "/tmp/a b; rm -rf /" | ||
| tagged = modal_env._cancellable_command("true", nasty) | ||
| assert shlex.quote(nasty) in tagged | ||
| assert "; rm -rf /" not in tagged.replace(shlex.quote(nasty), "") | ||
|
|
||
|
|
||
| def test_pidfile_write_cannot_break_the_command(): | ||
| """A read-only /tmp must not turn every command into a failure.""" | ||
| tagged = modal_env._cancellable_command("echo hi", "/tmp/.hermes-cancel/x") | ||
| assert "|| true" in tagged.split("echo hi")[0] | ||
|
|
||
|
|
||
| class _FakeProc: | ||
| def __init__(self): | ||
| self.wait = _Aio(0) | ||
|
|
||
|
|
||
| class _Aio: | ||
| """Mimics Modal's ``obj.method.aio(...)`` shape.""" | ||
|
|
||
| def __init__(self, result): | ||
| self._result = result | ||
|
|
||
| async def aio(self, *args, **kwargs): | ||
| return self._result | ||
|
|
||
|
|
||
| class _SandboxExec: | ||
| def __init__(self, sandbox): | ||
| self._sandbox = sandbox | ||
|
|
||
| async def aio(self, *args, **kwargs): | ||
| self._sandbox.exec_calls.append(args) | ||
| return _FakeProc() | ||
|
|
||
|
|
||
| class _SandboxTerminate: | ||
| def __init__(self, sandbox): | ||
| self._sandbox = sandbox | ||
|
|
||
| async def aio(self, *args, **kwargs): | ||
| self._sandbox.terminated = True | ||
|
|
||
|
|
||
| class _RecordingSandbox: | ||
| def __init__(self): | ||
| self.exec_calls = [] | ||
| self.terminated = False | ||
| self.exec = _SandboxExec(self) | ||
| self.terminate = _SandboxTerminate(self) | ||
|
|
||
|
|
||
| class _InlineWorker: | ||
| def run_coroutine(self, coro, timeout=600): | ||
| import asyncio | ||
|
|
||
| return asyncio.run(coro) | ||
|
|
||
|
|
||
| def _make_env(sandbox): | ||
| env = modal_env.ModalEnvironment.__new__(modal_env.ModalEnvironment) | ||
| env._sandbox = sandbox | ||
| env._worker = _InlineWorker() | ||
| return env | ||
|
|
||
|
|
||
| def test_cancel_does_not_terminate_the_sandbox(): | ||
| """The regression: cancelling one command used to destroy the session.""" | ||
| sandbox = _RecordingSandbox() | ||
| env = _make_env(sandbox) | ||
|
|
||
| handle = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5) | ||
| handle.kill() | ||
|
|
||
| assert sandbox.terminated is False, "cancel must not tear down the sandbox" | ||
|
|
||
|
|
||
| def test_cancel_issues_a_kill_through_the_same_sandbox(): | ||
| sandbox = _RecordingSandbox() | ||
| env = _make_env(sandbox) | ||
|
|
||
| handle = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5) | ||
| handle.kill() | ||
|
|
||
| kill_calls = [c for c in sandbox.exec_calls if modal_env._CANCEL_SCRIPT in c] | ||
| assert len(kill_calls) == 1, f"expected one cancel exec, got {sandbox.exec_calls}" | ||
| assert kill_calls[0][-1].startswith(modal_env._CANCEL_DIR) | ||
|
|
||
|
|
||
| def test_each_command_gets_a_distinct_pidfile(): | ||
| """Cancelling command A must not be able to signal command B.""" | ||
| sandbox = _RecordingSandbox() | ||
| env = _make_env(sandbox) | ||
|
|
||
| a = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5) | ||
| b = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5) | ||
| a.kill() | ||
| b.kill() | ||
|
|
||
| pidfiles = [c[-1] for c in sandbox.exec_calls if modal_env._CANCEL_SCRIPT in c] | ||
| assert len(pidfiles) == 2 | ||
| assert pidfiles[0] != pidfiles[1] | ||
|
|
||
|
|
||
| def test_cancel_failure_is_swallowed_and_leaves_the_sandbox_alone(): | ||
| class _FailingExec: | ||
| async def aio(self, *args, **kwargs): | ||
| raise RuntimeError("transport down") | ||
|
|
||
| sandbox = _RecordingSandbox() | ||
| sandbox.exec = _FailingExec() # type: ignore[assignment] | ||
| env = _make_env(sandbox) | ||
|
|
||
| handle = modal_env.ModalEnvironment._run_bash(env, "sleep 300", timeout=5) | ||
| handle.kill() # must not raise | ||
|
|
||
| assert sandbox.terminated is False | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Live E2E — real Modal sandboxes, real cancellation. | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| _LIVE = bool(os.environ.get("MODAL_TOKEN_ID") and os.environ.get("MODAL_TOKEN_SECRET")) | ||
| _live_only = pytest.mark.skipif(not _LIVE, reason="requires live Modal credentials") | ||
|
|
||
|
|
||
| def _live_cancel_cycle(inner_command: str, name: str): | ||
| """Run ``inner_command``, cancel it, and report (rc, sandbox_alive, state).""" | ||
| import asyncio | ||
|
|
||
| modal = pytest.importorskip("modal") | ||
|
|
||
| async def _run(): | ||
| app = await modal.App.lookup.aio("hermes-cancel-e2e", create_if_missing=True) | ||
| sandbox = await modal.Sandbox.create.aio( | ||
| "sleep", "infinity", image=modal.Image.debian_slim(), app=app, timeout=300, | ||
| ) | ||
| try: | ||
| seed = await sandbox.exec.aio("bash", "-c", "echo persisted > /tmp/state") | ||
| await seed.wait.aio() | ||
|
|
||
| pidfile = modal_env._cancel_pidfile(name) | ||
| target = await sandbox.exec.aio( | ||
| "bash", "-c", modal_env._cancellable_command(inner_command, pidfile), | ||
| timeout=300, | ||
| ) | ||
| await asyncio.sleep(3) | ||
|
|
||
| killer = await sandbox.exec.aio( | ||
| "bash", "-c", modal_env._CANCEL_SCRIPT, "--", pidfile, timeout=30, | ||
| ) | ||
| kill_out = await killer.stdout.read.aio() | ||
| await killer.wait.aio() | ||
|
|
||
| rc = await asyncio.wait_for(target.wait.aio(), timeout=30) | ||
|
|
||
| after = await sandbox.exec.aio("bash", "-c", "cat /tmp/state") | ||
| state = await after.stdout.read.aio() | ||
| await after.wait.aio() | ||
| return rc, state, await sandbox.poll.aio(), kill_out | ||
| finally: | ||
| await sandbox.terminate.aio() | ||
|
|
||
| return asyncio.run(_run()) | ||
|
|
||
|
|
||
| @_live_only | ||
| def test_live_cancel_keeps_the_sandbox_and_its_state_usable(): | ||
| rc, state, poll, kill_out = _live_cancel_cycle("sleep 300", "live-sleep") | ||
|
|
||
| assert "cancelled=1" in kill_out | ||
| assert rc != 0, "cancelled command should report a signal exit" | ||
| assert "persisted" in state, "sandbox filesystem must survive cancellation" | ||
| assert poll is None, "sandbox must still be running after a cancel" | ||
|
|
||
|
|
||
| @_live_only | ||
| def test_live_cancel_reaches_a_shell_builtin_only_command(): | ||
| """Regression: a command that never forks has no tagged child process.""" | ||
| rc, state, poll, kill_out = _live_cancel_cycle("while :; do :; done", "live-builtin") | ||
|
|
||
| assert "cancelled=1" in kill_out | ||
| assert rc != 0 | ||
| assert poll is None | ||
|
|
||
|
|
||
| @_live_only | ||
| def test_live_cancel_survives_the_env_snapshot_being_sourced(): | ||
| """Regression: the real path sources an ``export -p`` snapshot first.""" | ||
| inner = ( | ||
| "echo 'export HERMES_CANCEL_TOKEN=STALE' > /tmp/snap; " | ||
| "source /tmp/snap; sleep 300" | ||
| ) | ||
| rc, state, poll, kill_out = _live_cancel_cycle(inner, "live-snapshot") | ||
|
|
||
| assert "cancelled=1" in kill_out | ||
| assert rc != 0 | ||
| assert poll is None | ||
|
|
||
|
|
||
| @_live_only | ||
| def test_live_cancel_kills_the_whole_child_tree(): | ||
| rc, state, poll, kill_out = _live_cancel_cycle( | ||
| "sleep 300 & sleep 300 & wait", "live-tree" | ||
| ) | ||
|
|
||
| assert "cancelled=1" in kill_out | ||
| assert rc != 0 | ||
| assert poll is None | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,6 +7,7 @@ | |
| import asyncio | ||
| import base64 | ||
| import io | ||
| import itertools | ||
| import logging | ||
| import shlex | ||
| import tarfile | ||
|
|
@@ -34,6 +35,71 @@ | |
| _SNAPSHOT_STORE = get_hermes_home() / "modal_snapshots.json" | ||
| _DIRECT_SNAPSHOT_NAMESPACE = "direct" | ||
|
|
||
| # Cancellation | ||
| # ------------ | ||
| # Modal's ``ContainerProcess`` exposes only poll/wait/stdout/stderr — there is | ||
| # no kill. The obvious workaround, ``sandbox.terminate()``, cancels the command | ||
| # by destroying the entire sandbox: ``poll()`` then returns an exit code and | ||
| # every later ``exec`` raises ``NotFoundError``, so one interrupted command | ||
| # bricks the session (measured against modal==1.3.4). | ||
| # | ||
| # The sandbox itself, however, still accepts ``exec``. So we cancel a command | ||
| # the way a shell would: have the command record its own PID, then run a | ||
| # second exec that signals that process group. The sandbox, its filesystem, | ||
| # and every other running command survive. | ||
| # | ||
| # Why a PID file rather than an environment marker: the command runs under | ||
| # ``_wrap_command``, which sources the session's ``export -p`` snapshot. Any | ||
| # variable we export is therefore overwritten by the snapshot's value on every | ||
| # later command, and a shell-builtin-only command (``while :; do :; done``) | ||
| # never spawns a child carrying it at all. ``$$`` is written by the shell | ||
| # itself, so it is correct in both cases. Both verified against live Modal. | ||
| _CANCEL_DIR = "/tmp/.hermes-cancel" | ||
| _CANCEL_TIMEOUT_SECONDS = 20 | ||
| _cancel_id_counter = itertools.count() | ||
|
|
||
| # TERM the process group first so the command can run traps, then KILL | ||
| # whatever ignored it. A missing/stale PID file is a no-op, never an error: | ||
| # by the time cancel runs the command may already have exited. | ||
| _CANCEL_SCRIPT = r""" | ||
| pidfile="$1" | ||
| [ -f "$pidfile" ] || { echo "cancelled=0"; exit 0; } | ||
|
exiao marked this conversation as resolved.
Resolved
|
||
| pid=$(cat "$pidfile" 2>/dev/null) | ||
| rm -f "$pidfile" 2>/dev/null | ||
| [ -n "$pid" ] && [ -d "/proc/$pid" ] || { echo "cancelled=0"; exit 0; } | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a command backgrounds a child that inherits stdout, such as AGENTS.md reference: AGENTS.md:L49-L49 Useful? React with 👍 / 👎. |
||
| pgid=$(cut -d' ' -f5 "/proc/$pid/stat" 2>/dev/null) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When the recorded shell is replaced by an executable whose Linux AGENTS.md reference: AGENTS.md:L49-L49 Useful? React with 👍 / 👎. |
||
| if [ -n "$pgid" ] && [ "$pgid" != "0" ]; then | ||
| kill -TERM -"$pgid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null | ||
| else | ||
| kill -TERM "$pid" 2>/dev/null | ||
| fi | ||
| sleep 0.3 | ||
| if [ -d "/proc/$pid" ]; then | ||
|
exiao marked this conversation as resolved.
Resolved
|
||
| kill -KILL -"$pgid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null | ||
| fi | ||
| echo "cancelled=1" | ||
| """ | ||
|
|
||
|
|
||
| def _cancel_pidfile(command_id: str) -> str: | ||
| return f"{_CANCEL_DIR}/{command_id}" | ||
|
|
||
|
|
||
| def _cancellable_command(cmd_string: str, pidfile: str) -> str: | ||
| """Prefix ``cmd_string`` so it records its own PID for cancellation. | ||
|
|
||
| ``$$`` is the shell's own PID, written before the command runs, so this | ||
| works for shell builtins that never fork and survives the env snapshot | ||
| that ``_wrap_command`` sources. Writing the file must never break the | ||
| command, hence the ``|| true``. | ||
| """ | ||
| quoted = shlex.quote(pidfile) | ||
| return ( | ||
| f"mkdir -p {shlex.quote(_CANCEL_DIR)} 2>/dev/null; " | ||
| f"echo $$ > {quoted} 2>/dev/null || true; " | ||
| f"{cmd_string}" | ||
|
exiao marked this conversation as resolved.
Resolved
|
||
| ) | ||
|
|
||
|
|
||
| def _load_snapshots() -> dict: | ||
| return _load_json_store(_SNAPSHOT_STORE) | ||
|
|
@@ -165,7 +231,8 @@ class ModalEnvironment(BaseEnvironment): | |
| """Modal cloud execution via native Modal sandboxes. | ||
|
|
||
| Spawn-per-call via _ThreadedProcessHandle wrapping async SDK calls. | ||
| cancel_fn wired to sandbox.terminate for interrupt support. | ||
| cancel_fn signals the running command by token (see _CANCEL_SCRIPT) so an | ||
| interrupt or timeout never destroys the sandbox. | ||
| """ | ||
|
|
||
| _stdin_mode = "heredoc" | ||
|
|
@@ -411,17 +478,42 @@ def _run_bash(self, cmd_string: str, *, login: bool = False, | |
| """Return a _ThreadedProcessHandle wrapping an async Modal sandbox exec.""" | ||
| sandbox = self._sandbox | ||
| worker = self._worker | ||
| pidfile = _cancel_pidfile(f"{id(self):x}-{next(_cancel_id_counter)}") | ||
| tagged_command = _cancellable_command(cmd_string, pidfile) | ||
|
|
||
| def cancel(): | ||
| worker.run_coroutine(sandbox.terminate.aio(), timeout=15) | ||
| """Signal only this command, leaving the sandbox usable. | ||
|
|
||
| Previously this called ``sandbox.terminate()``, which cancelled the | ||
| command by destroying the session: every subsequent command in the | ||
| same environment then failed with an empty exit 1 forever. Killing | ||
| the recorded process group keeps the sandbox, its filesystem, and | ||
| any concurrently running command intact. | ||
| """ | ||
| if sandbox is None: | ||
| return | ||
| try: | ||
| async def _do_cancel(): | ||
| proc = await sandbox.exec.aio( | ||
| "bash", "-c", _CANCEL_SCRIPT, "--", pidfile, | ||
| timeout=_CANCEL_TIMEOUT_SECONDS, | ||
| ) | ||
| return await proc.wait.aio() | ||
|
|
||
| worker.run_coroutine(_do_cancel(), timeout=_CANCEL_TIMEOUT_SECONDS + 10) | ||
| except Exception as exc: | ||
| # Best-effort: the caller has already stopped waiting on this | ||
| # command. Leaving a stray process behind is strictly better | ||
| # than tearing down a live sandbox. | ||
| logger.warning("Modal: could not cancel remote command: %s", exc) | ||
|
|
||
| def exec_fn() -> tuple[str, int]: | ||
| async def _do(): | ||
| args = ["bash"] | ||
| if login: | ||
| args.extend(["-l", "-c", cmd_string]) | ||
| args.extend(["-l", "-c", tagged_command]) | ||
| else: | ||
| args.extend(["-c", cmd_string]) | ||
| args.extend(["-c", tagged_command]) | ||
| process = await sandbox.exec.aio(*args, timeout=timeout) | ||
| stdout = await process.stdout.read.aio() | ||
| stderr = await process.stderr.read.aio() | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This live cancellation helper invokes
sandbox.exec.aio()with_cancellable_commanddirectly, so it bypasses_run_bash's cancel callback, pending-cancel coordination,_ThreadedProcessHandle, and the inherited interrupt/timeout path used in production. Fresh evidence despite the earlier thread is that the current file's only liveModalEnvironment.execute()cancellation-adjacent test is the stale-file sweep at lines 603–624, which never cancels anything; a wiring regression can therefore pass every live test. Drive a real interrupt or timeout throughModalEnvironment.execute()instead.AGENTS.md reference: AGENTS.md:L55-L55
Useful? React with 👍 / 👎.