From b9c15777083476d9e06ec4823ccc1aea3f0b2018 Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 7 Oct 2026 16:44:26 -0700 Subject: [PATCH 01/21] feat: support persistent execution workers outside the agent loop --- README.md | 25 +++ src/rlm/acp.py | 6 + src/rlm/agent.py | 5 +- src/rlm/broker.py | 7 +- src/rlm/config.py | 5 + src/rlm/engine.py | 23 ++- src/rlm/execution.py | 346 ++++++++++++++++++++++++++++++++++ src/rlm/history.py | 13 +- src/rlm/prompt.py | 13 +- src/rlm/supervisor.py | 4 + tests/test_kernel_recovery.py | 107 +++++++++++ 11 files changed, 547 insertions(+), 7 deletions(-) create mode 100644 src/rlm/execution.py diff --git a/README.md b/README.md index d6094ecd..3e6efc1a 100644 --- a/README.md +++ b/README.md @@ -550,3 +550,28 @@ stops with a failure event when the inbox event limit is reached. There are no model-authored callbacks, event selectors, or output predicates. Activity subscriptions become `completed` after their target permanently terminates, flushing pending activity and releasing their active slot. Watches of idle persistent agents remain active. Watching an already-finished target returns a completed subscription. + +### Separate tool execution + +The ACP runtime metadata accepts `execution_command` (a trusted argv launching +`python -m rlm.execution`) and `execution_cwd` (the task workspace). For example, +an orchestrator can start the worker with `docker exec -i +/path/to/python -m rlm.execution`. Both processes must have the same nano-rlm +version installed. The orchestrator owns container provisioning and teardown. + +The engine, model credentials, supervisor, and authoritative conversation logs +remain in the harness process. Persistent IPython kernels and builtin skills run +in the worker environment. Recursive agents inherit the execution destination, +keep separate kernels, and share the task workspace. Communication and recursive +callbacks use worker stdio, so a task container can have networking disabled. +The command itself defines the isolation boundary; launching a local Python +worker alone provides process separation, not filesystem isolation. + +Split execution supports the `ipython` builtin and its skills. Native bash/edit/fetch +tools are rejected; use their builtin skills instead. Supervisor shell jobs +(`rlm.shell`) and filesystem watchers (`rlm.watch.path`) return explicit errors in +this mode. `await child.history()` uses the supervisor; the current agent's local +history is a snapshot refreshed before each cell. Worker frames, including history +snapshots and outputs, are limited to 16 MiB. Kernel recovery preserves the existing +no-replay behavior. Worker disconnects fail the session; the orchestrator must stop +remaining task processes when the rollout ends. diff --git a/src/rlm/acp.py b/src/rlm/acp.py index 8aacfdee..99a8ae3e 100644 --- a/src/rlm/acp.py +++ b/src/rlm/acp.py @@ -75,6 +75,8 @@ class _RuntimeMetadata(_ContractModel): leaf_append_to_system_prompt: str | None = None skills: list[Annotated[str, Field(min_length=1)]] builtin_tools: list[Annotated[str, Field(min_length=1)]] | None = None + execution_cwd: str | None = None + execution_command: list[Annotated[str, Field(min_length=1)]] | None = None kernel_env: dict[str, str] search_api_key: str | None network_block: list[str] = Field(default_factory=list) @@ -216,6 +218,10 @@ def _runtime_config(meta_kwargs: Any) -> tuple[RuntimeConfig, str]: else None ), kernel_env=tuple(payload.kernel_env.items()), + execution_cwd=payload.execution_cwd, + execution_command=tuple(payload.execution_command) + if payload.execution_command is not None + else None, search_api_key=payload.search_api_key, network_block=tuple(payload.network_block), ), diff --git a/src/rlm/agent.py b/src/rlm/agent.py index 507eb7cc..161f646d 100644 --- a/src/rlm/agent.py +++ b/src/rlm/agent.py @@ -48,7 +48,10 @@ async def info(self) -> AgentInfo: async def history(self) -> History: """Read a fresh snapshot of this agent's local conversation history.""" - return await history(session_dir=self.session_dir) + if not broker.is_active(): + return await history(session_dir=self.session_dir) + payload = await broker.agent_request("agent.history", agent_id=self.id) + return History.from_events(self.session_dir, payload) async def result(self, *, yield_after: float = 300) -> AgentResult: """Wait up to yield_after seconds for the child to finish, then return its latest state. diff --git a/src/rlm/broker.py b/src/rlm/broker.py index 354702ed..2e2d48f0 100644 --- a/src/rlm/broker.py +++ b/src/rlm/broker.py @@ -79,7 +79,7 @@ class BrokerAgentListRequest(TypedDict): class BrokerAgentHandleRequest(TypedDict): __pydantic_config__ = ConfigDict(extra="forbid", strict=True) - op: Literal["agent.info", "agent.cancel"] + op: Literal["agent.info", "agent.cancel", "agent.history"] capability: Annotated[str, Field(min_length=1)] scope_id: Annotated[str, Field(min_length=1)] agent_id: Annotated[str, Field(min_length=1)] @@ -403,6 +403,11 @@ def result_from_payload(value: dict[str, Any]) -> AgentResult: raise RuntimeError("invalid response from RLM supervisor") from None +def is_active() -> bool: + """Whether the current process is inside a brokered execution scope.""" + return _endpoint is not None and _scope_id is not None + + def configure(endpoint: BrokerEndpoint | None) -> None: global _endpoint _endpoint = endpoint diff --git a/src/rlm/config.py b/src/rlm/config.py index 0a819b53..f7c7f457 100644 --- a/src/rlm/config.py +++ b/src/rlm/config.py @@ -112,6 +112,11 @@ class RuntimeConfig(_ConfigModel): """Builtin tool set for every engine in the tree; None = the registry default (`ipython` alone). Validated against the registry when the engine starts.""" kernel_env: tuple[tuple[str, str], ...] = Field(default=(), repr=False) + execution_cwd: str | None = None + execution_command: tuple[str, ...] | None = None + """Trusted argv launching an rlm.execution worker in the task environment. + None starts the kernel locally. Split execution currently supports IPython + and its skills; supervisor shell jobs and path watchers are unavailable.""" search_api_key: str | None = Field(default=None, repr=False) network_block: NetworkBlocklist = () """Search-result destination rules shared by every agent in the session tree.""" diff --git a/src/rlm/engine.py b/src/rlm/engine.py index 8e9ebeee..ab5b1290 100644 --- a/src/rlm/engine.py +++ b/src/rlm/engine.py @@ -201,7 +201,7 @@ def __init__( self.runtime_config = runtime_config config = self.runtime_config self.model = config.model - self.cwd = cwd or os.getcwd() + self.cwd = config.execution_cwd or cwd or os.getcwd() self.exec_timeout = config.policy.exec_timeout self.max_total_turns = config.policy.max_total_turns self.max_tool_output_bytes = config.policy.max_tool_output_bytes @@ -520,6 +520,13 @@ async def _start(self, prompt: str) -> None: self._active_tools = get_active_builtin_tools( self.exec_timeout, self.builtin_tools ) + if self.runtime_config.execution_command is not None: + if not self.runtime_config.execution_command: + raise ValueError("execution_command must not be empty") + if any(tool.name != "ipython" for tool in self._active_tools): + raise ValueError( + "split execution supports only the ipython builtin; use skills for bash/edit/fetch" + ) self._active_tool_schemas = [tool.schema() for tool in self._active_tools] if self.compaction and self.summarize_at_tokens is None: @@ -579,7 +586,15 @@ async def _start(self, prompt: str) -> None: self._owns_supervisor = False raise - self._repl = IPythonREPL( + repl_type = IPythonREPL + repl_options = {} + if self.runtime_config.execution_command is not None: + from rlm.execution import RemoteREPL + + repl_type = RemoteREPL + repl_options["command"] = self.runtime_config.execution_command + self._repl = repl_type( + **repl_options, cwd=self.cwd, session=self.session, kernel_env=self.kernel_env, @@ -1481,7 +1496,8 @@ def _load_system_prompt(self, active_tools: list[BuiltinTool]) -> str: self.cwd, discover_skills(self.session.dir), depth=self.depth, - session_dir=str(self.session.dir), + session_dir=getattr(self._repl, "session_dir", None) + or str(self.session.dir), allow_recursion=self.depth < self.max_depth, delegation_prompt=self.runtime_config.policy.delegation_prompt, allow_git=self.allow_git, @@ -1495,6 +1511,7 @@ def _load_system_prompt(self, active_tools: list[BuiltinTool]) -> str: if self.system_prompt_path else None, extra_instructions=self.append_to_system_prompt, + split_execution=self.runtime_config.execution_command is not None, agent_info=self._supervisor.agent_context(self._invocation_id) if self._supervisor else None, diff --git a/src/rlm/execution.py b/src/rlm/execution.py new file mode 100644 index 00000000..4ddfa42a --- /dev/null +++ b/src/rlm/execution.py @@ -0,0 +1,346 @@ +"""Persistent execution sessions over a trusted worker command's stdio. + +The command is supplied by the orchestrator (for example docker exec -i). Only +kernel configuration and generated skills cross into the worker; model credentials, +conversation ownership, and recursive agent loops stay in the calling process. +""" + +from __future__ import annotations + +import concurrent.futures +import json +import socket +import struct +import subprocess +import sys +import tempfile +import threading +import uuid +from pathlib import Path +from types import SimpleNamespace + +from rlm.broker import BrokerEndpoint +from rlm.tools.ipython import IPythonREPL + +MAX_FRAME = 16 * 1024 * 1024 + + +def _read(stream, size): + chunks = bytearray() + while len(chunks) < size: + chunk = stream.read(size - len(chunks)) + if not chunk: + raise EOFError("execution worker disconnected") + chunks.extend(chunk) + return bytes(chunks) + + +def _receive(stream): + size = struct.unpack("!I", _read(stream, 4))[0] + if size > MAX_FRAME: + raise ValueError("execution frame exceeds 16 MiB") + value = json.loads(_read(stream, size)) + if not isinstance(value, dict): + raise ValueError("execution frame must be an object") + return value + + +def _send(stream, value): + data = json.dumps(value).encode() + if len(data) > MAX_FRAME: + raise ValueError("execution frame exceeds 16 MiB") + stream.write(struct.pack("!I", len(data)) + data) + stream.flush() + + +class _Peer: + """Bidirectional RPC: callbacks must keep flowing during a running cell.""" + + def __init__(self, reader, writer, handler): + self.reader, self.writer, self.handler = reader, writer, handler + self.lock = threading.Lock() + self.pending = {} + self.closed = threading.Event() + self.error = None + self.slots = threading.BoundedSemaphore(32) + self.thread = threading.Thread(target=self._listen, daemon=True) + self.thread.start() + + def send(self, message): + with self.lock: + if self.closed.is_set(): + raise RuntimeError("execution connection closed") from self.error + _send(self.writer, message) + + def call(self, method, params=None, timeout=None): + key = uuid.uuid4().hex + future = concurrent.futures.Future() + with self.lock: + if self.closed.is_set(): + raise RuntimeError("execution connection closed") from self.error + self.pending[key] = future + try: + self.send({"id": key, "method": method, "params": params}) + return future.result(timeout=timeout) + finally: + with self.lock: + self.pending.pop(key, None) + + def _answer(self, message): + try: + result = self.handler(message["method"], message.get("params")) + reply = {"id": message["id"], "result": result} + except Exception as exc: + reply = {"id": message["id"], "error": str(exc)} + try: + try: + self.send(reply) + except ValueError as exc: + self.send({"id": message["id"], "error": str(exc)}) + except (OSError, RuntimeError): + pass # the caller disconnected while its operation was settling + finally: + self.slots.release() + + def _listen(self): + try: + while True: + message = _receive(self.reader) + if "method" in message: + if not self.slots.acquire(blocking=False): + self.send( + { + "id": message["id"], + "error": "execution request limit reached", + } + ) + continue + threading.Thread( + target=self._answer, args=(message,), daemon=True + ).start() + else: + with self.lock: + future = self.pending.get(message["id"]) + if future is not None: + if "error" in message: + future.set_exception(RuntimeError(message["error"])) + else: + future.set_result(message["result"]) + except Exception as exc: + with self.lock: + self.error = exc + self.closed.set() + for future in self.pending.values(): + if not future.done(): + future.set_exception(exc) + + +class RemoteREPL(IPythonREPL): + """The IPython execution-session interface backed by a separate worker.""" + + def __init__(self, *, command, **kwargs): + super().__init__(**kwargs) + self.command = command + self._process = None + self._peer = None + self.session_dir = None + + def _broker(self, method, request): + if method != "broker" or self.broker_endpoint is None: + raise PermissionError("unknown execution callback") + if request.get("capability") != self.broker_endpoint.capability: + raise PermissionError("invalid execution capability") + # These operations currently launch jobs/watchers in the supervisor process. + # Refuse them until they have execution-session implementations. + if ( + request.get("op", "").startswith("shell.") + or request.get("op") == "watch.path" + ): + raise ValueError( + "split execution does not support rlm.shell or watch.path; use bash/IPython in the task workspace" + ) + with socket.socket(socket.AF_UNIX) as connection: + connection.connect(self.broker_endpoint.socket_path) + with connection.makefile("rwb", buffering=0) as stream: + _send(stream, request) + return _receive(stream) + + def start(self): + self._process = subprocess.Popen( + self.command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=None, + start_new_session=True, + ) + self._peer = _Peer(self._process.stdout, self._process.stdin, self._broker) + modules = { + path.name: path.read_text() for path in self.session.dir.glob("*.py") + } + self.session_dir = self._peer.call( + "start", + { + "cwd": self.cwd, + "kernel_env": self.kernel_env, + "depth": self.depth, + "max_depth": self.max_depth, + "exec_timeout": self.exec_timeout, + "allow_git": self.allow_git, + "capability": self.broker_endpoint.capability + if self.broker_endpoint + else None, + "modules": modules, + }, + timeout=120, + ) + + def execute(self, code, timeout=None): + result = self._peer.call( + "execute", + { + "code": code, + "timeout": timeout, + "scope_id": self._scope_id, + "history": (self.session.dir / "messages.jsonl").read_text(), + }, + ) + self._recovery_notices.extend(result["notices"]) + # Programmatic tool statistics are observational, just as with local kernels. + (self.session.dir / "programmatic_tool_calls.jsonl").write_text( + result["tool_log"] + ) + return result["output"] + + def interrupt(self): + if self._peer is not None and not self._peer.closed.is_set(): + self._peer.call("interrupt", timeout=5) + + def finish_interrupt(self): + if self._peer is not None and not self._peer.closed.is_set(): + self._peer.call("finish_interrupt", timeout=5) + + def shutdown(self): + process, self._process = self._process, None + if process is not None: + try: + if self._peer is not None and not self._peer.closed.is_set(): + self._peer.call("close", timeout=30) + finally: + # EOF is also a worker shutdown request if the explicit close failed. + process.stdin.close() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + process.stdout.close() + self._executor.shutdown(wait=False) + + +class _Worker: + def __init__(self): + self.repl = None + self.peer = None + self.directory = tempfile.TemporaryDirectory(prefix="rlm-execution-") + self.session_dir = Path(self.directory.name) + self.listener = None + self.closed = threading.Event() + + def _accept(self): + while not self.closed.is_set(): + try: + connection, _ = self.listener.accept() + except OSError: + return + threading.Thread( + target=self._proxy, args=(connection,), daemon=True + ).start() + + def _proxy(self, connection): + with connection, connection.makefile("rwb", buffering=0) as stream: + try: + request = _receive(stream) + result = self.peer.call("broker", request) + except Exception as exc: + result = {"error": str(exc)} + try: + _send(stream, result) + except OSError: + pass # interrupted kernels close their pending callbacks + + def handle(self, method, params): + if method == "start": + if self.repl is not None: + raise ValueError("execution session already started") + params = dict(params) + modules = params.pop("modules") + for name, source in modules.items(): + if Path(name).name != name or not name.endswith(".py"): + raise ValueError("invalid generated skill name") + (self.session_dir / name).write_text(source) + capability = params.pop("capability") + endpoint = None + if capability: + path = str(self.session_dir / "broker.sock") + self.listener = socket.socket(socket.AF_UNIX) + self.listener.bind(path) + self.listener.listen(32) + threading.Thread(target=self._accept, daemon=True).start() + endpoint = BrokerEndpoint(path, capability) + self.repl = IPythonREPL( + session=SimpleNamespace(dir=self.session_dir), + broker_endpoint=endpoint, + **params, + ) + # All Jupyter operations use the same dedicated thread. + self.repl._executor.submit(self.repl.start).result() + return str(self.session_dir) + if self.repl is None: + raise ValueError("execution session has not started") + if method == "execute": + + def execute(): + (self.session_dir / "messages.jsonl").write_text(params["history"]) + self.repl.set_broker_scope(params["scope_id"]) + output = self.repl.execute(params["code"], params["timeout"]) + log = self.session_dir / "programmatic_tool_calls.jsonl" + return { + "output": output, + "notices": self.repl.take_recovery_notices(), + "tool_log": log.read_text() if log.exists() else "", + } + + return self.repl._executor.submit(execute).result() + if method == "interrupt": + self.repl.interrupt() + elif method == "finish_interrupt": + self.repl.finish_interrupt() + elif method == "close": + self.repl.shutdown() + self.repl = None + else: + raise ValueError(f"unknown execution method: {method}") + + def close(self): + self.closed.set() + if self.listener is not None: + self.listener.close() + if self.repl is not None: + self.repl.interrupt() + self.repl.shutdown() + self.directory.cleanup() + + +def main(): + worker = _Worker() + output = sys.stdout.buffer + sys.stdout = sys.stderr # dependency diagnostics must not corrupt the wire + worker.peer = _Peer(sys.stdin.buffer, output, worker.handle) + try: + worker.peer.closed.wait() + finally: + worker.close() + + +if __name__ == "__main__": + main() diff --git a/src/rlm/history.py b/src/rlm/history.py index a118729c..78d48b56 100644 --- a/src/rlm/history.py +++ b/src/rlm/history.py @@ -35,7 +35,18 @@ class History: def __init__(self, session_dir: str | Path): self.session_dir = Path(session_dir).resolve() - self.events = list(read_records(self.session_dir / "messages.jsonl")) + self._load(list(read_records(self.session_dir / "messages.jsonl"))) + + @classmethod + def from_events(cls, session_dir: str | Path, events: list[dict]) -> History: + """Construct a history snapshot received from the owning supervisor.""" + history = cls.__new__(cls) + history.session_dir = Path(session_dir) + history._load(events) + return history + + def _load(self, events: list[dict]) -> None: + self.events = events self.messages: list[dict] = [] self.windows: list[ContextWindow] = [] for event in self.events: diff --git a/src/rlm/prompt.py b/src/rlm/prompt.py index 81348c72..9ea6bcd4 100644 --- a/src/rlm/prompt.py +++ b/src/rlm/prompt.py @@ -385,6 +385,7 @@ def build_system_prompt( extra_instructions: str | None = None, agent_info: dict | None = None, mcp_servers: list[dict] | None = None, + split_execution: bool = False, ) -> str: """Compose task instructions with the guide for this agent's actual runtime.""" has_ipython = _has_tool(active_tools, "ipython") @@ -444,7 +445,17 @@ def build_system_prompt( if has_ipython: parts.extend( [ - RUNTIME_PROMPT, + ( + "Execution uses a persistent IPython session in the task workspace. " + "Run commands through Python subprocesses or the bash skill, and edit task files there. " + "rlm.shell and rlm.watch.path are unavailable in this execution mode. " + "Recursive agents and their Python kernels share the task workspace. " + "Use await rlm.inbox.list() and await rlm.inbox.read(event_id) for supervisor events. " + "The native wait tool waits for agent events. Your conversation ledger is a snapshot " + "refreshed before each cell; await child.history() reads a child's current history." + if split_execution + else RUNTIME_PROMPT + ), HISTORY_PROMPT, IPYTHON_CONTROL_PROMPT, KERNEL_PACKAGES_PROMPT, diff --git a/src/rlm/supervisor.py b/src/rlm/supervisor.py index 945859e2..093a5c20 100644 --- a/src/rlm/supervisor.py +++ b/src/rlm/supervisor.py @@ -1194,6 +1194,10 @@ async def _agent_operation(self, request: dict) -> Any: ancestor = self._parents[ancestor] return agents child = self._child(parent, request.get("name_or_id", request.get("agent_id"))) + if op == "agent.history": + from rlm.history import read_records + + return list(read_records(child.session.dir / "messages.jsonl")) if op in {"agent.send", "agent.steer"}: if child.status in {"completed", "failed", "cancelled"}: raise RuntimeError("agent is no longer active") diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index f9803fee..52af6ac9 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -263,3 +263,110 @@ async def started(): repl._interrupt_requested.set() await pending await asyncio.to_thread(repl.shutdown) + + +async def test_remote_execution_preserves_state_and_brokers_children(session, tmp_path): + import sys + + config = _config(max_depth=1).model_copy( + update={ + "execution_command": (sys.executable, "-m", "rlm.execution"), + "skills": ("bash", "edit"), + "builtin_tools": ("ipython",), + "policy": _config(max_depth=1).policy.model_copy( + update={"compaction": False} + ), + } + ) + + def factory(**kwargs): + return RLMEngine( + client=DummyClient( + [ + _tool( + "from pathlib import Path; Path('child.txt').write_text('child'); print('CHILD')" + ), + DummyMessage(content="child complete"), + ] + ), + **kwargs, + ) + + supervisor = SessionTreeSupervisor( + root_session=session, + runtime_config=config, + cwd=str(tmp_path), + engine_factory=factory, + ) + client = DummyClient( + [ + _tool("value = 41; print(await bash('pwd'))"), + _tool(""" +from pathlib import Path +assert value == 41 +child = await rlm.agent.spawn('write child.txt') +await child.wait(timeout=30) +assert (await child.result()).answer == 'child complete' +assert 'write child.txt' in (await child.history()).user_messages()[0]['content'] +assert Path('child.txt').read_text() == 'child' +h = await rlm.history() +assert h.user_messages()[0]['content'] == 'run remotely' +try: + await rlm.shell.run('touch escaped') +except RuntimeError as e: + assert 'split execution' in str(e) +else: + raise AssertionError('host shell must be unavailable') +print('REMOTE_OK') +"""), + DummyMessage(content="done"), + ] + ) + engine = RLMEngine( + client=client, + session=session, + runtime_config=config, + cwd=str(tmp_path), + supervisor=supervisor, + ) + try: + await engine.prompt("run remotely") + logs = (session.dir / "messages.jsonl").read_text() + assert any( + record.get("type") == "tool_result" + and "REMOTE_OK" in record.get("content", "") + for record in map(json.loads, logs.splitlines()) + ) + assert not (tmp_path / "escaped").exists() + finally: + await engine.aclose() + await supervisor.aclose() + + +async def test_remote_execution_interrupt_and_recovery(session, tmp_path): + import sys + from rlm.execution import RemoteREPL + + repl = RemoteREPL( + command=(sys.executable, "-m", "rlm.execution"), + cwd=str(tmp_path), + session=session, + ) + try: + await repl.run_in_thread(repl.start) + task = asyncio.create_task( + repl.run_in_thread(repl.execute, "import time; time.sleep(60)", 120) + ) + await asyncio.sleep(1) + await asyncio.to_thread(repl.interrupt) + await asyncio.wait_for(task, 10) + repl.finish_interrupt() + result = await repl.run_in_thread(repl.execute, "print('STILL_ALIVE')", 10) + assert "STILL_ALIVE" in result + await repl.run_in_thread(repl.execute, "import os; os._exit(7)", 10) + assert repl.take_recovery_notices() + assert "RECOVERED" in await repl.run_in_thread( + repl.execute, "print('RECOVERED')", 10 + ) + finally: + await asyncio.to_thread(repl.shutdown) From ab4343b4bb89b129a33178b3158cd39c09df9dfe Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 7 Oct 2026 18:58:21 -0700 Subject: [PATCH 02/21] fix: bound execution output and terminate flooding commands --- src/rlm/tools/bash.py | 66 +++++++++++++++++++++++++++++------ src/rlm/tools/ipython.py | 20 +++++++++-- tests/test_kernel_recovery.py | 21 +++++++++++ 3 files changed, 93 insertions(+), 14 deletions(-) diff --git a/src/rlm/tools/bash.py b/src/rlm/tools/bash.py index 945209d6..4ec67c7a 100644 --- a/src/rlm/tools/bash.py +++ b/src/rlm/tools/bash.py @@ -8,6 +8,10 @@ from __future__ import annotations import os +import selectors +import signal +import time +import contextlib import subprocess from typing import Any @@ -68,19 +72,59 @@ def run_bash( blocked = find_blocked_command(command, allow_git=allow_git) if blocked: return refusal(blocked) - try: - proc = subprocess.run( + limit = 1024 * 1024 + deadline = time.monotonic() + timeout + parts = {"stdout": bytearray(), "stderr": bytearray()} + reason = "" + with ( + subprocess.Popen( ["bash", "-c", command], - capture_output=True, - text=True, - errors="replace", - timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, cwd=cwd or None, - ) - except subprocess.TimeoutExpired: - return f"Error: command timed out after {timeout}s" - out = proc.stdout + (("\n" + proc.stderr) if proc.stderr else "") - if proc.returncode != 0: + start_new_session=True, + ) as proc, + selectors.DefaultSelector() as selector, + ): + selector.register(proc.stdout, selectors.EVENT_READ, "stdout") + selector.register(proc.stderr, selectors.EVENT_READ, "stderr") + try: + while selector.get_map(): + remaining = deadline - time.monotonic() + if remaining <= 0: + reason = f"Error: command timed out after {timeout}s" + break + for key, _ in selector.select(min(remaining, 0.1)): + data = os.read(key.fileobj.fileno(), 65536) + if not data: + selector.unregister(key.fileobj) + continue + available = limit - sum(map(len, parts.values())) + parts[key.data].extend(data[:available]) + if len(data) >= available: + reason = ( + "[command output exceeded 1 MiB; process group terminated]" + ) + break + if reason: + break + if not reason: + try: + proc.wait(timeout=max(0.01, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + reason = f"Error: command timed out after {timeout}s" + finally: + if reason or proc.poll() is None: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + stdout, stderr = ( + parts[key].decode(errors="replace") for key in ("stdout", "stderr") + ) + out = stdout + (("\n" + stderr) if stderr else "") + if reason: + out += "\n" + reason + elif proc.returncode != 0: out += f"\n[exit code {proc.returncode}]" return out.strip() or "(no output)" diff --git a/src/rlm/tools/ipython.py b/src/rlm/tools/ipython.py index 8c152168..41249f9a 100644 --- a/src/rlm/tools/ipython.py +++ b/src/rlm/tools/ipython.py @@ -254,6 +254,7 @@ def __init__(self, output: str = ""): MAX_RECOVERY_ATTEMPTS = 3 +MAX_CELL_OUTPUT_BYTES = 1024 * 1024 class IPythonREPL: @@ -593,6 +594,7 @@ def _execute_locked(self, code: str, timeout: int | None) -> str: deadline = None if timeout is None else time.monotonic() + timeout outputs: list[str] = [] + output_bytes = 0 try: while True: if not self._km.is_alive(): @@ -622,18 +624,30 @@ def _execute_locked(self, code: str, timeout: int | None) -> str: msg_type = msg["msg_type"] content = msg["content"] + text = "" if msg_type == "stream": - outputs.append(content["text"]) + text = content["text"] elif msg_type == "execute_result": text = content.get("data", {}).get("text/plain", "") if text: - outputs.append(text + "\n") + text += "\n" elif msg_type == "error": tb = "\n".join(content.get("traceback", [])) tb = _ANSI_RE.sub("", tb) - outputs.append(tb) + text = tb elif msg_type == "status" and content["execution_state"] == "idle": break + if text: + data = text.encode("utf-8", errors="replace") + remaining = MAX_CELL_OUTPUT_BYTES - output_bytes + outputs.append(data[:remaining].decode("utf-8", errors="replace")) + output_bytes += min(len(data), remaining) + if len(data) >= remaining: + self._interrupt_and_recover(msg_id) + outputs.append( + "\n[cell output exceeded 1 MiB; execution interrupted]" + ) + break finally: try: timeout = 0.1 if self._interrupt_requested.is_set() else 5 diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 52af6ac9..c7a7a3fc 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -370,3 +370,24 @@ async def test_remote_execution_interrupt_and_recovery(session, tmp_path): ) finally: await asyncio.to_thread(repl.shutdown) + + +async def test_output_flood_is_bounded_and_kernel_can_continue(session): + import asyncio + from rlm.tools.ipython import MAX_CELL_OUTPUT_BYTES + + repl = IPythonREPL(session=session, cwd=str(session.dir)) + try: + await repl.run_in_thread(repl.start) + output = await asyncio.wait_for( + repl.run_in_thread( + repl.execute, "while True: print('x' * 65536, flush=True)", 15 + ), + timeout=25, + ) + assert len(output.encode()) < MAX_CELL_OUTPUT_BYTES + 256 + assert "output exceeded" in output + continued = await repl.run_in_thread(repl.execute, "print('STILL_ALIVE')", 10) + assert "STILL_ALIVE" in continued + finally: + await repl.run_in_thread(repl.shutdown) From aff5203462c562c33aa27ad94b6408cf268b6e40 Mon Sep 17 00:00:00 2001 From: hallerite Date: Tue, 13 Oct 2026 11:11:03 +0200 Subject: [PATCH 03/21] feat: proxy workspace tools over pluggable execution transports --- README.md | 19 +++- src/rlm/acp.py | 65 +++++++------ src/rlm/config.py | 30 +++++- src/rlm/engine.py | 28 +++--- src/rlm/execution.py | 168 ++++++++++++++++++++++++--------- src/rlm/execution_transport.py | 80 ++++++++++++++++ src/rlm/tools/base.py | 3 + src/rlm/tools/bash.py | 9 ++ tests/test_acp.py | 8 ++ 9 files changed, 322 insertions(+), 88 deletions(-) create mode 100644 src/rlm/execution_transport.py diff --git a/README.md b/README.md index 3e6efc1a..6fe20370 100644 --- a/README.md +++ b/README.md @@ -560,15 +560,28 @@ an orchestrator can start the worker with `docker exec -i version installed. The orchestrator owns container provisioning and teardown. The engine, model credentials, supervisor, and authoritative conversation logs -remain in the harness process. Persistent IPython kernels and builtin skills run +remain in the harness process. Native tools, persistent IPython kernels, and builtin skills run in the worker environment. Recursive agents inherit the execution destination, keep separate kernels, and share the task workspace. Communication and recursive callbacks use worker stdio, so a task container can have networking disabled. The command itself defines the isolation boundary; launching a local Python worker alone provides process separation, not filesystem isolation. -Split execution supports the `ipython` builtin and its skills. Native bash/edit/fetch -tools are rejected; use their builtin skills instead. Supervisor shell jobs +ACP initialization advertises `ai.prime.rlm/execution-v1` with the supported +transports (`stdio`, `tcp`) and tools (`ipython`, `bash`, `edit`, `fetch`). Tool +calls cross the boundary as a name and arguments; filesystem operations happen +inside the worker. `ExecutionTransport` supplies `connect()` (reader and writer) +and `close()` independently of tool dispatch. + +For a trusted private TCP network, launch `python -m rlm.execution --listen +0.0.0.0 --port 8765` with a random `RLM_EXECUTION_TOKEN` of at least 32 characters. +Set `execution_tcp` to `{"host": "worker-address", "port": 8765, "token": "..."}` +instead of `execution_command`. Each connection owns a separate execution session. +The token authenticates connections; this transport does not encrypt traffic, so +use a private network or an encrypted tunnel. The worker removes its token from +the environment before starting tools. The orchestrator owns listener teardown. + +Supervisor shell jobs (`rlm.shell`) and filesystem watchers (`rlm.watch.path`) return explicit errors in this mode. `await child.history()` uses the supervisor; the current agent's local history is a snapshot refreshed before each cell. Worker frames, including history diff --git a/src/rlm/acp.py b/src/rlm/acp.py index 99a8ae3e..7f173e03 100644 --- a/src/rlm/acp.py +++ b/src/rlm/acp.py @@ -45,6 +45,7 @@ from rlm.engine import RLMEngine from rlm.config import ( ExecutionPolicy, + TcpExecutionConfig, InvocationContext, ProviderConfig, RuntimeConfig, @@ -75,6 +76,7 @@ class _RuntimeMetadata(_ContractModel): leaf_append_to_system_prompt: str | None = None skills: list[Annotated[str, Field(min_length=1)]] builtin_tools: list[Annotated[str, Field(min_length=1)]] | None = None + execution_tcp: TcpExecutionConfig | None = None execution_cwd: str | None = None execution_command: list[Annotated[str, Field(min_length=1)]] | None = None kernel_env: dict[str, str] @@ -191,6 +193,33 @@ def _runtime_config(meta_kwargs: Any) -> tuple[RuntimeConfig, str]: payload = _RuntimeMetadata.model_validate( meta_kwargs.get(RUNTIME_METADATA_KEY), ) + result = ( + RuntimeConfig( + model=payload.model, + provider=payload.provider, + invocation=InvocationContext(), + policy=payload.policy, + system_prompt_path=payload.system_prompt_path, + append_to_system_prompt=payload.append_to_system_prompt, + subagent_append_to_system_prompt=payload.subagent_append_to_system_prompt, + leaf_append_to_system_prompt=payload.leaf_append_to_system_prompt, + skills=tuple(payload.skills), + builtin_tools=( + tuple(payload.builtin_tools) + if payload.builtin_tools is not None + else None + ), + kernel_env=tuple(payload.kernel_env.items()), + execution_tcp=payload.execution_tcp, + execution_cwd=payload.execution_cwd, + execution_command=tuple(payload.execution_command) + if payload.execution_command is not None + else None, + search_api_key=payload.search_api_key, + network_block=tuple(payload.network_block), + ), + payload.session_id, + ) except ValidationError as error: raise RequestError.invalid_params( { @@ -201,32 +230,7 @@ def _runtime_config(meta_kwargs: Any) -> tuple[RuntimeConfig, str]: } ) from error - return ( - RuntimeConfig( - model=payload.model, - provider=payload.provider, - invocation=InvocationContext(), - policy=payload.policy, - system_prompt_path=payload.system_prompt_path, - append_to_system_prompt=payload.append_to_system_prompt, - subagent_append_to_system_prompt=payload.subagent_append_to_system_prompt, - leaf_append_to_system_prompt=payload.leaf_append_to_system_prompt, - skills=tuple(payload.skills), - builtin_tools=( - tuple(payload.builtin_tools) - if payload.builtin_tools is not None - else None - ), - kernel_env=tuple(payload.kernel_env.items()), - execution_cwd=payload.execution_cwd, - execution_command=tuple(payload.execution_command) - if payload.execution_command is not None - else None, - search_api_key=payload.search_api_key, - network_block=tuple(payload.network_block), - ), - payload.session_id, - ) + return result def _mcp_servers( @@ -299,7 +303,14 @@ async def initialize( ), ), agent_info=Implementation(name="rlm", title="RLM", version=version("rlm")), - field_meta={CONTRACT_METADATA_KEY: True, "steering": {"supported": True}}, + field_meta={ + CONTRACT_METADATA_KEY: True, + "steering": {"supported": True}, + "ai.prime.rlm/execution-v1": { + "transports": ["stdio", "tcp"], + "tools": ["ipython", "bash", "edit", "fetch"], + }, + }, ) async def new_session( diff --git a/src/rlm/config.py b/src/rlm/config.py index f7c7f457..7aa932bb 100644 --- a/src/rlm/config.py +++ b/src/rlm/config.py @@ -7,7 +7,7 @@ from __future__ import annotations -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from rlm.network import NetworkBlocklist from rlm.semantic import ACP_EXTENSION_HEADER_NAMES @@ -96,6 +96,14 @@ class ExecutionPolicy(_ConfigModel): allow_git: bool = False +class TcpExecutionConfig(_ConfigModel): + """Authenticated worker endpoint on a trusted private network.""" + + host: str = Field(min_length=1) + port: int = Field(gt=0, le=65535) + token: str = Field(min_length=32, repr=False) + + class RuntimeConfig(_ConfigModel): """Configuration resolved once at an RLM process boundary.""" @@ -114,13 +122,27 @@ class RuntimeConfig(_ConfigModel): kernel_env: tuple[tuple[str, str], ...] = Field(default=(), repr=False) execution_cwd: str | None = None execution_command: tuple[str, ...] | None = None - """Trusted argv launching an rlm.execution worker in the task environment. - None starts the kernel locally. Split execution currently supports IPython - and its skills; supervisor shell jobs and path watchers are unavailable.""" + """Trusted argv launching an rlm.execution worker in the task environment.""" + execution_tcp: TcpExecutionConfig | None = None search_api_key: str | None = Field(default=None, repr=False) network_block: NetworkBlocklist = () """Search-result destination rules shared by every agent in the session tree.""" + @model_validator(mode="after") + def _validate_execution(self): + if self.execution_command is not None: + if not self.execution_command or any( + not arg for arg in self.execution_command + ): + raise ValueError("execution_command must contain nonempty arguments") + if self.execution_tcp is not None: + raise ValueError("choose execution_command or execution_tcp") + return self + + @property + def split_execution(self) -> bool: + return self.execution_command is not None or self.execution_tcp is not None + @property def resolved_append_to_system_prompt(self) -> str | None: """The append for this engine's role in the session tree. diff --git a/src/rlm/engine.py b/src/rlm/engine.py index ab5b1290..d7211ee6 100644 --- a/src/rlm/engine.py +++ b/src/rlm/engine.py @@ -520,13 +520,10 @@ async def _start(self, prompt: str) -> None: self._active_tools = get_active_builtin_tools( self.exec_timeout, self.builtin_tools ) - if self.runtime_config.execution_command is not None: - if not self.runtime_config.execution_command: - raise ValueError("execution_command must not be empty") - if any(tool.name != "ipython" for tool in self._active_tools): - raise ValueError( - "split execution supports only the ipython builtin; use skills for bash/edit/fetch" - ) + if self.runtime_config.split_execution: + from rlm.execution import validate_tools + + validate_tools(tool.name for tool in self._active_tools) self._active_tool_schemas = [tool.schema() for tool in self._active_tools] if self.compaction and self.summarize_at_tokens is None: @@ -588,11 +585,13 @@ async def _start(self, prompt: str) -> None: repl_type = IPythonREPL repl_options = {} - if self.runtime_config.execution_command is not None: + if self.runtime_config.split_execution: from rlm.execution import RemoteREPL repl_type = RemoteREPL repl_options["command"] = self.runtime_config.execution_command + repl_options["tcp"] = self.runtime_config.execution_tcp + repl_options["tools"] = [tool.name for tool in self._active_tools] self._repl = repl_type( **repl_options, cwd=self.cwd, @@ -939,9 +938,14 @@ async def _run_loop(self) -> RLMResult: if scope_id is not None: repl.set_broker_scope(scope_id) run = repl.run_in_thread if repl is not None else asyncio.to_thread - tool_task = asyncio.create_task( - run(tool.execute, tool_args, self._tool_context(messages)) - ) + if self.runtime_config.split_execution: + tool_task = asyncio.create_task( + run(repl.execute_tool, tool.name, tool_args) + ) + else: + tool_task = asyncio.create_task( + run(tool.execute, tool_args, self._tool_context(messages)) + ) try: tool_result = await asyncio.shield(tool_task) except asyncio.CancelledError: @@ -1511,7 +1515,7 @@ def _load_system_prompt(self, active_tools: list[BuiltinTool]) -> str: if self.system_prompt_path else None, extra_instructions=self.append_to_system_prompt, - split_execution=self.runtime_config.execution_command is not None, + split_execution=self.runtime_config.split_execution, agent_info=self._supervisor.agent_context(self._invocation_id) if self._supervisor else None, diff --git a/src/rlm/execution.py b/src/rlm/execution.py index 4ddfa42a..6135b561 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -1,4 +1,4 @@ -"""Persistent execution sessions over a trusted worker command's stdio. +"""Tool-call execution sessions over orchestrator-supplied transports. The command is supplied by the orchestrator (for example docker exec -i). Only kernel configuration and generated skills cross into the worker; model credentials, @@ -11,7 +11,10 @@ import json import socket import struct -import subprocess +import argparse +import hmac +import os +from dataclasses import asdict import sys import tempfile import threading @@ -20,7 +23,20 @@ from types import SimpleNamespace from rlm.broker import BrokerEndpoint -from rlm.tools.ipython import IPythonREPL +from rlm.tools.ipython import IPythonREPL, build_kernel_env +from rlm.tools.base import ToolContext, ToolOutcome +from rlm.tools.registry import get_builtin_tool +from rlm.types import IpythonExecuted, RLMMetrics, TokenUsage +from rlm.execution_transport import ExecutionTransport, StdioTransport, TcpTransport + +WORKSPACE_TOOLS = ("ipython", "bash", "edit", "fetch") + + +def validate_tools(names): + unknown = set(names) - set(WORKSPACE_TOOLS) + if unknown: + raise ValueError(f"split execution does not support tools: {sorted(unknown)}") + MAX_FRAME = 16 * 1024 * 1024 @@ -138,10 +154,23 @@ def _listen(self): class RemoteREPL(IPythonREPL): """The IPython execution-session interface backed by a separate worker.""" - def __init__(self, *, command, **kwargs): + def __init__( + self, + *, + command=None, + tcp=None, + transport: ExecutionTransport | None = None, + tools=("ipython",), + **kwargs, + ): super().__init__(**kwargs) - self.command = command - self._process = None + if sum(option is not None for option in (command, tcp, transport)) != 1: + raise ValueError("choose one execution transport") + validate_tools(tools) + self.tools = list(tools) + self.transport = transport or ( + TcpTransport(tcp) if tcp else StdioTransport(command) + ) self._peer = None self.session_dir = None @@ -166,20 +195,15 @@ def _broker(self, method, request): return _receive(stream) def start(self): - self._process = subprocess.Popen( - self.command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=None, - start_new_session=True, - ) - self._peer = _Peer(self._process.stdout, self._process.stdin, self._broker) + reader, writer = self.transport.connect() + self._peer = _Peer(reader, writer, self._broker) modules = { path.name: path.read_text() for path in self.session.dir.glob("*.py") } self.session_dir = self._peer.call( "start", { + "tools": self.tools, "cwd": self.cwd, "kernel_env": self.kernel_env, "depth": self.depth, @@ -194,22 +218,29 @@ def start(self): timeout=120, ) - def execute(self, code, timeout=None): + def execute_tool(self, name, args): result = self._peer.call( - "execute", + "tool", { - "code": code, - "timeout": timeout, + "name": name, + "args": args, "scope_id": self._scope_id, "history": (self.session.dir / "messages.jsonl").read_text(), }, ) self._recovery_notices.extend(result["notices"]) - # Programmatic tool statistics are observational, just as with local kernels. (self.session.dir / "programmatic_tool_calls.jsonl").write_text( result["tool_log"] ) - return result["output"] + return ToolOutcome( + content=result["output"], + metric_events=[ + IpythonExecuted(**event) for event in result["metric_events"] + ], + ) + + def execute(self, code, timeout=None): + return self.execute_tool("ipython", {"code": code, "timeout": timeout}).content def interrupt(self): if self._peer is not None and not self._peer.closed.is_set(): @@ -220,26 +251,19 @@ def finish_interrupt(self): self._peer.call("finish_interrupt", timeout=5) def shutdown(self): - process, self._process = self._process, None - if process is not None: - try: - if self._peer is not None and not self._peer.closed.is_set(): - self._peer.call("close", timeout=30) - finally: - # EOF is also a worker shutdown request if the explicit close failed. - process.stdin.close() - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - process.kill() - process.wait() - process.stdout.close() - self._executor.shutdown(wait=False) + try: + if self._peer is not None and not self._peer.closed.is_set(): + self._peer.call("close", timeout=30) + finally: + self.transport.close() + self._executor.shutdown(wait=False) class _Worker: def __init__(self): self.repl = None + self.tools = [] + self.context = None self.peer = None self.directory = tempfile.TemporaryDirectory(prefix="rlm-execution-") self.session_dir = Path(self.directory.name) @@ -273,6 +297,8 @@ def handle(self, method, params): if self.repl is not None: raise ValueError("execution session already started") params = dict(params) + self.tools = params.pop("tools") + validate_tools(self.tools) modules = params.pop("modules") for name, source in modules.items(): if Path(name).name != name or not name.endswith(".py"): @@ -292,20 +318,36 @@ def handle(self, method, params): broker_endpoint=endpoint, **params, ) + self.context = ToolContext( + messages=[], + metrics=RLMMetrics(), + total_usage=TokenUsage(), + last_prompt_tokens=0, + exec_timeout=self.repl.exec_timeout or 300, + allow_git=self.repl.allow_git, + repl=self.repl, + cwd=self.repl.cwd, + env=build_kernel_env(self.repl.kernel_env), + interrupt_event=self.repl._interrupt_requested, + ) # All Jupyter operations use the same dedicated thread. self.repl._executor.submit(self.repl.start).result() return str(self.session_dir) if self.repl is None: raise ValueError("execution session has not started") - if method == "execute": + if method == "tool": def execute(): (self.session_dir / "messages.jsonl").write_text(params["history"]) self.repl.set_broker_scope(params["scope_id"]) - output = self.repl.execute(params["code"], params["timeout"]) + tool = get_builtin_tool(params["name"], self.tools) + if tool is None: + raise ValueError("tool is not enabled for this execution session") + outcome = tool.execute(params["args"], self.context) log = self.session_dir / "programmatic_tool_calls.jsonl" return { - "output": output, + "output": outcome.content, + "metric_events": [asdict(event) for event in outcome.metric_events], "notices": self.repl.take_recovery_notices(), "tool_log": log.read_text() if log.exists() else "", } @@ -331,16 +373,58 @@ def close(self): self.directory.cleanup() -def main(): +def _serve(reader, writer): worker = _Worker() - output = sys.stdout.buffer - sys.stdout = sys.stderr # dependency diagnostics must not corrupt the wire - worker.peer = _Peer(sys.stdin.buffer, output, worker.handle) + worker.peer = _Peer(reader, writer, worker.handle) try: worker.peer.closed.wait() finally: worker.close() +def _serve_tcp(connection, token, slots): + try: + with connection, connection.makefile("rwb", buffering=0) as stream: + connection.settimeout(10) + supplied = _receive(stream).get("token") + if not isinstance(supplied, str) or not hmac.compare_digest( + supplied, token + ): + _send(stream, {"authenticated": False}) + return + _send(stream, {"authenticated": True}) + connection.settimeout(None) + _serve(stream, stream) + except (OSError, EOFError, ValueError): + pass # unauthenticated or disconnected clients have no remaining session + finally: + slots.release() + + +def main(): + parser = argparse.ArgumentParser(description="Tool execution worker") + parser.add_argument("--listen", metavar="HOST") + parser.add_argument("--port", type=int, default=8765) + args = parser.parse_args() + output = sys.stdout.buffer + sys.stdout = sys.stderr # dependency diagnostics must not corrupt the wire + if args.listen is None: + _serve(sys.stdin.buffer, output) + return + token = os.environ.pop("RLM_EXECUTION_TOKEN", "") + if len(token) < 32: + parser.error("TCP workers require RLM_EXECUTION_TOKEN (at least 32 characters)") + slots = threading.BoundedSemaphore(32) + with socket.create_server((args.listen, args.port)) as listener: + while True: + connection, _ = listener.accept() + if not slots.acquire(blocking=False): + connection.close() + continue + threading.Thread( + target=_serve_tcp, args=(connection, token, slots), daemon=True + ).start() + + if __name__ == "__main__": main() diff --git a/src/rlm/execution_transport.py b/src/rlm/execution_transport.py new file mode 100644 index 00000000..3da49a2a --- /dev/null +++ b/src/rlm/execution_transport.py @@ -0,0 +1,80 @@ +"""Byte-stream transports for tool execution, independent of provisioning.""" + +from __future__ import annotations + +import socket +import subprocess +from typing import BinaryIO, Protocol + +from rlm.config import TcpExecutionConfig + + +class ExecutionTransport(Protocol): + def connect(self) -> tuple[BinaryIO, BinaryIO]: ... + + def close(self) -> None: ... + + +class StdioTransport: + def __init__(self, command): + self.command = command + self.process = None + + def connect(self): + self.process = subprocess.Popen( + self.command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=None, + start_new_session=True, + ) + return self.process.stdout, self.process.stdin + + def close(self): + process, self.process = self.process, None + if process is not None: + process.stdin.close() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + process.stdout.close() + + +class TcpTransport: + """One execution session per connection on a trusted private network.""" + + def __init__(self, endpoint: TcpExecutionConfig): + self.endpoint = endpoint + self.connection = None + self.stream = None + + def connect(self): + from rlm.execution import _receive, _send + + self.connection = socket.create_connection( + (self.endpoint.host, self.endpoint.port), timeout=10 + ) + self.stream = self.connection.makefile("rwb", buffering=0) + try: + _send(self.stream, {"token": self.endpoint.token}) + if _receive(self.stream) != {"authenticated": True}: + raise PermissionError("execution worker authentication failed") + self.connection.settimeout(None) + except BaseException: + self.close() + raise + return self.stream, self.stream + + def close(self): + if self.connection is not None: + try: + self.connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass # the worker may already have disconnected + self.connection.close() + self.connection = None + if self.stream is not None: + self.stream.close() + self.stream = None diff --git a/src/rlm/tools/base.py b/src/rlm/tools/base.py index 295d6ff7..ec5d7d5c 100644 --- a/src/rlm/tools/base.py +++ b/src/rlm/tools/base.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from typing import Any, Protocol +from threading import Event from rlm.types import BuiltinMetricEvent, RLMMetrics, TokenUsage @@ -29,6 +30,8 @@ class ToolContext: repl: Any | None = None state: dict[str, Any] = field(default_factory=dict) cwd: str = "" + env: dict[str, str] | None = None + interrupt_event: Event | None = None class BuiltinTool(Protocol): diff --git a/src/rlm/tools/bash.py b/src/rlm/tools/bash.py index 4ec67c7a..bda18cea 100644 --- a/src/rlm/tools/bash.py +++ b/src/rlm/tools/bash.py @@ -13,6 +13,7 @@ import time import contextlib import subprocess +from threading import Event from typing import Any from rlm.tools.base import ToolContext, ToolOutcome @@ -65,6 +66,8 @@ def run_bash( timeout: int, cwd: str | None = None, allow_git: bool | None = None, + env: dict[str, str] | None = None, + interrupt_event: Event | None = None, ) -> str: """Guarded ``bash -c`` execution shared by the bash tool and the bash skill.""" if not isinstance(command, str) or not command.strip(): @@ -82,6 +85,7 @@ def run_bash( stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd or None, + env=env, start_new_session=True, ) as proc, selectors.DefaultSelector() as selector, @@ -90,6 +94,9 @@ def run_bash( selector.register(proc.stderr, selectors.EVENT_READ, "stderr") try: while selector.get_map(): + if interrupt_event is not None and interrupt_event.is_set(): + reason = "[command interrupted; process group terminated]" + break remaining = deadline - time.monotonic() if remaining <= 0: reason = f"Error: command timed out after {timeout}s" @@ -144,6 +151,8 @@ def execute(self, args: dict[str, Any], context: ToolContext) -> ToolOutcome: context.exec_timeout, cwd=context.cwd, allow_git=context.allow_git, + env=context.env, + interrupt_event=context.interrupt_event, ) ) diff --git a/tests/test_acp.py b/tests/test_acp.py index 9e62d478..eb97a92e 100644 --- a/tests/test_acp.py +++ b/tests/test_acp.py @@ -855,6 +855,10 @@ async def test_acp_session_reuses_engine(monkeypatch, tmp_path): assert initialized.field_meta == { CONTRACT_METADATA_KEY: True, "steering": {"supported": True}, + "ai.prime.rlm/execution-v1": { + "transports": ["stdio", "tcp"], + "tools": ["ipython", "bash", "edit", "fetch"], + }, } created = await agent.new_session( @@ -1113,6 +1117,10 @@ async def test_acp_stdio_lifecycle(tmp_path): assert initialized.field_meta == { CONTRACT_METADATA_KEY: True, "steering": {"supported": True}, + "ai.prime.rlm/execution-v1": { + "transports": ["stdio", "tcp"], + "tools": ["ipython", "bash", "edit", "fetch"], + }, } assert created.field_meta is None assert closed.field_meta[SESSION_METADATA_KEY]["session_id"] == "wire-session" From c2149539258c8c53ad17141fbebdf30fdb07b76c Mon Sep 17 00:00:00 2001 From: hallerite Date: Thu, 29 Oct 2026 22:12:41 +0000 Subject: [PATCH 04/21] fix: refuse rlm.services in split execution Services started through the supervisor would run beside the agent loop instead of in the task workspace. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- src/rlm/execution.py | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6fe20370..c61f7660 100644 --- a/README.md +++ b/README.md @@ -582,8 +582,8 @@ use a private network or an encrypted tunnel. The worker removes its token from the environment before starting tools. The orchestrator owns listener teardown. Supervisor shell jobs -(`rlm.shell`) and filesystem watchers (`rlm.watch.path`) return explicit errors in -this mode. `await child.history()` uses the supervisor; the current agent's local +(`rlm.shell`), services (`rlm.services`) and filesystem watchers +(`rlm.watch.path`) return explicit errors in this mode. `await child.history()` uses the supervisor; the current agent's local history is a snapshot refreshed before each cell. Worker frames, including history snapshots and outputs, are limited to 16 MiB. Kernel recovery preserves the existing no-replay behavior. Worker disconnects fail the session; the orchestrator must stop diff --git a/src/rlm/execution.py b/src/rlm/execution.py index 6135b561..2222dfe2 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -179,14 +179,12 @@ def _broker(self, method, request): raise PermissionError("unknown execution callback") if request.get("capability") != self.broker_endpoint.capability: raise PermissionError("invalid execution capability") - # These operations currently launch jobs/watchers in the supervisor process. - # Refuse them until they have execution-session implementations. - if ( - request.get("op", "").startswith("shell.") - or request.get("op") == "watch.path" - ): + # These operations currently launch jobs/services/watchers in the supervisor + # process. Refuse them until they have execution-session implementations. + op = request.get("op", "") + if op.startswith(("shell.", "services.")) or op == "watch.path": raise ValueError( - "split execution does not support rlm.shell or watch.path; use bash/IPython in the task workspace" + "split execution does not support rlm.shell, rlm.services or watch.path; use bash/IPython in the task workspace" ) with socket.socket(socket.AF_UNIX) as connection: connection.connect(self.broker_endpoint.socket_path) From 762e0d4f4a01b7563091e0f08549f924614cd4fc Mon Sep 17 00:00:00 2001 From: hallerite Date: Thu, 29 Oct 2026 23:57:37 +0000 Subject: [PATCH 05/21] fix: stop kernel commands with the kernel and keep worker control off the loop Shell commands start sessions of their own so timeouts and output floods can kill their whole process group, which also took them out of the kernel's group: a kernel restart or shutdown left a running bash skill command alive. The REPL now kills the process groups of the kernel's descendants first. RemoteREPL sent interrupt, finish_interrupt and close as blocking RPCs from the supervisor's event loop, where a slow worker could stall every agent for up to 30 seconds. They now run in order on a control thread; a tool call waits for pending control calls so it cannot overtake an interrupt's release. Co-Authored-By: Claude Opus 5.5 --- src/rlm/execution.py | 48 +++++++++++++++++++++++++++++++--------- src/rlm/tools/ipython.py | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/src/rlm/execution.py b/src/rlm/execution.py index 2222dfe2..dd23c20f 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -13,6 +13,7 @@ import struct import argparse import hmac +import logging import os from dataclasses import asdict import sys @@ -29,6 +30,8 @@ from rlm.types import IpythonExecuted, RLMMetrics, TokenUsage from rlm.execution_transport import ExecutionTransport, StdioTransport, TcpTransport +logger = logging.getLogger(__name__) + WORKSPACE_TOOLS = ("ipython", "bash", "edit", "fetch") @@ -173,6 +176,24 @@ def __init__( ) self._peer = None self.session_dir = None + # Interrupts and teardown run off the caller's thread, which can be the + # supervisor's event loop, in order and without waiting on a slow worker. + self._control = concurrent.futures.ThreadPoolExecutor( + 1, thread_name_prefix="rlm-control" + ) + self._control_done: concurrent.futures.Future | None = None + + def _control_call(self, method, timeout): + peer = self._peer + + def call(): + if peer is not None and not peer.closed.is_set(): + try: + peer.call(method, timeout=timeout) + except Exception: + logger.warning("execution worker %s failed", method, exc_info=True) + + self._control_done = self._control.submit(call) def _broker(self, method, request): if method != "broker" or self.broker_endpoint is None: @@ -217,6 +238,9 @@ def start(self): ) def execute_tool(self, name, args): + if self._control_done is not None: + # A call must not overtake a pending interrupt or its release. + self._control_done.result() result = self._peer.call( "tool", { @@ -241,20 +265,24 @@ def execute(self, code, timeout=None): return self.execute_tool("ipython", {"code": code, "timeout": timeout}).content def interrupt(self): - if self._peer is not None and not self._peer.closed.is_set(): - self._peer.call("interrupt", timeout=5) + self._control_call("interrupt", 5) def finish_interrupt(self): - if self._peer is not None and not self._peer.closed.is_set(): - self._peer.call("finish_interrupt", timeout=5) + self._control_call("finish_interrupt", 5) def shutdown(self): - try: - if self._peer is not None and not self._peer.closed.is_set(): - self._peer.call("close", timeout=30) - finally: - self.transport.close() - self._executor.shutdown(wait=False) + def close(): + try: + if self._peer is not None and not self._peer.closed.is_set(): + self._peer.call("close", timeout=30) + except Exception: + logger.warning("execution worker close failed", exc_info=True) + finally: + self.transport.close() + self._executor.shutdown(wait=False) + + self._control.submit(close) + self._control.shutdown(wait=False) class _Worker: diff --git a/src/rlm/tools/ipython.py b/src/rlm/tools/ipython.py index 41249f9a..c1217589 100644 --- a/src/rlm/tools/ipython.py +++ b/src/rlm/tools/ipython.py @@ -3,12 +3,14 @@ from __future__ import annotations import asyncio +import contextlib import copy import functools import os from queue import Empty import re import shutil +import signal import subprocess import sys import tempfile @@ -257,6 +259,35 @@ def __init__(self, output: str = ""): MAX_CELL_OUTPUT_BYTES = 1024 * 1024 +def kill_descendant_groups(pid: int) -> None: + """SIGKILL the process groups of `pid`'s descendants, read from /proc; a no-op + where there is none. The caller's own group is spared.""" + try: + entries = [entry for entry in os.listdir("/proc") if entry.isdigit()] + except FileNotFoundError: + return + parents: dict[int, list[int]] = {} + groups: dict[int, int] = {} + for entry in entries: + try: + with open(f"/proc/{entry}/stat") as stat: + # The command name can hold spaces and parentheses; fields follow it. + fields = stat.read().rsplit(")", 1)[1].split() + except (OSError, IndexError): + continue + parents.setdefault(int(fields[1]), []).append(int(entry)) + groups[int(entry)] = int(fields[2]) + targets, pending = set(), [pid] + while pending: + for child in parents.get(pending.pop(), []): + targets.add(groups[child]) + pending.append(child) + targets.discard(os.getpgrp()) + for group in targets: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(group, signal.SIGKILL) + + class IPythonREPL: """Persistent IPython kernel communicating via Jupyter protocol.""" @@ -523,10 +554,18 @@ def _wait_for_idle(self, msg_id: str, timeout: float) -> bool: ): return True + def _kill_kernel_commands(self): + """Kill the process groups the kernel's commands run in. Shell commands start + sessions of their own, so killing the kernel's process group misses them.""" + pid = getattr(self._km.provisioner, "pid", None) if self._km else None + if pid is not None: + kill_descendant_groups(pid) + def restart_kernel(self): """Restart the kernel and restore the initial REPL state.""" if self._kc: self._kc.stop_channels() + self._kill_kernel_commands() self._km.restart_kernel(now=True) self._kc = self._km.client() self._kc.start_channels() @@ -664,6 +703,7 @@ def shutdown(self): self._kc.stop_channels() self._kc = None if self._km and self._km.has_kernel: + self._kill_kernel_commands() self._km.shutdown_kernel(now=True) if self._km: self._km = None From 665c1135ba36d28c3ac65d52f2d60846dce71857 Mon Sep 17 00:00:00 2001 From: hallerite Date: Fri, 30 Oct 2026 00:15:44 +0000 Subject: [PATCH 06/21] fix: find a kernel's commands by an environment token Walking /proc from the kernel's PID missed the commands of a kernel that had already died, whose children are reparented, and a provisioner PID of 0 would have walked the whole process tree. Each kernel now carries a random RLM_KERNEL_TOKEN in its environment, which every process it starts inherits; restart and shutdown kill the process groups of processes that carry it, including after a crash. Interrupts and closes after shutdown are no-ops instead of failing on the closed control thread. Co-Authored-By: Claude Opus 5.5 --- src/rlm/execution.py | 6 ++++++ src/rlm/tools/ipython.py | 46 ++++++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/rlm/execution.py b/src/rlm/execution.py index dd23c20f..dbf2d413 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -182,8 +182,11 @@ def __init__( 1, thread_name_prefix="rlm-control" ) self._control_done: concurrent.futures.Future | None = None + self._control_closed = False def _control_call(self, method, timeout): + if self._control_closed: + return peer = self._peer def call(): @@ -281,6 +284,9 @@ def close(): self.transport.close() self._executor.shutdown(wait=False) + if self._control_closed: + return + self._control_closed = True self._control.submit(close) self._control.shutdown(wait=False) diff --git a/src/rlm/tools/ipython.py b/src/rlm/tools/ipython.py index c1217589..34fa78b0 100644 --- a/src/rlm/tools/ipython.py +++ b/src/rlm/tools/ipython.py @@ -16,6 +16,7 @@ import tempfile import threading import time +import uuid from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -259,36 +260,37 @@ def __init__(self, output: str = ""): MAX_CELL_OUTPUT_BYTES = 1024 * 1024 -def kill_descendant_groups(pid: int) -> None: - """SIGKILL the process groups of `pid`'s descendants, read from /proc; a no-op - where there is none. The caller's own group is spared.""" +def kill_marked_groups(marker: str) -> None: + """SIGKILL the process group of every process whose environment holds `marker`, + read from /proc; a no-op where there is none. Reparented and detached processes + keep their environment, so this also reaches the commands of a dead kernel. The + caller's own group is spared.""" try: entries = [entry for entry in os.listdir("/proc") if entry.isdigit()] except FileNotFoundError: return - parents: dict[int, list[int]] = {} - groups: dict[int, int] = {} + needle = marker.encode() + groups = set() for entry in entries: try: - with open(f"/proc/{entry}/stat") as stat: - # The command name can hold spaces and parentheses; fields follow it. - fields = stat.read().rsplit(")", 1)[1].split() - except (OSError, IndexError): + with open(f"/proc/{entry}/environ", "rb") as environ: + if needle not in environ.read().split(b"\0"): + continue + groups.add(os.getpgid(int(entry))) + except OSError: continue - parents.setdefault(int(fields[1]), []).append(int(entry)) - groups[int(entry)] = int(fields[2]) - targets, pending = set(), [pid] - while pending: - for child in parents.get(pending.pop(), []): - targets.add(groups[child]) - pending.append(child) - targets.discard(os.getpgrp()) - for group in targets: + groups -= {0, 1, os.getpgrp()} + for group in groups: with contextlib.suppress(ProcessLookupError, PermissionError): os.killpg(group, signal.SIGKILL) +KERNEL_TOKEN_ENV = "RLM_KERNEL_TOKEN" + + class IPythonREPL: + _kernel_token: str | None = None + """Persistent IPython kernel communicating via Jupyter protocol.""" def __init__( @@ -358,6 +360,9 @@ def start(self): self.kernel_env, private_dir=self._ipc_dir, ) + # Marks every process the kernel starts, wherever it is reparented. + self._kernel_token = uuid.uuid4().hex + kernel_env[KERNEL_TOKEN_ENV] = self._kernel_token launcher = shutil.which(sys.argv[0]) or os.path.abspath(sys.argv[0]) launcher_dir = os.path.dirname(os.path.abspath(launcher)) path_entries = kernel_env.get("PATH", "").split(os.pathsep) @@ -557,9 +562,8 @@ def _wait_for_idle(self, msg_id: str, timeout: float) -> bool: def _kill_kernel_commands(self): """Kill the process groups the kernel's commands run in. Shell commands start sessions of their own, so killing the kernel's process group misses them.""" - pid = getattr(self._km.provisioner, "pid", None) if self._km else None - if pid is not None: - kill_descendant_groups(pid) + if self._kernel_token is not None: + kill_marked_groups(f"{KERNEL_TOKEN_ENV}={self._kernel_token}") def restart_kernel(self): """Restart the kernel and restore the initial REPL state.""" From ff8767d3bdc311ca55a98fea2c6d4438c006741a Mon Sep 17 00:00:00 2001 From: hallerite Date: Fri, 30 Oct 2026 01:11:42 +0000 Subject: [PATCH 07/21] feat: announce the TCP worker's bound port With --port 0 the kernel picks a free port, and the worker prints it once it accepts connections, so an orchestrator needs no port selection of its own. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 ++ src/rlm/execution.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index c61f7660..39da2167 100644 --- a/README.md +++ b/README.md @@ -575,6 +575,8 @@ and `close()` independently of tool dispatch. For a trusted private TCP network, launch `python -m rlm.execution --listen 0.0.0.0 --port 8765` with a random `RLM_EXECUTION_TOKEN` of at least 32 characters. +With `--port 0` the kernel picks a free port, which the worker prints on stderr as +`listening on HOST:PORT` once it accepts connections. Set `execution_tcp` to `{"host": "worker-address", "port": 8765, "token": "..."}` instead of `execution_command`. Each connection owns a separate execution session. The token authenticates connections; this transport does not encrypt traffic, so diff --git a/src/rlm/execution.py b/src/rlm/execution.py index dbf2d413..1022d4cc 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -448,6 +448,8 @@ def main(): parser.error("TCP workers require RLM_EXECUTION_TOKEN (at least 32 characters)") slots = threading.BoundedSemaphore(32) with socket.create_server((args.listen, args.port)) as listener: + # With --port 0 the kernel picks the port; the orchestrator reads it here. + print(f"listening on {args.listen}:{listener.getsockname()[1]}", flush=True) while True: connection, _ = listener.accept() if not slots.acquire(blocking=False): From 7cbe516e142c47ed0fde34834e7fbea47edbd94b Mon Sep 17 00:00:00 2001 From: hallerite Date: Sat, 31 Oct 2026 01:22:48 +0000 Subject: [PATCH 08/21] fix: stop native shell commands with their turn and session The bash tool starts each command in a session of its own, which the kernel's cleanup didn't reach: a cancelled local turn waited for the command's timeout, and a background job it left behind outlived the session. Native shell commands now carry the kernel's token, so restart and shutdown kill them like the kernel's own commands, and a local turn's cancellation interrupts them. Tools run one at a time on the REPL's thread, so a restart never meets a running shell command. The compaction notice named the harness's messages.jsonl, which in split execution is on another machine; it now names the worker's copy, as the system prompt does. Co-Authored-By: Claude Opus 5.5 --- src/rlm/engine.py | 15 ++++++++--- src/rlm/execution.py | 2 +- src/rlm/tools/ipython.py | 12 ++++++--- tests/test_compaction.py | 15 +++++++++++ tests/test_kernel_recovery.py | 48 +++++++++++++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 7 deletions(-) diff --git a/src/rlm/engine.py b/src/rlm/engine.py index d7211ee6..7b3db291 100644 --- a/src/rlm/engine.py +++ b/src/rlm/engine.py @@ -1397,7 +1397,7 @@ async def _compact_branch( + "\n\n" + summary_text + "\n\nFull conversation history is available in " - + str(self.session.dir / "messages.jsonl") + + str(Path(self._kernel_session_dir()) / "messages.jsonl") + ". Use `from rlm import history; h = await history()` to inspect " "`h.windows[w].messages[i]`, `h.messages[i]`, or `h.user_messages()`. " "Search or read relevant records with Python when the summary lacks context. The log includes failed attempts: prompt_rollback.prompt_id " @@ -1495,13 +1495,17 @@ def execution_snapshot(self) -> dict: } return snapshot + def _kernel_session_dir(self) -> str: + """The session directory as the kernel sees it: in split execution, the + worker's, since the harness's own is on another machine.""" + return getattr(self._repl, "session_dir", None) or str(self.session.dir) + def _load_system_prompt(self, active_tools: list[BuiltinTool]) -> str: return build_system_prompt( self.cwd, discover_skills(self.session.dir), depth=self.depth, - session_dir=getattr(self._repl, "session_dir", None) - or str(self.session.dir), + session_dir=self._kernel_session_dir(), allow_recursion=self.depth < self.max_depth, delegation_prompt=self.runtime_config.policy.delegation_prompt, allow_git=self.allow_git, @@ -1522,6 +1526,7 @@ def _load_system_prompt(self, active_tools: list[BuiltinTool]) -> str: ) def _tool_context(self, messages: list[dict]) -> ToolContext: + repl = self._repl if isinstance(self._repl, IPythonREPL) else None return ToolContext( messages=messages, metrics=self._metrics, @@ -1532,6 +1537,10 @@ def _tool_context(self, messages: list[dict]) -> ToolContext: repl=self._repl, state=self._tool_state, cwd=self.cwd, + # A shell command stops with a cancelled turn, and with the session's + # kernel like the kernel's own commands. + env=repl.command_env(os.environ) if repl else None, + interrupt_event=repl._interrupt_requested if repl else None, ) diff --git a/src/rlm/execution.py b/src/rlm/execution.py index 1022d4cc..b394300b 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -359,7 +359,7 @@ def handle(self, method, params): allow_git=self.repl.allow_git, repl=self.repl, cwd=self.repl.cwd, - env=build_kernel_env(self.repl.kernel_env), + env=self.repl.command_env(build_kernel_env(self.repl.kernel_env)), interrupt_event=self.repl._interrupt_requested, ) # All Jupyter operations use the same dedicated thread. diff --git a/src/rlm/tools/ipython.py b/src/rlm/tools/ipython.py index 34fa78b0..6ad45559 100644 --- a/src/rlm/tools/ipython.py +++ b/src/rlm/tools/ipython.py @@ -317,6 +317,9 @@ def __init__( self._ipc_dir = None self._lock = threading.Lock() self._interrupt_requested = threading.Event() + # Marks every process the kernel and the shell tools start, wherever it is + # reparented: restart and shutdown kill what carries it. + self._kernel_token = uuid.uuid4().hex self._scope_id: str | None = None self._recovery_attempts = 0 self._recovery_failed = False @@ -360,8 +363,6 @@ def start(self): self.kernel_env, private_dir=self._ipc_dir, ) - # Marks every process the kernel starts, wherever it is reparented. - self._kernel_token = uuid.uuid4().hex kernel_env[KERNEL_TOKEN_ENV] = self._kernel_token launcher = shutil.which(sys.argv[0]) or os.path.abspath(sys.argv[0]) launcher_dir = os.path.dirname(os.path.abspath(launcher)) @@ -565,6 +566,11 @@ def _kill_kernel_commands(self): if self._kernel_token is not None: kill_marked_groups(f"{KERNEL_TOKEN_ENV}={self._kernel_token}") + def command_env(self, env: Mapping[str, str]) -> dict[str, str]: + """`env` marked like the kernel's own commands, for a shell command a tool runs + beside the kernel, so that restart and shutdown stop it too.""" + return {**env, KERNEL_TOKEN_ENV: self._kernel_token} + def restart_kernel(self): """Restart the kernel and restore the initial REPL state.""" if self._kc: @@ -706,8 +712,8 @@ def shutdown(self): if self._kc: self._kc.stop_channels() self._kc = None + self._kill_kernel_commands() if self._km and self._km.has_kernel: - self._kill_kernel_commands() self._km.shutdown_kernel(now=True) if self._km: self._km = None diff --git a/tests/test_compaction.py b/tests/test_compaction.py index 1b4c8f1c..c0bbd70c 100644 --- a/tests/test_compaction.py +++ b/tests/test_compaction.py @@ -676,3 +676,18 @@ async def test_hollowed_input_falls_back_to_last_good_snapshot(session): assert len(client.calls) == 3 finally: engine.close() + + +def test_compaction_names_the_history_where_the_kernel_runs(tmp_path): + """In split execution the kernel runs beside the worker, whose session directory + holds the history snapshot; the harness's own directory is on another machine.""" + from types import SimpleNamespace + + harness = SimpleNamespace(dir=tmp_path / "harness") + local = SimpleNamespace(_repl=SimpleNamespace(), session=harness) + split = SimpleNamespace( + _repl=SimpleNamespace(session_dir="/worker/s"), session=harness + ) + + assert RLMEngine._kernel_session_dir(local) == str(harness.dir) + assert RLMEngine._kernel_session_dir(split) == "/worker/s" diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index c7a7a3fc..1e49d364 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -391,3 +391,51 @@ async def test_output_flood_is_bounded_and_kernel_can_continue(session): assert "STILL_ALIVE" in continued finally: await repl.run_in_thread(repl.shutdown) + + +async def test_shell_tool_commands_stop_with_the_turn_and_the_session( + session, tmp_path +): + """A native bash command stops when its turn is cancelled, and a background job it + leaves behind stops when the session's kernel shuts down.""" + import subprocess + from types import SimpleNamespace + + from rlm.tools.bash import BashTool + + repl = IPythonREPL(session=session, cwd=str(tmp_path)) + engine = SimpleNamespace( + _repl=repl, + _metrics=None, + _total_usage=None, + _last_prompt_tokens=0, + exec_timeout=60, + allow_git=False, + _tool_state={}, + cwd=str(tmp_path), + ) + context = RLMEngine._tool_context(engine, []) + alive = tmp_path / "alive" + await repl.run_in_thread(repl.start) + try: + running = asyncio.create_task( + asyncio.to_thread(BashTool().execute, {"command": "sleep 60"}, context) + ) + await asyncio.sleep(0.5) + repl.interrupt() + outcome = await asyncio.wait_for(running, 5) + repl.finish_interrupt() + assert "interrupted" in outcome.content + + loop = f"(while :; do touch {alive}; sleep 0.1; done) >/dev/null 2>&1 &" + BashTool().execute({"command": loop}, context) + await asyncio.sleep(0.5) + assert alive.exists() + finally: + await repl.run_in_thread(repl.shutdown) + try: + alive.unlink() + await asyncio.sleep(0.5) + assert not alive.exists() + finally: + subprocess.run(["pkill", "-f", f"touch {alive}"], check=False) From d98bc273ad3a8004f87fd205542c1f5dd8d0a6d2 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sat, 31 Oct 2026 01:37:48 +0000 Subject: [PATCH 09/21] fix: sync the session ledger to the execution worker incrementally Every tool call sent the whole append-only messages.jsonl to the worker and got back the whole programmatic tool log, so a long split session failed once the ledger outgrew one 16 MiB frame. The harness now sends only the ledger bytes the worker hasn't seen, in chunks that fit a frame, and the worker writes each at its offset; the worker returns only the tool calls logged since the last call, which the harness appends. Co-Authored-By: Claude Opus 5.5 --- src/rlm/execution.py | 61 +++++++++++++++++++++++++++++++---- tests/test_kernel_recovery.py | 38 ++++++++++++++++++++++ 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/src/rlm/execution.py b/src/rlm/execution.py index b394300b..2921a146 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -12,6 +12,7 @@ import socket import struct import argparse +import base64 import hmac import logging import os @@ -42,6 +43,9 @@ def validate_tools(names): MAX_FRAME = 16 * 1024 * 1024 +HISTORY_CHUNK = 4 * 1024 * 1024 +"""Raw bytes of the session ledger per frame; base64 keeps a chunk well under +MAX_FRAME, and a long session's ledger syncs as several.""" def _read(stream, size): @@ -176,6 +180,7 @@ def __init__( ) self._peer = None self.session_dir = None + self._history_sent = 0 # Interrupts and teardown run off the caller's thread, which can be the # supervisor's event loop, in order and without waiting on a slow worker. self._control = concurrent.futures.ThreadPoolExecutor( @@ -219,6 +224,7 @@ def _broker(self, method, request): def start(self): reader, writer = self.transport.connect() self._peer = _Peer(reader, writer, self._broker) + self._history_sent = 0 modules = { path.name: path.read_text() for path in self.session.dir.glob("*.py") } @@ -244,19 +250,31 @@ def execute_tool(self, name, args): if self._control_done is not None: # A call must not overtake a pending interrupt or its release. self._control_done.result() + # The ledger only grows: the worker gets what it hasn't seen, in chunks + # that fit a frame however long the session runs. + with open(self.session.dir / "messages.jsonl", "rb") as ledger: + ledger.seek(self._history_sent) + new = ledger.read() + *head, last = [ + new[start : start + HISTORY_CHUNK] + for start in range(0, len(new), HISTORY_CHUNK) + ] or [b""] + for chunk in head: + self._peer.call("history", self._history_chunk(chunk)) + self._history_sent += len(chunk) result = self._peer.call( "tool", { "name": name, "args": args, "scope_id": self._scope_id, - "history": (self.session.dir / "messages.jsonl").read_text(), + "history": self._history_chunk(last), }, ) + self._history_sent += len(last) self._recovery_notices.extend(result["notices"]) - (self.session.dir / "programmatic_tool_calls.jsonl").write_text( - result["tool_log"] - ) + with open(self.session.dir / "programmatic_tool_calls.jsonl", "ab") as log: + log.write(base64.b64decode(result["tool_log"])) return ToolOutcome( content=result["output"], metric_events=[ @@ -264,6 +282,9 @@ def execute_tool(self, name, args): ], ) + def _history_chunk(self, data: bytes) -> dict: + return {"offset": self._history_sent, "data": base64.b64encode(data).decode()} + def execute(self, code, timeout=None): return self.execute_tool("ipython", {"code": code, "timeout": timeout}).content @@ -301,6 +322,7 @@ def __init__(self): self.session_dir = Path(self.directory.name) self.listener = None self.closed = threading.Event() + self._tool_log_sent = 0 def _accept(self): while not self.closed.is_set(): @@ -367,21 +389,23 @@ def handle(self, method, params): return str(self.session_dir) if self.repl is None: raise ValueError("execution session has not started") + if method == "history": + self._write_history(params) + return {} if method == "tool": def execute(): - (self.session_dir / "messages.jsonl").write_text(params["history"]) + self._write_history(params["history"]) self.repl.set_broker_scope(params["scope_id"]) tool = get_builtin_tool(params["name"], self.tools) if tool is None: raise ValueError("tool is not enabled for this execution session") outcome = tool.execute(params["args"], self.context) - log = self.session_dir / "programmatic_tool_calls.jsonl" return { "output": outcome.content, "metric_events": [asdict(event) for event in outcome.metric_events], "notices": self.repl.take_recovery_notices(), - "tool_log": log.read_text() if log.exists() else "", + "tool_log": base64.b64encode(self._new_tool_log()).decode(), } return self.repl._executor.submit(execute).result() @@ -395,6 +419,29 @@ def execute(): else: raise ValueError(f"unknown execution method: {method}") + def _write_history(self, chunk): + """Write a chunk of the harness's ledger at its offset. Rewriting the same + bytes is harmless, so a retried chunk can't duplicate history.""" + path = self.session_dir / "messages.jsonl" + path.touch() + with path.open("r+b") as ledger: + if chunk["offset"] > ledger.seek(0, os.SEEK_END): + raise ValueError("execution history is out of sync") + ledger.seek(chunk["offset"]) + ledger.write(base64.b64decode(chunk["data"])) + ledger.truncate() + + def _new_tool_log(self) -> bytes: + """The programmatic tool calls logged since the last tool call returned.""" + path = self.session_dir / "programmatic_tool_calls.jsonl" + if not path.exists(): + return b"" + with path.open("rb") as log: + log.seek(self._tool_log_sent) + new = log.read() + self._tool_log_sent += len(new) + return new + def close(self): self.closed.set() if self.listener is not None: diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 1e49d364..352de2fb 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -439,3 +439,41 @@ async def test_shell_tool_commands_stop_with_the_turn_and_the_session( assert not alive.exists() finally: subprocess.run(["pkill", "-f", f"touch {alive}"], check=False) + + +async def test_remote_history_syncs_past_the_frame_limit(session, tmp_path): + """The worker gets only the ledger bytes it hasn't seen, in chunks that fit a frame, + so a ledger larger than one frame still reaches it whole.""" + import sys + + from rlm.execution import MAX_FRAME, RemoteREPL + + ledger = session.dir / "messages.jsonl" + repl = RemoteREPL( + command=(sys.executable, "-m", "rlm.execution"), + cwd=str(tmp_path), + session=session, + ) + try: + await repl.run_in_thread(repl.start) + count = ( + "import hashlib, pathlib\n" + "data = (pathlib.Path(r'{}') / 'messages.jsonl').read_bytes()\n" + "print(len(data), hashlib.sha256(data).hexdigest())" + ).format(repl.session_dir) + line = '{"type": "note", "text": "' + "x" * 1000 + '"}\n' + with open(ledger, "a") as out: + out.write(line * (MAX_FRAME // len(line) + 1000)) + first = await repl.run_in_thread(repl.execute, count, 60) + with open(ledger, "a") as out: + out.write('{"type": "note", "text": "after"}\n') + second = await repl.run_in_thread(repl.execute, count, 60) + finally: + await asyncio.to_thread(repl.shutdown) + + import hashlib + + assert ledger.stat().st_size > MAX_FRAME + data = ledger.read_bytes() + assert second.split() == [str(len(data)), hashlib.sha256(data).hexdigest()] + assert int(first.split()[0]) < len(data) From 283d1369ce46339a08fb6ac57008bec8857227a5 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sat, 31 Oct 2026 01:48:58 +0000 Subject: [PATCH 10/21] test: drive the split-execution checks through the engine The shell cleanup, history sync and compaction notice checks now run as end-to-end engine sessions with a scripted model instead of calling helpers on stand-in objects: a cancelled turn stops its bash command and closing the engine stops a job it left behind; a prompt larger than one frame reaches the split worker's history whole; and the compaction test runs local and split, checking the notice names the history the kernel reads. Co-Authored-By: Claude Opus 5.5 --- tests/test_compaction.py | 40 +++++------ tests/test_kernel_recovery.py | 122 +++++++++++++++------------------- 2 files changed, 75 insertions(+), 87 deletions(-) diff --git a/tests/test_compaction.py b/tests/test_compaction.py index c0bbd70c..67fb7f66 100644 --- a/tests/test_compaction.py +++ b/tests/test_compaction.py @@ -1,7 +1,10 @@ from __future__ import annotations import json +import re +import sys from copy import deepcopy +from pathlib import Path from types import SimpleNamespace from typing import Any @@ -256,7 +259,18 @@ async def test_compaction_retries_reasoning_without_final_content(session, conte engine.close() -async def test_tool_result_overflow_compacts_and_retries(session): +@pytest.mark.parametrize("split", [False, True]) +async def test_tool_result_overflow_compacts_and_retries(session, tmp_path, split): + """Under split execution the kernel reads the worker's synced copy of the history, + and the compaction notice names that copy.""" + config = _config() + if split: + config = config.model_copy( + update={ + "execution_command": (sys.executable, "-m", "rlm.execution"), + "builtin_tools": ("ipython",), + } + ) client = _ScriptedClient( [ _response( @@ -294,13 +308,14 @@ async def test_tool_result_overflow_compacts_and_retries(session): engine = RLMEngine( client=client, # type: ignore[arg-type] session=session, - runtime_config=_config(), + runtime_config=config, + cwd=str(tmp_path), ) try: result = await engine.run("produce a large tool result") finally: - engine.close() + await engine.aclose() assert result.answer == "done" assert engine._metrics.num_compactions == 1 @@ -310,7 +325,9 @@ async def test_tool_result_overflow_compacts_and_retries(session): assert retry_messages[1]["content"].startswith( '\n' + SUMMARY_FRAMING ) - assert str(session.dir / "messages.jsonl") in retry_messages[1]["content"] + named = re.search(r"available in (\S+)\. ", retry_messages[1]["content"]) + assert named is not None + assert (Path(named.group(1)) != session.dir / "messages.jsonl") is split records = [ json.loads(line) for line in (session.dir / "messages.jsonl").read_text().splitlines() @@ -676,18 +693,3 @@ async def test_hollowed_input_falls_back_to_last_good_snapshot(session): assert len(client.calls) == 3 finally: engine.close() - - -def test_compaction_names_the_history_where_the_kernel_runs(tmp_path): - """In split execution the kernel runs beside the worker, whose session directory - holds the history snapshot; the harness's own directory is on another machine.""" - from types import SimpleNamespace - - harness = SimpleNamespace(dir=tmp_path / "harness") - local = SimpleNamespace(_repl=SimpleNamespace(), session=harness) - split = SimpleNamespace( - _repl=SimpleNamespace(session_dir="/worker/s"), session=harness - ) - - assert RLMEngine._kernel_session_dir(local) == str(harness.dir) - assert RLMEngine._kernel_session_dir(split) == "/worker/s" diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 352de2fb..48101edb 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -393,87 +393,73 @@ async def test_output_flood_is_bounded_and_kernel_can_continue(session): await repl.run_in_thread(repl.shutdown) -async def test_shell_tool_commands_stop_with_the_turn_and_the_session( +async def test_bash_commands_stop_with_a_cancelled_turn_and_the_session( session, tmp_path ): - """A native bash command stops when its turn is cancelled, and a background job it - leaves behind stops when the session's kernel shuts down.""" - import subprocess - from types import SimpleNamespace - - from rlm.tools.bash import BashTool - - repl = IPythonREPL(session=session, cwd=str(tmp_path)) - engine = SimpleNamespace( - _repl=repl, - _metrics=None, - _total_usage=None, - _last_prompt_tokens=0, - exec_timeout=60, - allow_git=False, - _tool_state={}, + """A cancelled turn stops its running bash command at once, and closing the engine + stops a background job an earlier command left behind.""" + alive = tmp_path / "alive" + loop = f"(while :; do touch {alive}; sleep 0.1; done) >/dev/null 2>&1 &" + client = DummyClient( + [ + DummyMessage(tool_calls=[DummyToolCall("bash", {"command": loop})]), + DummyMessage(tool_calls=[DummyToolCall("bash", {"command": "sleep 60"})]), + ] + ) + engine = RLMEngine( + client=client, + session=session, + runtime_config=_config().model_copy( + update={"builtin_tools": ("ipython", "bash")} + ), cwd=str(tmp_path), ) - context = RLMEngine._tool_context(engine, []) - alive = tmp_path / "alive" - await repl.run_in_thread(repl.start) try: - running = asyncio.create_task( - asyncio.to_thread(BashTool().execute, {"command": "sleep 60"}, context) - ) - await asyncio.sleep(0.5) - repl.interrupt() - outcome = await asyncio.wait_for(running, 5) - repl.finish_interrupt() - assert "interrupted" in outcome.content - - loop = f"(while :; do touch {alive}; sleep 0.1; done) >/dev/null 2>&1 &" - BashTool().execute({"command": loop}, context) + turn = asyncio.create_task(engine.prompt("run the commands")) + while len(client.calls) < 2: + await asyncio.sleep(0.1) await asyncio.sleep(0.5) + turn.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(turn, 5) assert alive.exists() finally: - await repl.run_in_thread(repl.shutdown) - try: - alive.unlink() - await asyncio.sleep(0.5) - assert not alive.exists() - finally: - subprocess.run(["pkill", "-f", f"touch {alive}"], check=False) + await engine.aclose() + alive.unlink() + await asyncio.sleep(0.5) + assert not alive.exists() -async def test_remote_history_syncs_past_the_frame_limit(session, tmp_path): - """The worker gets only the ledger bytes it hasn't seen, in chunks that fit a frame, - so a ledger larger than one frame still reaches it whole.""" +async def test_split_history_reaches_the_worker_past_the_frame_limit(session, tmp_path): + """The worker gets the ledger in frame-sized chunks: a prompt larger than one frame + still reaches the kernel's history whole.""" import sys - from rlm.execution import MAX_FRAME, RemoteREPL + from rlm.execution import MAX_FRAME - ledger = session.dir / "messages.jsonl" - repl = RemoteREPL( - command=(sys.executable, "-m", "rlm.execution"), - cwd=str(tmp_path), - session=session, + prompt = "x" * (MAX_FRAME + 1024) + config = _config().model_copy( + update={ + "execution_command": (sys.executable, "-m", "rlm.execution"), + "builtin_tools": ("ipython",), + } + ) + client = DummyClient( + [ + _tool( + "from rlm import history\n" + "h = await history()\n" + f"assert len(h.user_messages()[0]['content']) == {len(prompt)}\n" + "print('SYNCED')" + ), + DummyMessage(content="done"), + ] + ) + engine = RLMEngine( + client=client, session=session, runtime_config=config, cwd=str(tmp_path) ) try: - await repl.run_in_thread(repl.start) - count = ( - "import hashlib, pathlib\n" - "data = (pathlib.Path(r'{}') / 'messages.jsonl').read_bytes()\n" - "print(len(data), hashlib.sha256(data).hexdigest())" - ).format(repl.session_dir) - line = '{"type": "note", "text": "' + "x" * 1000 + '"}\n' - with open(ledger, "a") as out: - out.write(line * (MAX_FRAME // len(line) + 1000)) - first = await repl.run_in_thread(repl.execute, count, 60) - with open(ledger, "a") as out: - out.write('{"type": "note", "text": "after"}\n') - second = await repl.run_in_thread(repl.execute, count, 60) + await engine.prompt(prompt) finally: - await asyncio.to_thread(repl.shutdown) - - import hashlib - - assert ledger.stat().st_size > MAX_FRAME - data = ledger.read_bytes() - assert second.split() == [str(len(data)), hashlib.sha256(data).hexdigest()] - assert int(first.split()[0]) < len(data) + await engine.aclose() + assert "SYNCED" in client.calls[1]["messages"][-1]["content"] From 365b8f69d4732b20ac8a5eb8478a46a03f18aea3 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sat, 31 Oct 2026 02:30:50 +0000 Subject: [PATCH 11/21] feat: run the supervisor's shell jobs, services and path watches in the workspace Under split execution the supervisor stays with the agent loop, but rlm.shell, rlm.services and rlm.watch.path started processes and watched files on the supervisor's machine, so the worker refused them and the system prompt was swapped for a reduced one. The execution worker now hosts them in the workspace with the same job, service and watch code (WorkspaceHost), the supervisor reaches it through the root agent's worker (RemoteHost) and keeps its records and a copy of each job's output, and split execution uses the same runtime prompt as local execution. Co-Authored-By: Claude Opus 5.5 --- src/rlm/engine.py | 9 +- src/rlm/execution.py | 20 +- src/rlm/prompt.py | 13 +- src/rlm/shell_jobs.py | 6 +- src/rlm/subscriptions.py | 68 ++++-- src/rlm/supervisor.py | 83 ++++++-- src/rlm/workspace_host.py | 384 ++++++++++++++++++++++++++++++++++ tests/test_kernel_recovery.py | 36 +++- 8 files changed, 553 insertions(+), 66 deletions(-) create mode 100644 src/rlm/workspace_host.py diff --git a/src/rlm/engine.py b/src/rlm/engine.py index 7b3db291..e8d9a827 100644 --- a/src/rlm/engine.py +++ b/src/rlm/engine.py @@ -616,6 +616,14 @@ async def _start(self, prompt: str) -> None: cancelled = True if cancelled: raise asyncio.CancelledError + if ( + self.runtime_config.split_execution + and self.depth == 0 + and self._supervisor is not None + ): + # The root's worker lives for the session, so the session's jobs and + # services run through it, wherever an agent started them. + self._supervisor.attach_host(self._repl.host_call) system_prompt = self._load_system_prompt(self._active_tools) @@ -1519,7 +1527,6 @@ def _load_system_prompt(self, active_tools: list[BuiltinTool]) -> str: if self.system_prompt_path else None, extra_instructions=self.append_to_system_prompt, - split_execution=self.runtime_config.split_execution, agent_info=self._supervisor.agent_context(self._invocation_id) if self._supervisor else None, diff --git a/src/rlm/execution.py b/src/rlm/execution.py index 2921a146..47152f8e 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -29,6 +29,7 @@ from rlm.tools.base import ToolContext, ToolOutcome from rlm.tools.registry import get_builtin_tool from rlm.types import IpythonExecuted, RLMMetrics, TokenUsage +from rlm.workspace_host import WorkspaceHost from rlm.execution_transport import ExecutionTransport, StdioTransport, TcpTransport logger = logging.getLogger(__name__) @@ -208,13 +209,6 @@ def _broker(self, method, request): raise PermissionError("unknown execution callback") if request.get("capability") != self.broker_endpoint.capability: raise PermissionError("invalid execution capability") - # These operations currently launch jobs/services/watchers in the supervisor - # process. Refuse them until they have execution-session implementations. - op = request.get("op", "") - if op.startswith(("shell.", "services.")) or op == "watch.path": - raise ValueError( - "split execution does not support rlm.shell, rlm.services or watch.path; use bash/IPython in the task workspace" - ) with socket.socket(socket.AF_UNIX) as connection: connection.connect(self.broker_endpoint.socket_path) with connection.makefile("rwb", buffering=0) as stream: @@ -288,6 +282,11 @@ def _history_chunk(self, data: bytes) -> dict: def execute(self, code, timeout=None): return self.execute_tool("ipython", {"code": code, "timeout": timeout}).content + def host_call(self, method, params): + """Call the worker's WorkspaceHost: the supervisor's jobs, services and path + watches in the workspace.""" + return self._peer.call(method, params) + def interrupt(self): self._control_call("interrupt", 5) @@ -323,6 +322,7 @@ def __init__(self): self.listener = None self.closed = threading.Event() self._tool_log_sent = 0 + self.host = None def _accept(self): while not self.closed.is_set(): @@ -392,6 +392,10 @@ def handle(self, method, params): if method == "history": self._write_history(params) return {} + if method.split(".", 1)[0] in ("host", "jobs", "services", "watch"): + if self.host is None: + self.host = WorkspaceHost(self.session_dir) + return self.host.handle(method, params) if method == "tool": def execute(): @@ -446,6 +450,8 @@ def close(self): self.closed.set() if self.listener is not None: self.listener.close() + if self.host is not None: + self.host.close() if self.repl is not None: self.repl.interrupt() self.repl.shutdown() diff --git a/src/rlm/prompt.py b/src/rlm/prompt.py index 9ea6bcd4..81348c72 100644 --- a/src/rlm/prompt.py +++ b/src/rlm/prompt.py @@ -385,7 +385,6 @@ def build_system_prompt( extra_instructions: str | None = None, agent_info: dict | None = None, mcp_servers: list[dict] | None = None, - split_execution: bool = False, ) -> str: """Compose task instructions with the guide for this agent's actual runtime.""" has_ipython = _has_tool(active_tools, "ipython") @@ -445,17 +444,7 @@ def build_system_prompt( if has_ipython: parts.extend( [ - ( - "Execution uses a persistent IPython session in the task workspace. " - "Run commands through Python subprocesses or the bash skill, and edit task files there. " - "rlm.shell and rlm.watch.path are unavailable in this execution mode. " - "Recursive agents and their Python kernels share the task workspace. " - "Use await rlm.inbox.list() and await rlm.inbox.read(event_id) for supervisor events. " - "The native wait tool waits for agent events. Your conversation ledger is a snapshot " - "refreshed before each cell; await child.history() reads a child's current history." - if split_execution - else RUNTIME_PROMPT - ), + RUNTIME_PROMPT, HISTORY_PROMPT, IPYTHON_CONTROL_PROMPT, KERNEL_PACKAGES_PROMPT, diff --git a/src/rlm/shell_jobs.py b/src/rlm/shell_jobs.py index e1b5f422..f2544435 100644 --- a/src/rlm/shell_jobs.py +++ b/src/rlm/shell_jobs.py @@ -123,7 +123,7 @@ def read(self, job: JobRecord, cursor: int, max_bytes: int) -> dict: # A cursor past the retained output is not an error: it returns an empty # chunk positioned at the end, so "read until done" loops terminate. cursor = min(cursor, job.info.output_bytes) - with open(job.info.output_path, "rb") as stream: + with open(self.output_file(job), "rb") as stream: stream.seek(cursor) data = stream.read(max_bytes) return { @@ -134,6 +134,10 @@ def read(self, job: JobRecord, cursor: int, max_bytes: int) -> dict: "truncated": job.info.output_truncated, } + def output_file(self, job: JobRecord) -> str: + """Where this supervisor reads the job's output.""" + return job.info.output_path + def run_text(self, job: JobRecord) -> dict: """Text for a finished run(): the whole output when it fits RUN_TEXT_BYTES, otherwise its first and last halves around an explicit gap marker (test diff --git a/src/rlm/subscriptions.py b/src/rlm/subscriptions.py index e412921e..bac713a0 100644 --- a/src/rlm/subscriptions.py +++ b/src/rlm/subscriptions.py @@ -6,7 +6,7 @@ import json import time import uuid -from collections.abc import Callable +from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import asdict, dataclass, field, replace from pathlib import Path @@ -31,6 +31,36 @@ class Subscription: thresholds: dict = field(default_factory=dict) +PathChanges = AsyncIterator[tuple[set[str], bool]] +"""Changes under a watched path: the changed paths and whether the path still exists.""" + + +async def watch_path(target: Path, recursive: bool) -> tuple[Path, PathChanges]: + """Watch `target` on this machine; returns the resolved path and its changes.""" + target = target.resolve() + if not target.exists(): + raise FileNotFoundError(target) + return target, _changes(target, recursive) + + +async def _changes(target: Path, recursive: bool) -> PathChanges: + is_file = target.is_file() + async for changes in awatch( + target.parent if is_file else target, + recursive=recursive if not is_file else False, + watch_filter=None, + debounce=1, + step=1, + rust_timeout=100, + yield_on_timeout=True, + ignore_permission_denied=False, + ): + yield ( + {name for _, name in changes if not is_file or name == str(target)}, + target.exists(), + ) + + class Subscriptions: def __init__( self, @@ -40,6 +70,10 @@ def __init__( self.items: dict[str, Subscription] = {} self.publish = publish self.record = record + self.watch: Callable[[Path, bool], Awaitable[tuple[Path, PathChanges]]] = ( + watch_path + ) + """Where path watches run: this machine, or the execution workspace.""" def register( self, @@ -115,10 +149,13 @@ def progress( self._fail(sub, str(exc)) async def path(self, owner_id: str, target: Path, recursive: bool) -> Subscription: - if not target.exists(): - raise FileNotFoundError(target) - sub = self.register(owner_id, "path", str(target), recursive=recursive) - sub.task = asyncio.create_task(self._watch_path(sub, target)) + target, changes = await self.watch(target, recursive) + try: + sub = self.register(owner_id, "path", str(target), recursive=recursive) + except BaseException: + await changes.aclose() + raise + sub.task = asyncio.create_task(self._watch_path(sub, changes)) try: await sub.ready.wait() except BaseException: @@ -193,26 +230,12 @@ def _fail(self, sub: Subscription, error: str) -> None: self.record(sub) self.publish(sub, "watch.failed", {"error": error}) - async def _watch_path(self, sub: Subscription, target: Path) -> None: - is_file = target.is_file() - watched = target.parent if is_file else target + async def _watch_path(self, sub: Subscription, changes: PathChanges) -> None: try: - async for changes in awatch( - watched, - recursive=sub.info.recursive if not is_file else False, - watch_filter=None, - debounce=1, - step=1, - rust_timeout=100, - yield_on_timeout=True, - ignore_permission_denied=False, - ): + async for paths, exists in changes: sub.ready.set() if sub.info.status != "active": return - paths = { - name for _, name in changes if not is_file or name == str(target) - } if paths: pending = sub.pending or {"paths": set(), "truncated": False} for name in sorted(paths): @@ -225,7 +248,7 @@ async def _watch_path(self, sub: Subscription, target: Path) -> None: pending["truncated"] = True sub.pending = pending self._schedule(sub) - if not target.exists(): + if not exists: self._flush(sub) if sub.info.status == "active": self._fail( @@ -239,6 +262,7 @@ async def _watch_path(self, sub: Subscription, target: Path) -> None: self._fail(sub, str(exc)) finally: sub.ready.set() + await changes.aclose() async def cancel(self, sub: Subscription) -> dict: if sub.info.status == "active": diff --git a/src/rlm/supervisor.py b/src/rlm/supervisor.py index 093a5c20..b1a7e440 100644 --- a/src/rlm/supervisor.py +++ b/src/rlm/supervisor.py @@ -47,6 +47,7 @@ from rlm.service_manager import ServiceManager from rlm.subscriptions import Subscription, Subscriptions from rlm.tools.ipython import build_kernel_env +from rlm.workspace_host import RemoteHost, RemoteShellJobs from rlm.tools.git_block import find_blocked_command, refusal from rlm.session import Session from rlm.skills.search import run_with_api_key as run_search @@ -228,6 +229,7 @@ def __init__( self._services = ServiceManager( Path(os.environ.get("RLM_SERVICES_DIR", "/tmp/rlm-services")) ) + self._host: RemoteHost | None = None self._root_config = runtime_config if "search" in runtime_config.skills: capability = secrets.token_urlsafe(24) @@ -835,9 +837,8 @@ async def _watch_operation(self, parent: _Invocation, request: dict) -> Any: completed=job.finished is not None, ) elif op == "watch.path": - target = (Path(parent.cwd) / request["path"]).resolve() sub = await self._subscriptions.path( - parent.id, target, request["recursive"] + parent.id, Path(parent.cwd) / request["path"], request["recursive"] ) elif op == "watch.list": return [ @@ -948,16 +949,10 @@ async def _shell_operation(self, parent: _Invocation, request: dict) -> Any: if yield_after is None else min(float(yield_after), RUN_BLOCK_MAX_SECONDS) ) - info = self._shell_jobs.start( + job_options = dict( owner_id=parent.id, command=command, - cwd=str(cwd.resolve()), directory=parent.session.dir, - env={ - **build_kernel_env(dict(parent.runtime_config.kernel_env)), - **parent.shell_env, - **(request.get("env") or {}), - }, source_request_id=self._scopes[request["scope_id"]].request_id, # yield_after= bounds how long run() blocks; timeout= is the kill deadline. timeout=request.get("timeout"), @@ -965,6 +960,17 @@ async def _shell_operation(self, parent: _Invocation, request: dict) -> Any: # handed back while still running posts shell.completed when it ends. notify=block == 0, ) + if isinstance(self._shell_jobs, RemoteShellJobs): + # The workspace resolves the directory and environment it runs in. + info = await self._shell_jobs.start_remote( + cwd=str(cwd), env=self._env_layers(parent, request), **job_options + ) + else: + info = self._shell_jobs.start( + cwd=str(cwd.resolve()), + env=self._command_env(parent, request), + **job_options, + ) job = self._shell_jobs.get(parent.id, info["id"]) if block == 0: return self._running_payload( @@ -1051,6 +1057,22 @@ async def _shell_operation(self, parent: _Invocation, request: dict) -> Any: return await self._shell_jobs.cancel(job) return job.snapshot() + @staticmethod + def _env_layers(parent: _Invocation, request: dict) -> dict: + return { + "kernel_env": dict(parent.runtime_config.kernel_env), + "shell_env": dict(parent.shell_env), + "env": dict(request.get("env") or {}), + } + + def _command_env(self, parent: _Invocation, request: dict) -> dict[str, str]: + layers = self._env_layers(parent, request) + return { + **build_kernel_env(layers["kernel_env"]), + **layers["shell_env"], + **layers["env"], + } + async def _services_operation(self, parent: _Invocation, request: dict) -> Any: op = request["op"] if op == "services.start": @@ -1063,17 +1085,38 @@ async def _services_operation(self, parent: _Invocation, request: dict) -> Any: if blocked: raise PermissionError(refusal(blocked)) cwd = Path(parent.cwd) / (request["cwd"] or ".") + if self._host is not None: + return await self._host.call( + op, + { + "name": request["name"], + "command": command, + "cwd": str(cwd), + "env": self._env_layers(parent, request), + "owner_id": parent.id, + }, + ) return self._services.start( name=request["name"], command=command, cwd=str(cwd.resolve()), - env={ - **build_kernel_env(dict(parent.runtime_config.kernel_env)), - **parent.shell_env, - **(request.get("env") or {}), - }, + env=self._command_env(parent, request), owner_id=parent.id, ) + if self._host is not None and op in ( + "services.list", + "services.get", + "services.tail", + "services.stop", + ): + return await self._host.call( + op, + { + key: request[key] + for key in ("name", "lines", "grace") + if key in request + }, + ) if op == "services.list": return self._services.list() if op == "services.get": @@ -1504,6 +1547,16 @@ async def _handle_connection( pass self._connection_writers.discard(writer) + def attach_host(self, call: Callable[[str, dict], Any]) -> None: + """Run shell jobs, services and path watches in the execution workspace, through + `call` on the root agent's worker, as split execution needs.""" + self._host = RemoteHost(call) + self._shell_jobs = RemoteShellJobs( + self._host, self._publish_job, self._publish_job_output + ) + self._host.follow(self._shell_jobs) + self._subscriptions.watch = self._host.watch + async def aclose(self) -> None: if self._closed and self._close_task is None: return @@ -1541,6 +1594,8 @@ async def _aclose_impl(self) -> None: return_exceptions=True, ) ) + if self._host is not None: + await self._host.close() tasks = list(self._tasks) for task in tasks: task.cancel() diff --git a/src/rlm/workspace_host.py b/src/rlm/workspace_host.py new file mode 100644 index 00000000..fb9605ed --- /dev/null +++ b/src/rlm/workspace_host.py @@ -0,0 +1,384 @@ +"""Shell jobs, services and path watches under split execution. + +The supervisor stays with the agent loop, but every process the agent starts must run +in the task workspace, beside its kernel. The execution worker hosts them there with the +same job, service and watch code a local supervisor uses (`WorkspaceHost`); the +supervisor keeps its records and reaches the worker through `RemoteHost`. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import dataclasses +import os +import threading +import time +import uuid +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from rlm.service_manager import ServiceManager +from rlm.shell import JobInfo +from rlm.shell_jobs import JobRecord, ShellJobs +from rlm.subscriptions import PathChanges, watch_path +from rlm.tools.ipython import build_kernel_env + +EVENT_BYTES = 4 * 1024 * 1024 +"""Job output one events call carries, so its reply stays well under a frame.""" +EVENT_WAIT = 1.0 +"""Seconds an events call waits for something to report.""" +_JOB_FIELDS = [field.name for field in dataclasses.fields(JobInfo)] + + +def _env(layers: dict) -> dict[str, str]: + """A command's environment, built where it runs from the supervisor's layers.""" + return { + **build_kernel_env(layers["kernel_env"]), + **layers["shell_env"], + **layers["env"], + } + + +class WorkspaceHost: + """Runs the supervisor's shell jobs, services and path watches in this workspace, + on an event loop of its own beside the worker's kernel.""" + + def __init__(self, directory: Path) -> None: + self.directory = directory + self.loop = asyncio.new_event_loop() + threading.Thread( + target=self.loop.run_forever, name="rlm-host", daemon=True + ).start() + self.jobs = ShellJobs(self._finished, self._output) + self.services = ServiceManager( + Path(os.environ.get("RLM_SERVICES_DIR", "/tmp/rlm-services")) + ) + self.watches: dict[str, asyncio.Task] = {} + self._changed = threading.Condition() + self._jobs_changed: dict[str, bool] = {} # job id -> finished + self._sent: dict[str, int] = {} # job id -> output bytes reported + self._paths: list[dict] = [] + + def handle(self, method: str, params: dict) -> Any: + if method == "host.events": + return self._events(params["wait"]) + if method == "jobs.start": + return self._run(self._start(params)) + if method == "jobs.cancel": + return self._run(self.jobs.cancel(self.jobs.jobs[params["id"]])) + if method == "jobs.close": + return self._run(self.jobs.close(params["owner_id"])) + if method == "services.start": + return self.services.start( + name=params["name"], + command=params["command"], + cwd=os.path.realpath(params["cwd"]), + env=_env(params["env"]), + owner_id=params["owner_id"], + ) + if method == "services.list": + return self.services.list() + if method == "services.get": + return self.services.get(params["name"]) + if method == "services.tail": + return self.services.tail(params["name"], params["lines"]) + if method == "services.stop": + return self._run(self.services.stop(params["name"], params["grace"])) + if method == "watch.path": + return self._run(self._watch(params)) + if method == "watch.cancel": + if (task := self.watches.pop(params["id"], None)) is not None: + self.loop.call_soon_threadsafe(task.cancel) + return None + raise ValueError(f"unknown execution method: {method}") + + def close(self) -> None: + """End the session's jobs and watches; services outlive it, as they do a + supervisor.""" + self._run(self.jobs.close(None)) + for task in self.watches.values(): + self.loop.call_soon_threadsafe(task.cancel) + self.loop.call_soon_threadsafe(self.loop.stop) + + def _run(self, coroutine) -> Any: + return asyncio.run_coroutine_threadsafe(coroutine, self.loop).result() + + async def _start(self, params: dict) -> dict: + info = self.jobs.start( + owner_id=params["owner_id"], + command=params["command"], + cwd=os.path.realpath(params["cwd"]), + directory=self.directory, + env=_env(params["env"]), + source_request_id=None, + timeout=params["timeout"], + ) + job = self.jobs.jobs[info["id"]] + while job.info.status == "starting" and job.finished is None: + await asyncio.sleep(0.01) + return job.snapshot() + + def _output(self, job: JobRecord, offset: int) -> None: + with self._changed: + self._jobs_changed.setdefault(job.info.id, False) + self._changed.notify_all() + + def _finished(self, job: JobRecord) -> None: + with self._changed: + self._jobs_changed[job.info.id] = True + self._changed.notify_all() + + def _events(self, wait: float) -> list[dict]: + """What changed since the last call: each changed job's state and new output, + in all at most EVENT_BYTES of it, and the path watches' changes.""" + with self._changed: + self._changed.wait_for( + lambda: self._jobs_changed or self._paths, timeout=wait + ) + changed, self._jobs_changed = self._jobs_changed, {} + paths, self._paths = self._paths, [] + events, budget = [], EVENT_BYTES + for job_id, finished in changed.items(): + job = self.jobs.jobs[job_id] + sent = self._sent.get(job_id, 0) + with open(job.info.output_path, "rb") as output: + output.seek(sent) + data = output.read(budget) + budget -= len(data) + self._sent[job_id] = sent + len(data) + more = self._sent[job_id] < job.info.output_bytes + if more: # the rest goes with the next call + with self._changed: + self._jobs_changed[job_id] = ( + self._jobs_changed.get(job_id, False) or finished + ) + events.append( + { + "type": "job", + "job": job.snapshot(), + "data": base64.b64encode(data).decode(), + "finished": finished and not more, + } + ) + return events + paths + + async def _watch(self, params: dict) -> str: + target, changes = await watch_path(Path(params["path"]), params["recursive"]) + self.watches[params["id"]] = asyncio.create_task( + self._forward(params["id"], changes) + ) + return str(target) + + async def _forward(self, watch_id: str, changes: PathChanges) -> None: + first = True + try: + async for paths, exists in changes: + # The first report says the watch is ready; later ones only changes. + if first or paths or not exists: + self._push_path( + { + "type": "path", + "id": watch_id, + "paths": sorted(paths), + "exists": exists, + } + ) + first = False + if not exists: + return + except asyncio.CancelledError: + raise + except Exception as exc: + self._push_path({"type": "path", "id": watch_id, "error": str(exc)}) + finally: + await changes.aclose() + + def _push_path(self, event: dict) -> None: + with self._changed: + self._paths.append(event) + self._changed.notify_all() + + +class RemoteShellJobs(ShellJobs): + """Shell jobs that run in the execution workspace. The worker owns each process and + its output file; the supervisor keeps the job's record and a copy of its output.""" + + def __init__( + self, + host: RemoteHost, + publish: Callable[[JobRecord], None], + output: Callable[[JobRecord, int], None] | None = None, + ): + super().__init__(publish, output) + self.host = host + self._done: dict[str, asyncio.Future] = {} + self._copies: dict[str, Path] = {} + + async def start_remote( + self, + *, + owner_id: str, + command: str, + cwd: str, + directory: Path, + env: dict, + source_request_id: str | None, + timeout: float | None = None, + notify: bool = True, + ) -> dict: + snapshot = await self.host.call( + "jobs.start", + { + "owner_id": owner_id, + "command": command, + "cwd": cwd, + "env": env, + "timeout": timeout, + }, + ) + job = JobRecord( + JobInfo(**{name: snapshot[name] for name in _JOB_FIELDS}), + source_request_id, + notify=notify, + ) + job.update(output_bytes=0) # counts the copy, which fills from events + copy = directory / "jobs" / job.info.id / "output.bin" + copy.parent.mkdir(parents=True) + copy.touch() + self._copies[job.info.id] = copy + self._done[job.info.id] = asyncio.get_running_loop().create_future() + self.jobs[job.info.id] = job + job.task = asyncio.create_task(self._follow(job)) + job.task.add_done_callback(self._observe_failure) + return job.snapshot() + + def output_file(self, job: JobRecord) -> str: + return str(self._copies[job.info.id]) + + async def cancel(self, job: JobRecord) -> dict: + job.cancel_requested = True + if job.finished is None: + await self.host.call("jobs.cancel", {"id": job.info.id}) + await asyncio.shield(job.task) + return job.snapshot() + + async def close(self, owner_id: str | None = None) -> None: + jobs = [ + job + for job in self.jobs.values() + if job.finished is None + and (owner_id is None or job.info.owner_id == owner_id) + ] + for job in jobs: + job.cancel_requested = True + if jobs: + try: + await self.host.call("jobs.close", {"owner_id": owner_id}) + except Exception as exc: + self.disconnect(str(exc)) + await asyncio.gather( + *(asyncio.shield(job.task) for job in jobs), return_exceptions=True + ) + + def apply(self, event: dict) -> None: + job = self.jobs.get(event["job"]["id"]) + if job is None or job.finished is not None: + return + data = base64.b64decode(event["data"]) + start = job.info.output_bytes + if data: + with open(self._copies[job.info.id], "ab") as copy: + copy.write(data) + job.info = JobInfo( + **{ + **{name: event["job"][name] for name in _JOB_FIELDS}, + "output_bytes": start + len(data), + } + ) + if data and self.output is not None: + self.output(job, start) + if event["finished"]: + self._done[job.info.id].set_result(None) + + def disconnect(self, error: str) -> None: + """Fail every running job: the worker that ran them is gone.""" + for job_id, done in self._done.items(): + if not done.done(): + self.jobs[job_id].update( + status="failed", error=f"execution worker disconnected: {error}" + ) + done.set_result(None) + + async def _follow(self, job: JobRecord) -> None: + try: + await self._done[job.info.id] + finally: + job.finished = time.monotonic() + self.publish(job) + + +class RemoteHost: + """The supervisor's handle on the WorkspaceHost behind the root agent's worker.""" + + def __init__(self, call: Callable[[str, dict], Any]) -> None: + self._call = call + self.jobs: RemoteShellJobs | None = None + self._paths: dict[str, asyncio.Queue] = {} + self._events: asyncio.Task | None = None + + async def call(self, method: str, params: dict) -> Any: + return await asyncio.to_thread(self._call, method, params) + + def follow(self, jobs: RemoteShellJobs) -> None: + self.jobs = jobs + self._events = asyncio.create_task(self._pump()) + + async def close(self) -> None: + if self._events is not None: + self._events.cancel() + await asyncio.gather(self._events, return_exceptions=True) + + async def watch(self, target: Path, recursive: bool) -> tuple[Path, PathChanges]: + watch_id = uuid.uuid4().hex + queue: asyncio.Queue = asyncio.Queue() + self._paths[watch_id] = queue + try: + resolved = await self.call( + "watch.path", + {"id": watch_id, "path": str(target), "recursive": recursive}, + ) + except BaseException: + del self._paths[watch_id] + raise + return Path(resolved), self._changes(watch_id, queue) + + async def _changes(self, watch_id: str, queue: asyncio.Queue) -> PathChanges: + try: + while True: + event = await queue.get() + if "error" in event: + raise RuntimeError(event["error"]) + yield set(event["paths"]), event["exists"] + finally: + self._paths.pop(watch_id, None) + with contextlib.suppress(Exception): + await self.call("watch.cancel", {"id": watch_id}) + + async def _pump(self) -> None: + assert self.jobs is not None + try: + while True: + for event in await self.call("host.events", {"wait": EVENT_WAIT}): + if event["type"] == "job": + self.jobs.apply(event) + elif (queue := self._paths.get(event["id"])) is not None: + queue.put_nowait(event) + except asyncio.CancelledError: + raise + except Exception as exc: # the worker is gone + self.jobs.disconnect(str(exc)) + for queue in self._paths.values(): + queue.put_nowait({"error": f"execution worker disconnected: {exc}"}) diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 48101edb..929b804d 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -2,6 +2,8 @@ import asyncio import json +import os +import uuid import pytest @@ -301,7 +303,8 @@ def factory(**kwargs): client = DummyClient( [ _tool("value = 41; print(await bash('pwd'))"), - _tool(""" + _tool( + """ from pathlib import Path assert value == 41 child = await rlm.agent.spawn('write child.txt') @@ -311,14 +314,30 @@ def factory(**kwargs): assert Path('child.txt').read_text() == 'child' h = await rlm.history() assert h.user_messages()[0]['content'] == 'run remotely' -try: - await rlm.shell.run('touch escaped') -except RuntimeError as e: - assert 'split execution' in str(e) -else: - raise AssertionError('host shell must be unavailable') +import asyncio +# The supervisor's jobs, services and watches run in the workspace, beside this kernel. +job = await rlm.shell.run('echo $PPID; touch from-shell') +assert job.ok and int(job.text.split()[0]) != SUPERVISOR_PID, job.text +assert Path('from-shell').exists() +svc = await rlm.services.start(SERVICE, 'echo service-up; sleep 30') +for _ in range(50): + if 'service-up' in await svc.tail(): + break + await asyncio.sleep(0.1) +assert 'service-up' in await svc.tail() +assert not (await svc.stop(grace=1)).running +await rlm.watch.path('.') +Path('watched.txt').write_text('x') +for _ in range(50): + if any(e['type'] == 'watch.path' for e in await rlm.inbox.list()): + break + await asyncio.sleep(0.1) +assert any(e['type'] == 'watch.path' for e in await rlm.inbox.list()) print('REMOTE_OK') -"""), +""".replace("SUPERVISOR_PID", str(os.getpid())).replace( + "SERVICE", repr(f"split-{uuid.uuid4().hex[:8]}") + ) + ), DummyMessage(content="done"), ] ) @@ -337,7 +356,6 @@ def factory(**kwargs): and "REMOTE_OK" in record.get("content", "") for record in map(json.loads, logs.splitlines()) ) - assert not (tmp_path / "escaped").exists() finally: await engine.aclose() await supervisor.aclose() From 9340d005b326377f6f7dbb1706722e2ffe763b7a Mon Sep 17 00:00:00 2001 From: hallerite Date: Sat, 31 Oct 2026 02:52:09 +0000 Subject: [PATCH 12/21] fix: keep a remote job's events that arrive before it is registered A fast job can finish, and its output and completion reach the supervisor, before the jobs.start reply that registers it. On a one-CPU workspace VM this happens every time: those events were dropped, so the job came back without its output and run() waited out yield_after. They are now held and applied once the job is registered. The split e2e test pins its worker to one CPU, which reproduces it. Co-Authored-By: Claude Opus 5.5 --- src/rlm/workspace_host.py | 9 ++++++++- tests/test_kernel_recovery.py | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/rlm/workspace_host.py b/src/rlm/workspace_host.py index fb9605ed..72ad96ca 100644 --- a/src/rlm/workspace_host.py +++ b/src/rlm/workspace_host.py @@ -216,6 +216,7 @@ def __init__( self.host = host self._done: dict[str, asyncio.Future] = {} self._copies: dict[str, Path] = {} + self._early: dict[str, list[dict]] = {} async def start_remote( self, @@ -253,6 +254,8 @@ async def start_remote( self.jobs[job.info.id] = job job.task = asyncio.create_task(self._follow(job)) job.task.add_done_callback(self._observe_failure) + for event in self._early.pop(job.info.id, []): + self.apply(event) return job.snapshot() def output_file(self, job: JobRecord) -> str: @@ -285,7 +288,11 @@ async def close(self, owner_id: str | None = None) -> None: def apply(self, event: dict) -> None: job = self.jobs.get(event["job"]["id"]) - if job is None or job.finished is not None: + if job is None: + # A fast job's events can arrive before the reply that registers it. + self._early.setdefault(event["job"]["id"], []).append(event) + return + if job.finished is not None: return data = base64.b64decode(event["data"]) start = job.info.output_bytes diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 929b804d..94f30242 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -270,9 +270,19 @@ async def started(): async def test_remote_execution_preserves_state_and_brokers_children(session, tmp_path): import sys + # One CPU, like a sandbox VM: a fast job then finishes before the worker + # acknowledges its start, which is when its events used to get lost. + cpu = str(min(os.sched_getaffinity(0))) config = _config(max_depth=1).model_copy( update={ - "execution_command": (sys.executable, "-m", "rlm.execution"), + "execution_command": ( + "taskset", + "-c", + cpu, + sys.executable, + "-m", + "rlm.execution", + ), "skills": ("bash", "edit"), "builtin_tools": ("ipython",), "policy": _config(max_depth=1).policy.model_copy( @@ -319,6 +329,9 @@ def factory(**kwargs): job = await rlm.shell.run('echo $PPID; touch from-shell') assert job.ok and int(job.text.split()[0]) != SUPERVISOR_PID, job.text assert Path('from-shell').exists() +import time +started = time.monotonic() +assert (await rlm.shell.run('true')).ok and time.monotonic() - started < 5 svc = await rlm.services.start(SERVICE, 'echo service-up; sleep 30') for _ in range(50): if 'service-up' in await svc.tail(): From 9184e4a28081f6e4ed0d681c702eb3ba25f89d28 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sat, 31 Oct 2026 12:18:35 +0000 Subject: [PATCH 13/21] fix: send split execution frames without Nagle delay Calls overlap on the execution connection (the event long-poll, kernel callbacks, tool calls), so one end can send a small frame while its previous one is still unacknowledged; Nagle then holds it for the peer's delayed ACK. Measured on Prime, that added 40 ms to the slowest tenth of rlm.shell.run calls with output in every run; with TCP_NODELAY on both ends it is gone and medians are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/rlm/execution.py | 1 + src/rlm/execution_transport.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/rlm/execution.py b/src/rlm/execution.py index 47152f8e..8b97f92e 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -470,6 +470,7 @@ def _serve(reader, writer): def _serve_tcp(connection, token, slots): try: with connection, connection.makefile("rwb", buffering=0) as stream: + connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) connection.settimeout(10) supplied = _receive(stream).get("token") if not isinstance(supplied, str) or not hmac.compare_digest( diff --git a/src/rlm/execution_transport.py b/src/rlm/execution_transport.py index 3da49a2a..f7db1639 100644 --- a/src/rlm/execution_transport.py +++ b/src/rlm/execution_transport.py @@ -56,6 +56,9 @@ def connect(self): self.connection = socket.create_connection( (self.endpoint.host, self.endpoint.port), timeout=10 ) + # Each frame is one write, and calls overlap (the event long-poll, kernel + # callbacks): Nagle would hold a frame until the previous one is acknowledged. + self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.stream = self.connection.makefile("rwb", buffering=0) try: _send(self.stream, {"token": self.endpoint.token}) From efac509a0cfb8fea08e7c2dc5590d543853afada Mon Sep 17 00:00:00 2001 From: hallerite Date: Sat, 31 Oct 2026 12:35:08 +0000 Subject: [PATCH 14/21] perf: report a short job's output and exit together under split execution The worker answered the supervisor's events long-poll as soon as a job printed, before it exited, so learning that the job finished took another round trip: rlm.shell.run of a command with output cost three round trips instead of two. An events call now holds a running job's new output up to 50 ms for the job to finish. A running job's output reaches the supervisor at most that much later. Co-Authored-By: Claude Opus 5.5 --- src/rlm/workspace_host.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rlm/workspace_host.py b/src/rlm/workspace_host.py index 72ad96ca..64a6c4c8 100644 --- a/src/rlm/workspace_host.py +++ b/src/rlm/workspace_host.py @@ -30,6 +30,9 @@ """Job output one events call carries, so its reply stays well under a frame.""" EVENT_WAIT = 1.0 """Seconds an events call waits for something to report.""" +EVENT_LINGER = 0.05 +"""Seconds an events call holds a running job's new output for the job to finish, so +a short command's output and exit cost the supervisor one round trip, not two.""" _JOB_FIELDS = [field.name for field in dataclasses.fields(JobInfo)] @@ -138,6 +141,9 @@ def _events(self, wait: float) -> list[dict]: self._changed.wait_for( lambda: self._jobs_changed or self._paths, timeout=wait ) + self._changed.wait_for( + lambda: all(self._jobs_changed.values()), timeout=EVENT_LINGER + ) changed, self._jobs_changed = self._jobs_changed, {} paths, self._paths = self._paths, [] events, budget = [], EVENT_BYTES From 65885476abdf7e3de2fb9fe5f5a80765b905e6b4 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 1 Nov 2026 01:26:04 +0000 Subject: [PATCH 15/21] fix: keep a long output's start and end instead of stopping it at 1 MiB A bash command or cell that printed more than 1 MiB was killed or interrupted, though the conversation only ever shows a result's first and last 10 KB. Output is now kept to its first and last 512 KiB while it is produced (bash splits that between stdout and stderr), so the command runs to its end in bounded memory and a tool result still fits one execution frame. The result carries how much more the whole output measured, so the conversation shows the same start, end, token count and line count as for the whole output. The session log keeps the start and end, with a note where the middle was dropped. The bash tool again translates \r\n and \r to \n, as subprocess text mode did. Co-Authored-By: Claude Opus 5.5 --- src/rlm/compaction.py | 27 ++++++---- src/rlm/engine.py | 4 +- src/rlm/execution.py | 7 ++- src/rlm/tools/base.py | 4 ++ src/rlm/tools/bash.py | 66 +++++++++++++++---------- src/rlm/tools/ipython.py | 33 ++++++------- src/rlm/tools/output.py | 92 +++++++++++++++++++++++++++++++++++ tests/test_kernel_recovery.py | 72 ++++++++++++++++++++++++--- 8 files changed, 245 insertions(+), 60 deletions(-) create mode 100644 src/rlm/tools/output.py diff --git a/src/rlm/compaction.py b/src/rlm/compaction.py index 9330c790..c3d30b02 100644 --- a/src/rlm/compaction.py +++ b/src/rlm/compaction.py @@ -5,6 +5,8 @@ from openai import APIError, APIStatusError, AsyncOpenAI +from rlm.tools.output import OutputSize + CHECKPOINT_PROMPT = """Create a concise continuation summary for the current task. Preserve what is needed to resume accurately: - The user's objective, exact requirements, constraints, and unresolved decisions. @@ -136,24 +138,31 @@ async def discover_threshold(client: AsyncOpenAI, model: str) -> int | None: return default_threshold(window) if window is not None else None -def truncate_tool_output(text: str, max_bytes: int = TOOL_OUTPUT_MAX_BYTES) -> str: - """Keep the head and tail of an oversized tool result and say what was cut.""" +def truncate_tool_output( + text: str, + max_bytes: int = TOOL_OUTPUT_MAX_BYTES, + omitted: OutputSize = OutputSize(), +) -> str: + """Keep the head and tail of an oversized tool result and say what was cut. + `omitted` is how much more the whole result measures than `text`, which kept only + its start and end; the note describes the whole result.""" data = text.encode("utf-8") - if len(data) <= max_bytes: + size = OutputSize.of(text) + omitted + if size.nbytes <= max_bytes: return text keep = max_bytes // 2 head = data[:keep].decode("utf-8", errors="ignore") tail = data[-keep:].decode("utf-8", errors="ignore") return ( - f"Warning: truncated output (original token count: {estimated_tokens(text)})\n" - f"Total output lines: {text.count(chr(10)) + 1}\n\n" - f"{head}\n[... {len(data) - 2 * keep} bytes truncated ...]\n{tail}" + f"Warning: truncated output (original token count: {estimated_tokens(size.nchars)})\n" + f"Total output lines: {size.newlines + 1}\n\n" + f"{head}\n[... {size.nbytes - 2 * keep} bytes truncated ...]\n{tail}" ) -def estimated_tokens(chars: str) -> int: - """Rough token count at four characters per token.""" - return (len(chars) + 3) // 4 +def estimated_tokens(chars: str | int) -> int: + """Rough token count at four characters per token, of text or a character count.""" + return ((chars if isinstance(chars, int) else len(chars)) + 3) // 4 def compactable(messages: list[dict]) -> bool: diff --git a/src/rlm/engine.py b/src/rlm/engine.py index e8d9a827..ad4745e9 100644 --- a/src/rlm/engine.py +++ b/src/rlm/engine.py @@ -1001,7 +1001,9 @@ async def _run_loop(self) -> RLMResult: result = tool_result.content content = truncate_tool_output( - result, self.max_tool_output_bytes or TOOL_OUTPUT_MAX_BYTES + result, + self.max_tool_output_bytes or TOOL_OUTPUT_MAX_BYTES, + tool_result.omitted, ) self.session.log_tool_result( turn, diff --git a/src/rlm/execution.py b/src/rlm/execution.py index 8b97f92e..2b81958b 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -27,6 +27,7 @@ from rlm.broker import BrokerEndpoint from rlm.tools.ipython import IPythonREPL, build_kernel_env from rlm.tools.base import ToolContext, ToolOutcome +from rlm.tools.output import OutputSize from rlm.tools.registry import get_builtin_tool from rlm.types import IpythonExecuted, RLMMetrics, TokenUsage from rlm.workspace_host import WorkspaceHost @@ -274,13 +275,16 @@ def execute_tool(self, name, args): metric_events=[ IpythonExecuted(**event) for event in result["metric_events"] ], + omitted=OutputSize(**result["omitted"]), ) def _history_chunk(self, data: bytes) -> dict: return {"offset": self._history_sent, "data": base64.b64encode(data).decode()} def execute(self, code, timeout=None): - return self.execute_tool("ipython", {"code": code, "timeout": timeout}).content + outcome = self.execute_tool("ipython", {"code": code, "timeout": timeout}) + self.omitted = outcome.omitted + return outcome.content def host_call(self, method, params): """Call the worker's WorkspaceHost: the supervisor's jobs, services and path @@ -407,6 +411,7 @@ def execute(): outcome = tool.execute(params["args"], self.context) return { "output": outcome.content, + "omitted": asdict(outcome.omitted), "metric_events": [asdict(event) for event in outcome.metric_events], "notices": self.repl.take_recovery_notices(), "tool_log": base64.b64encode(self._new_tool_log()).decode(), diff --git a/src/rlm/tools/base.py b/src/rlm/tools/base.py index ec5d7d5c..6cd2100d 100644 --- a/src/rlm/tools/base.py +++ b/src/rlm/tools/base.py @@ -6,6 +6,7 @@ from typing import Any, Protocol from threading import Event +from rlm.tools.output import OutputSize from rlm.types import BuiltinMetricEvent, RLMMetrics, TokenUsage @@ -15,6 +16,9 @@ class ToolOutcome: content: str metric_events: list[BuiltinMetricEvent] = field(default_factory=list) + omitted: OutputSize = OutputSize() + """How much more the whole output measures than `content`, which kept only its + start and end of a long output.""" @dataclass diff --git a/src/rlm/tools/bash.py b/src/rlm/tools/bash.py index bda18cea..08f380ae 100644 --- a/src/rlm/tools/bash.py +++ b/src/rlm/tools/bash.py @@ -7,6 +7,8 @@ from __future__ import annotations +import codecs +import io import os import selectors import signal @@ -18,6 +20,7 @@ from rlm.tools.base import ToolContext, ToolOutcome from rlm.tools.git_block import find_blocked_command, refusal +from rlm.tools.output import OUTPUT_KEEP_BYTES, OutputCapture BASH_SCHEMA = { "type": "function", @@ -70,14 +73,33 @@ def run_bash( interrupt_event: Event | None = None, ) -> str: """Guarded ``bash -c`` execution shared by the bash tool and the bash skill.""" + return bash_outcome(command, timeout, cwd, allow_git, env, interrupt_event).content + + +def bash_outcome( + command: str, + timeout: int, + cwd: str | None = None, + allow_git: bool | None = None, + env: dict[str, str] | None = None, + interrupt_event: Event | None = None, +) -> ToolOutcome: + """`run_bash`, with how much of a long output its content left out.""" if not isinstance(command, str) or not command.strip(): - return "Error: empty command" + return ToolOutcome(content="Error: empty command") blocked = find_blocked_command(command, allow_git=allow_git) if blocked: - return refusal(blocked) - limit = 1024 * 1024 + return ToolOutcome(content=refusal(blocked)) deadline = time.monotonic() + timeout - parts = {"stdout": bytearray(), "stderr": bytearray()} + streams = ("stdout", "stderr") + # stdout and stderr each keep half of one result's bound. + captures = {name: OutputCapture(OUTPUT_KEEP_BYTES // 2) for name in streams} + decoders = { + name: io.IncrementalNewlineDecoder( + codecs.getincrementaldecoder("utf-8")(errors="replace"), translate=True + ) + for name in streams + } reason = "" with ( subprocess.Popen( @@ -106,15 +128,7 @@ def run_bash( if not data: selector.unregister(key.fileobj) continue - available = limit - sum(map(len, parts.values())) - parts[key.data].extend(data[:available]) - if len(data) >= available: - reason = ( - "[command output exceeded 1 MiB; process group terminated]" - ) - break - if reason: - break + captures[key.data].add(decoders[key.data].decode(data)) if not reason: try: proc.wait(timeout=max(0.01, deadline - time.monotonic())) @@ -125,15 +139,19 @@ def run_bash( with contextlib.suppress(ProcessLookupError): os.killpg(proc.pid, signal.SIGKILL) proc.wait() - stdout, stderr = ( - parts[key].decode(errors="replace") for key in ("stdout", "stderr") + for name in streams: + captures[name].add(decoders[name].decode(b"", final=True)) + (stdout, stdout_omitted), (stderr, stderr_omitted) = ( + captures[name].result() for name in streams ) out = stdout + (("\n" + stderr) if stderr else "") if reason: out += "\n" + reason elif proc.returncode != 0: out += f"\n[exit code {proc.returncode}]" - return out.strip() or "(no output)" + return ToolOutcome( + content=out.strip() or "(no output)", omitted=stdout_omitted + stderr_omitted + ) class BashTool: @@ -145,15 +163,13 @@ def schema(self) -> dict[str, Any]: return BASH_SCHEMA def execute(self, args: dict[str, Any], context: ToolContext) -> ToolOutcome: - return ToolOutcome( - content=run_bash( - args.get("command", ""), - context.exec_timeout, - cwd=context.cwd, - allow_git=context.allow_git, - env=context.env, - interrupt_event=context.interrupt_event, - ) + return bash_outcome( + args.get("command", ""), + context.exec_timeout, + cwd=context.cwd, + allow_git=context.allow_git, + env=context.env, + interrupt_event=context.interrupt_event, ) diff --git a/src/rlm/tools/ipython.py b/src/rlm/tools/ipython.py index 6ad45559..f5fe5481 100644 --- a/src/rlm/tools/ipython.py +++ b/src/rlm/tools/ipython.py @@ -24,6 +24,7 @@ from rlm.tools.base import ToolContext, ToolOutcome from rlm.tools.git_block import find_blocked_in_ipython, refusal +from rlm.tools.output import OutputCapture, OutputSize from rlm.tools.skills import discover_skills from rlm.types import IpythonExecuted @@ -240,8 +241,10 @@ def execute(self, args: dict[str, Any], context: ToolContext) -> ToolOutcome: metric_events=metric_events, ) + content = context.repl.execute(code, timeout=timeout) return ToolOutcome( - content=context.repl.execute(code, timeout=timeout), + content=content, + omitted=getattr(context.repl, "omitted", OutputSize()), metric_events=metric_events, ) @@ -257,7 +260,6 @@ def __init__(self, output: str = ""): MAX_RECOVERY_ATTEMPTS = 3 -MAX_CELL_OUTPUT_BYTES = 1024 * 1024 def kill_marked_groups(marker: str) -> None: @@ -324,6 +326,9 @@ def __init__( self._recovery_attempts = 0 self._recovery_failed = False self._cell_submitted = False + self.omitted = OutputSize() + """How much more the last cell's whole output measured than what `execute` + returned, which kept only its start and end of a long output.""" self._recovery_notices: list[str] = [] self._executor = ThreadPoolExecutor( max_workers=1, thread_name_prefix="rlm-repl" @@ -605,6 +610,7 @@ def execute(self, code: str, timeout: int | None = None) -> str: """Execute once; a lost kernel is recovered without replaying the cell.""" with self._lock: self._cell_submitted = False + self.omitted = OutputSize() try: if self._interrupt_requested.is_set(): return "" @@ -642,12 +648,12 @@ def _execute_locked(self, code: str, timeout: int | None) -> str: self._cell_submitted = True deadline = None if timeout is None else time.monotonic() + timeout - outputs: list[str] = [] - output_bytes = 0 + capture = OutputCapture() try: while True: if not self._km.is_alive(): - raise _KernelDied("".join(outputs)) + text, self.omitted = capture.result() + raise _KernelDied(text) if self._interrupt_requested.is_set(): self._interrupt_and_recover(msg_id) break @@ -657,7 +663,7 @@ def _execute_locked(self, code: str, timeout: int | None) -> str: remaining = deadline - time.monotonic() if remaining <= 0: self._interrupt_and_recover(msg_id) - outputs.append( + capture.add( f"\n[execution timed out after {timeout}s and was interrupted]" ) break @@ -686,17 +692,7 @@ def _execute_locked(self, code: str, timeout: int | None) -> str: text = tb elif msg_type == "status" and content["execution_state"] == "idle": break - if text: - data = text.encode("utf-8", errors="replace") - remaining = MAX_CELL_OUTPUT_BYTES - output_bytes - outputs.append(data[:remaining].decode("utf-8", errors="replace")) - output_bytes += min(len(data), remaining) - if len(data) >= remaining: - self._interrupt_and_recover(msg_id) - outputs.append( - "\n[cell output exceeded 1 MiB; execution interrupted]" - ) - break + capture.add(text) finally: try: timeout = 0.1 if self._interrupt_requested.is_set() else 5 @@ -705,7 +701,8 @@ def _execute_locked(self, code: str, timeout: int | None) -> str: except Exception: pass - return "".join(outputs) + text, self.omitted = capture.result() + return text def shutdown(self): """Stop the kernel.""" diff --git a/src/rlm/tools/output.py b/src/rlm/tools/output.py new file mode 100644 index 00000000..ee6d0ee9 --- /dev/null +++ b/src/rlm/tools/output.py @@ -0,0 +1,92 @@ +"""Bounded capture of a command's output.""" + +from __future__ import annotations + +from dataclasses import dataclass + +OUTPUT_KEEP_BYTES = 1024 * 1024 +"""The most of one tool result's output that is kept, half from its start and half from +its end. The conversation shows far less of it; the rest stays in the session log, and +a tool result always fits one execution frame.""" + + +@dataclass(frozen=True) +class OutputSize: + """Text measured as the conversation's truncation measures it.""" + + nbytes: int = 0 + """UTF-8 bytes.""" + nchars: int = 0 + newlines: int = 0 + + @classmethod + def of(cls, text: str) -> OutputSize: + return cls( + len(text.encode("utf-8", "surrogatepass")), len(text), text.count("\n") + ) + + def __add__(self, other: OutputSize) -> OutputSize: + return OutputSize( + self.nbytes + other.nbytes, + self.nchars + other.nchars, + self.newlines + other.newlines, + ) + + def __sub__(self, other: OutputSize) -> OutputSize: + return OutputSize( + self.nbytes - other.nbytes, + self.nchars - other.nchars, + self.newlines - other.newlines, + ) + + +def _char_start(data: bytes | bytearray, at: int) -> int: + """`at`, moved back to the start of the UTF-8 character it falls in.""" + while 0 < at < len(data) and data[at] & 0xC0 == 0x80: + at -= 1 + return at + + +class OutputCapture: + """Output as it is produced, kept to its first and last `keep // 2` bytes. The middle + is dropped but measured, so a long command runs to its end in bounded memory and the + whole output's size is still known.""" + + def __init__(self, keep: int = OUTPUT_KEEP_BYTES): + self._half = keep // 2 + self._head = bytearray() + self._head_full = False + self._tail = bytearray() + self._dropped = False + self.size = OutputSize() + """The whole output's size.""" + + def add(self, text: str) -> None: + if not text: + return + self.size += OutputSize.of(text) + data = text.encode("utf-8", "surrogatepass") + if not self._head_full: + room = self._half - len(self._head) + if len(data) <= room: + self._head += data + return + cut = _char_start(data, room) + self._head += data[:cut] + self._head_full = True + data = data[cut:] + self._tail += data + if len(self._tail) > self._half: + del self._tail[: _char_start(self._tail, len(self._tail) - self._half)] + self._dropped = True + + def result(self) -> tuple[str, OutputSize]: + """The kept text, and how much more the whole output measures than it: when + its middle was dropped, the middle less the note that replaces it.""" + head = self._head.decode("utf-8", "surrogatepass") + tail = self._tail.decode("utf-8", "surrogatepass") + if not self._dropped: + return head + tail, OutputSize() + dropped = self.size.nbytes - len(self._head) - len(self._tail) + text = f"{head}\n[... {dropped} bytes of output omitted ...]\n{tail}" + return text, self.size - OutputSize.of(text) diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 94f30242..601c9d39 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -403,25 +403,85 @@ async def test_remote_execution_interrupt_and_recovery(session, tmp_path): await asyncio.to_thread(repl.shutdown) -async def test_output_flood_is_bounded_and_kernel_can_continue(session): - import asyncio - from rlm.tools.ipython import MAX_CELL_OUTPUT_BYTES +async def test_endless_output_is_bounded_until_the_timeout(session): + from rlm.tools.bash import run_bash + from rlm.tools.output import OUTPUT_KEEP_BYTES repl = IPythonREPL(session=session, cwd=str(session.dir)) try: await repl.run_in_thread(repl.start) output = await asyncio.wait_for( repl.run_in_thread( - repl.execute, "while True: print('x' * 65536, flush=True)", 15 + repl.execute, "while True: print('x' * 65536, flush=True)", 5 ), timeout=25, ) - assert len(output.encode()) < MAX_CELL_OUTPUT_BYTES + 256 - assert "output exceeded" in output + assert len(output.encode()) < OUTPUT_KEEP_BYTES + 256 + assert output.endswith("[execution timed out after 5s and was interrupted]") + assert repl.omitted.nbytes > 0 continued = await repl.run_in_thread(repl.execute, "print('STILL_ALIVE')", 10) assert "STILL_ALIVE" in continued finally: await repl.run_in_thread(repl.shutdown) + output = await asyncio.to_thread(run_bash, "yes", 3) + assert len(output.encode()) < OUTPUT_KEEP_BYTES + 256 + assert output.endswith("Error: command timed out after 3s") + + +@pytest.mark.parametrize("split", [False, True], ids=["local", "split"]) +@pytest.mark.parametrize("tool", ["bash", "ipython"]) +async def test_long_output_runs_to_its_end_and_reads_as_whole( + session, tmp_path, split, tool +): + """A command that prints far more than a tool result keeps runs to its end, and the + model sees the truncation of its whole output: the same start, end and counts.""" + import sys + + from rlm.compaction import truncate_tool_output + from rlm.tools.output import OUTPUT_KEEP_BYTES + + lines = "".join(f"{i}\n" for i in range(1, 300001)) + if tool == "bash": + call = DummyToolCall( + "bash", + { + "command": "printf 'a\\r\\nb\\rc\\n'; seq 300000; echo DONE; " + "seq 200000 >&2; echo ERR_END >&2" + }, + ) + errors = "".join(f"{i}\n" for i in range(1, 200001)) + whole = f"a\nb\nc\n{lines}DONE\n\n{errors}ERR_END\n".strip() + else: + call = DummyToolCall( + "ipython", {"code": "for i in range(1, 300001): print(i)\nprint('DONE')"} + ) + whole = f"{lines}DONE\n" + config = _config().model_copy(update={"builtin_tools": ("ipython", "bash")}) + if split: + config = config.model_copy( + update={"execution_command": (sys.executable, "-m", "rlm.execution")} + ) + client = DummyClient( + [DummyMessage(tool_calls=[call]), DummyMessage(content="done")] + ) + engine = RLMEngine( + client=client, session=session, runtime_config=config, cwd=str(tmp_path) + ) + try: + await engine.prompt("print a lot") + finally: + await engine.aclose() + seen = [m for m in client.calls[1]["messages"] if m["role"] == "tool"][-1] + assert seen["content"] == truncate_tool_output(whole) + logged = [ + record["content"] + for record in map(json.loads, (session.dir / "messages.jsonl").open()) + if record.get("type") == "tool_result" + ][-1] + assert "bytes of output omitted" in logged + assert len(logged.encode()) < OUTPUT_KEEP_BYTES + 512 + assert logged[:10_000] == whole[:10_000] + assert logged[-10_000:] == whole[-10_000:] async def test_bash_commands_stop_with_a_cancelled_turn_and_the_session( From 2e4629b78c62a96edb5590258e993baac77e7de0 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 1 Nov 2026 01:44:56 +0000 Subject: [PATCH 16/21] fix: show no more of a long output's ends than were kept With a tool output budget larger than what a long output kept, the head and tail slices overlapped, so the conversation showed part of the output twice and miscounted the truncated bytes. When an output's middle was dropped, each end shown is now at most what every tool keeps of it. Co-Authored-By: Claude Opus 5.5 --- src/rlm/compaction.py | 8 +++++--- src/rlm/execution.py | 4 ++-- src/rlm/tools/base.py | 6 +++--- src/rlm/tools/bash.py | 6 ++++-- src/rlm/tools/ipython.py | 10 +++++----- src/rlm/tools/output.py | 13 +++++++++---- tests/test_kernel_recovery.py | 23 +++++++++++++++++++---- 7 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/rlm/compaction.py b/src/rlm/compaction.py index c3d30b02..98d5ae6e 100644 --- a/src/rlm/compaction.py +++ b/src/rlm/compaction.py @@ -5,7 +5,7 @@ from openai import APIError, APIStatusError, AsyncOpenAI -from rlm.tools.output import OutputSize +from rlm.tools.output import KEPT_PER_END, OutputSize CHECKPOINT_PROMPT = """Create a concise continuation summary for the current task. Preserve what is needed to resume accurately: @@ -141,16 +141,18 @@ async def discover_threshold(client: AsyncOpenAI, model: str) -> int | None: def truncate_tool_output( text: str, max_bytes: int = TOOL_OUTPUT_MAX_BYTES, - omitted: OutputSize = OutputSize(), + omitted: OutputSize | None = None, ) -> str: """Keep the head and tail of an oversized tool result and say what was cut. `omitted` is how much more the whole result measures than `text`, which kept only its start and end; the note describes the whole result.""" data = text.encode("utf-8") - size = OutputSize.of(text) + omitted + size = OutputSize.of(text) + (omitted or OutputSize()) if size.nbytes <= max_bytes: return text keep = max_bytes // 2 + if omitted is not None: + keep = min(keep, KEPT_PER_END) head = data[:keep].decode("utf-8", errors="ignore") tail = data[-keep:].decode("utf-8", errors="ignore") return ( diff --git a/src/rlm/execution.py b/src/rlm/execution.py index 2b81958b..be91972c 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -275,7 +275,7 @@ def execute_tool(self, name, args): metric_events=[ IpythonExecuted(**event) for event in result["metric_events"] ], - omitted=OutputSize(**result["omitted"]), + omitted=result["omitted"] and OutputSize(**result["omitted"]), ) def _history_chunk(self, data: bytes) -> dict: @@ -411,7 +411,7 @@ def execute(): outcome = tool.execute(params["args"], self.context) return { "output": outcome.content, - "omitted": asdict(outcome.omitted), + "omitted": outcome.omitted and asdict(outcome.omitted), "metric_events": [asdict(event) for event in outcome.metric_events], "notices": self.repl.take_recovery_notices(), "tool_log": base64.b64encode(self._new_tool_log()).decode(), diff --git a/src/rlm/tools/base.py b/src/rlm/tools/base.py index 6cd2100d..6298a33a 100644 --- a/src/rlm/tools/base.py +++ b/src/rlm/tools/base.py @@ -16,9 +16,9 @@ class ToolOutcome: content: str metric_events: list[BuiltinMetricEvent] = field(default_factory=list) - omitted: OutputSize = OutputSize() - """How much more the whole output measures than `content`, which kept only its - start and end of a long output.""" + omitted: OutputSize | None = None + """How much more the whole output measures than `content`, when `content` kept + only its start and end.""" @dataclass diff --git a/src/rlm/tools/bash.py b/src/rlm/tools/bash.py index 08f380ae..da1f8562 100644 --- a/src/rlm/tools/bash.py +++ b/src/rlm/tools/bash.py @@ -20,7 +20,7 @@ from rlm.tools.base import ToolContext, ToolOutcome from rlm.tools.git_block import find_blocked_command, refusal -from rlm.tools.output import OUTPUT_KEEP_BYTES, OutputCapture +from rlm.tools.output import OUTPUT_KEEP_BYTES, OutputCapture, OutputSize BASH_SCHEMA = { "type": "function", @@ -149,8 +149,10 @@ def bash_outcome( out += "\n" + reason elif proc.returncode != 0: out += f"\n[exit code {proc.returncode}]" + omitted = [size for size in (stdout_omitted, stderr_omitted) if size is not None] return ToolOutcome( - content=out.strip() or "(no output)", omitted=stdout_omitted + stderr_omitted + content=out.strip() or "(no output)", + omitted=sum(omitted, OutputSize()) if omitted else None, ) diff --git a/src/rlm/tools/ipython.py b/src/rlm/tools/ipython.py index f5fe5481..92866194 100644 --- a/src/rlm/tools/ipython.py +++ b/src/rlm/tools/ipython.py @@ -24,7 +24,7 @@ from rlm.tools.base import ToolContext, ToolOutcome from rlm.tools.git_block import find_blocked_in_ipython, refusal -from rlm.tools.output import OutputCapture, OutputSize +from rlm.tools.output import OutputCapture from rlm.tools.skills import discover_skills from rlm.types import IpythonExecuted @@ -244,7 +244,7 @@ def execute(self, args: dict[str, Any], context: ToolContext) -> ToolOutcome: content = context.repl.execute(code, timeout=timeout) return ToolOutcome( content=content, - omitted=getattr(context.repl, "omitted", OutputSize()), + omitted=getattr(context.repl, "omitted", None), metric_events=metric_events, ) @@ -326,9 +326,9 @@ def __init__( self._recovery_attempts = 0 self._recovery_failed = False self._cell_submitted = False - self.omitted = OutputSize() + self.omitted = None """How much more the last cell's whole output measured than what `execute` - returned, which kept only its start and end of a long output.""" + returned, when that kept only its start and end.""" self._recovery_notices: list[str] = [] self._executor = ThreadPoolExecutor( max_workers=1, thread_name_prefix="rlm-repl" @@ -610,7 +610,7 @@ def execute(self, code: str, timeout: int | None = None) -> str: """Execute once; a lost kernel is recovered without replaying the cell.""" with self._lock: self._cell_submitted = False - self.omitted = OutputSize() + self.omitted = None try: if self._interrupt_requested.is_set(): return "" diff --git a/src/rlm/tools/output.py b/src/rlm/tools/output.py index ee6d0ee9..5390d4f1 100644 --- a/src/rlm/tools/output.py +++ b/src/rlm/tools/output.py @@ -9,6 +9,11 @@ its end. The conversation shows far less of it; the rest stays in the session log, and a tool result always fits one execution frame.""" +KEPT_PER_END = OUTPUT_KEEP_BYTES // 4 - 4096 +"""Bytes of a long output's start, and of its end, that a tool result holds at least: +bash keeps a quarter of the bound from each end of each stream, less what a cut at a +character boundary or stripped whitespace takes.""" + @dataclass(frozen=True) class OutputSize: @@ -80,13 +85,13 @@ def add(self, text: str) -> None: del self._tail[: _char_start(self._tail, len(self._tail) - self._half)] self._dropped = True - def result(self) -> tuple[str, OutputSize]: - """The kept text, and how much more the whole output measures than it: when - its middle was dropped, the middle less the note that replaces it.""" + def result(self) -> tuple[str, OutputSize | None]: + """The kept text and, when its middle was dropped, how much more the whole + output measures than the text: the middle, less the note that replaces it.""" head = self._head.decode("utf-8", "surrogatepass") tail = self._tail.decode("utf-8", "surrogatepass") if not self._dropped: - return head + tail, OutputSize() + return head + tail, None dropped = self.size.nbytes - len(self._head) - len(self._tail) text = f"{head}\n[... {dropped} bytes of output omitted ...]\n{tail}" return text, self.size - OutputSize.of(text) diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 601c9d39..54f29447 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -428,13 +428,15 @@ async def test_endless_output_is_bounded_until_the_timeout(session): assert output.endswith("Error: command timed out after 3s") +@pytest.mark.parametrize("budget", [None, 3 << 20], ids=["default", "large"]) @pytest.mark.parametrize("split", [False, True], ids=["local", "split"]) @pytest.mark.parametrize("tool", ["bash", "ipython"]) async def test_long_output_runs_to_its_end_and_reads_as_whole( - session, tmp_path, split, tool + session, tmp_path, split, tool, budget ): """A command that prints far more than a tool result keeps runs to its end, and the - model sees the truncation of its whole output: the same start, end and counts.""" + model sees the truncation of its whole output: the same start, end and counts. A + budget larger than what was kept shows no more of either end than that, once.""" import sys from rlm.compaction import truncate_tool_output @@ -456,7 +458,15 @@ async def test_long_output_runs_to_its_end_and_reads_as_whole( "ipython", {"code": "for i in range(1, 300001): print(i)\nprint('DONE')"} ) whole = f"{lines}DONE\n" - config = _config().model_copy(update={"builtin_tools": ("ipython", "bash")}) + config = _config() + config = config.model_copy( + update={ + "builtin_tools": ("ipython", "bash"), + "policy": config.policy.model_copy( + update={"max_tool_output_bytes": budget} + ), + } + ) if split: config = config.model_copy( update={"execution_command": (sys.executable, "-m", "rlm.execution")} @@ -472,7 +482,12 @@ async def test_long_output_runs_to_its_end_and_reads_as_whole( finally: await engine.aclose() seen = [m for m in client.calls[1]["messages"] if m["role"] == "tool"][-1] - assert seen["content"] == truncate_tool_output(whole) + if budget is None: + assert seen["content"] == truncate_tool_output(whole) + else: + assert seen["content"].count(whole[:200]) == 1 + assert seen["content"].count(whole[-200:]) == 1 + assert len(seen["content"].encode()) <= budget logged = [ record["content"] for record in map(json.loads, (session.dir / "messages.jsonl").open()) From 023b3e9a7658d0e02c8a13050d5fa8d9eae32f08 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 1 Nov 2026 01:56:33 +0000 Subject: [PATCH 17/21] fix: stop what is attached to the kernel, and leave daemons running A kernel restart or shutdown killed every process that inherited the kernel's environment marker, which includes daemons the agent started and background jobs a bash command left behind, though a daemon is meant to outlive the terminal that started it and the grader may need it running. Restart and shutdown now kill what is still in the kernel's session, as a closing terminal does: the kernel leads a session of its own, and a bash command leads a process group in its caller's session, so a timeout or a cancel still stops the command and what it started. A process in a session of its own outlives the kernel and the agent, and the bash tool's commands, which the kernel does not run, are not stopped with it. Co-Authored-By: Claude Opus 5.5 --- src/rlm/engine.py | 4 +- src/rlm/execution.py | 2 +- src/rlm/tools/bash.py | 10 ++++- src/rlm/tools/ipython.py | 47 ++++++++-------------- tests/test_kernel_recovery.py | 73 +++++++++++++++++++++++------------ 5 files changed, 75 insertions(+), 61 deletions(-) diff --git a/src/rlm/engine.py b/src/rlm/engine.py index ad4745e9..9139d13a 100644 --- a/src/rlm/engine.py +++ b/src/rlm/engine.py @@ -1546,9 +1546,7 @@ def _tool_context(self, messages: list[dict]) -> ToolContext: repl=self._repl, state=self._tool_state, cwd=self.cwd, - # A shell command stops with a cancelled turn, and with the session's - # kernel like the kernel's own commands. - env=repl.command_env(os.environ) if repl else None, + # A shell command stops with a cancelled turn. interrupt_event=repl._interrupt_requested if repl else None, ) diff --git a/src/rlm/execution.py b/src/rlm/execution.py index be91972c..6d2ad1c5 100644 --- a/src/rlm/execution.py +++ b/src/rlm/execution.py @@ -385,7 +385,7 @@ def handle(self, method, params): allow_git=self.repl.allow_git, repl=self.repl, cwd=self.repl.cwd, - env=self.repl.command_env(build_kernel_env(self.repl.kernel_env)), + env=build_kernel_env(self.repl.kernel_env), interrupt_event=self.repl._interrupt_requested, ) # All Jupyter operations use the same dedicated thread. diff --git a/src/rlm/tools/bash.py b/src/rlm/tools/bash.py index da1f8562..90cf929a 100644 --- a/src/rlm/tools/bash.py +++ b/src/rlm/tools/bash.py @@ -12,6 +12,7 @@ import os import selectors import signal +import sys import time import contextlib import subprocess @@ -22,6 +23,13 @@ from rlm.tools.git_block import find_blocked_command, refusal from rlm.tools.output import OUTPUT_KEEP_BYTES, OutputCapture, OutputSize +_OWN_GROUP = ( + {"process_group": 0} if sys.version_info >= (3, 11) else {"preexec_fn": os.setpgrp} +) +"""A command leads a process group in its caller's session: a timeout or a cancel stops +the command and what it started, a kernel restart or shutdown stops a command the kernel +ran, and a daemon, in a session of its own, outlives both.""" + BASH_SCHEMA = { "type": "function", "function": { @@ -108,7 +116,7 @@ def bash_outcome( stderr=subprocess.PIPE, cwd=cwd or None, env=env, - start_new_session=True, + **_OWN_GROUP, ) as proc, selectors.DefaultSelector() as selector, ): diff --git a/src/rlm/tools/ipython.py b/src/rlm/tools/ipython.py index 92866194..374598aa 100644 --- a/src/rlm/tools/ipython.py +++ b/src/rlm/tools/ipython.py @@ -16,7 +16,6 @@ import tempfile import threading import time -import uuid from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -262,36 +261,28 @@ def __init__(self, output: str = ""): MAX_RECOVERY_ATTEMPTS = 3 -def kill_marked_groups(marker: str) -> None: - """SIGKILL the process group of every process whose environment holds `marker`, - read from /proc; a no-op where there is none. Reparented and detached processes - keep their environment, so this also reaches the commands of a dead kernel. The - caller's own group is spared.""" +def kill_session(session: int) -> None: + """SIGKILL the process group of every process in `session`, read from /proc; a no-op + where there is none. Reparented processes stay in their session, so this also + reaches the commands of a dead kernel, while a daemon, in a session of its own, is + left running. The caller's own group is spared.""" try: entries = [entry for entry in os.listdir("/proc") if entry.isdigit()] except FileNotFoundError: return - needle = marker.encode() groups = set() for entry in entries: - try: - with open(f"/proc/{entry}/environ", "rb") as environ: - if needle not in environ.read().split(b"\0"): - continue - groups.add(os.getpgid(int(entry))) - except OSError: - continue + with contextlib.suppress(OSError): + if os.getsid(int(entry)) == session: + groups.add(os.getpgid(int(entry))) groups -= {0, 1, os.getpgrp()} for group in groups: with contextlib.suppress(ProcessLookupError, PermissionError): os.killpg(group, signal.SIGKILL) -KERNEL_TOKEN_ENV = "RLM_KERNEL_TOKEN" - - class IPythonREPL: - _kernel_token: str | None = None + _kernel_session: int | None = None """Persistent IPython kernel communicating via Jupyter protocol.""" @@ -319,9 +310,6 @@ def __init__( self._ipc_dir = None self._lock = threading.Lock() self._interrupt_requested = threading.Event() - # Marks every process the kernel and the shell tools start, wherever it is - # reparented: restart and shutdown kill what carries it. - self._kernel_token = uuid.uuid4().hex self._scope_id: str | None = None self._recovery_attempts = 0 self._recovery_failed = False @@ -368,7 +356,6 @@ def start(self): self.kernel_env, private_dir=self._ipc_dir, ) - kernel_env[KERNEL_TOKEN_ENV] = self._kernel_token launcher = shutil.which(sys.argv[0]) or os.path.abspath(sys.argv[0]) launcher_dir = os.path.dirname(os.path.abspath(launcher)) path_entries = kernel_env.get("PATH", "").split(os.pathsep) @@ -380,6 +367,7 @@ def start(self): stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + self._kernel_session = self._km.provisioner.pid self._kc = self._km.client() self._kc.start_channels() self._kc.wait_for_ready(timeout=30) @@ -566,15 +554,11 @@ def _wait_for_idle(self, msg_id: str, timeout: float) -> bool: return True def _kill_kernel_commands(self): - """Kill the process groups the kernel's commands run in. Shell commands start - sessions of their own, so killing the kernel's process group misses them.""" - if self._kernel_token is not None: - kill_marked_groups(f"{KERNEL_TOKEN_ENV}={self._kernel_token}") - - def command_env(self, env: Mapping[str, str]) -> dict[str, str]: - """`env` marked like the kernel's own commands, for a shell command a tool runs - beside the kernel, so that restart and shutdown stop it too.""" - return {**env, KERNEL_TOKEN_ENV: self._kernel_token} + """Kill what the kernel started that is still attached to it, the way a closing + terminal does: everything in the kernel's session. The kernel leads a session + of its own, and a shell command leads a process group in it.""" + if self._kernel_session: + kill_session(self._kernel_session) def restart_kernel(self): """Restart the kernel and restore the initial REPL state.""" @@ -582,6 +566,7 @@ def restart_kernel(self): self._kc.stop_channels() self._kill_kernel_commands() self._km.restart_kernel(now=True) + self._kernel_session = self._km.provisioner.pid self._kc = self._km.client() self._kc.start_channels() self._kc.wait_for_ready(timeout=30) diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 54f29447..dfad3dfe 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import contextlib import json import os import uuid @@ -499,41 +500,63 @@ async def test_long_output_runs_to_its_end_and_reads_as_whole( assert logged[-10_000:] == whole[-10_000:] -async def test_bash_commands_stop_with_a_cancelled_turn_and_the_session( - session, tmp_path -): - """A cancelled turn stops its running bash command at once, and closing the engine - stops a background job an earlier command left behind.""" - alive = tmp_path / "alive" - loop = f"(while :; do touch {alive}; sleep 0.1; done) >/dev/null 2>&1 &" +@pytest.mark.parametrize("split", [False, True], ids=["local", "split"]) +async def test_processes_live_as_on_a_terminal(session, tmp_path, split): + """A cancelled turn stops its running bash command at once. The session's end stops + what is attached to the kernel, like a closing terminal, and leaves a daemon and a + bash command's background job running.""" + import signal + import sys + + def loop(name): + return f"echo $$ > {name}.pid; while :; do touch {name}.alive; sleep 0.1; done" + + cell = ( + "import subprocess\n" + f"subprocess.Popen(['sh', '-c', {loop('attached')!r}])\n" + f"subprocess.Popen(['sh', '-c', {loop('daemon')!r}], start_new_session=True)\n" + ) + background = f"(sh -c {loop('job')!r}) >/dev/null 2>&1 &" client = DummyClient( [ - DummyMessage(tool_calls=[DummyToolCall("bash", {"command": loop})]), + _tool(cell), + DummyMessage(tool_calls=[DummyToolCall("bash", {"command": background})]), DummyMessage(tool_calls=[DummyToolCall("bash", {"command": "sleep 60"})]), ] ) + config = _config().model_copy(update={"builtin_tools": ("ipython", "bash")}) + if split: + config = config.model_copy( + update={"execution_command": (sys.executable, "-m", "rlm.execution")} + ) engine = RLMEngine( - client=client, - session=session, - runtime_config=_config().model_copy( - update={"builtin_tools": ("ipython", "bash")} - ), - cwd=str(tmp_path), + client=client, session=session, runtime_config=config, cwd=str(tmp_path) ) + names = ("attached", "daemon", "job") try: - turn = asyncio.create_task(engine.prompt("run the commands")) - while len(client.calls) < 2: - await asyncio.sleep(0.1) + try: + turn = asyncio.create_task(engine.prompt("run the commands")) + while len(client.calls) < 3: + await asyncio.sleep(0.1) + await asyncio.sleep(0.5) + turn.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(turn, 5) + assert all((tmp_path / f"{name}.alive").exists() for name in names) + finally: + await engine.aclose() + for name in names: + (tmp_path / f"{name}.alive").unlink(missing_ok=True) await asyncio.sleep(0.5) - turn.cancel() - with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(turn, 5) - assert alive.exists() + assert not (tmp_path / "attached.alive").exists() + assert (tmp_path / "daemon.alive").exists() + assert (tmp_path / "job.alive").exists() finally: - await engine.aclose() - alive.unlink() - await asyncio.sleep(0.5) - assert not alive.exists() + for name in names: + pid = tmp_path / f"{name}.pid" + if pid.exists(): + with contextlib.suppress(ProcessLookupError): + os.kill(int(pid.read_text()), signal.SIGKILL) async def test_split_history_reaches_the_worker_past_the_frame_limit(session, tmp_path): From 3f76141b49e0311b802acb2eda979e1e5f935291 Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 1 Nov 2026 02:17:47 +0000 Subject: [PATCH 18/21] fix: give bash commands an empty stdin A command inherited its caller's stdin: from a terminal, a command reading it in its own background process group stopped on SIGTTIN and waited out its timeout, and under ACP it read the harness's own protocol stream. A command now reads end-of-file instead. Co-Authored-By: Claude Opus 5.5 --- src/rlm/tools/bash.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/rlm/tools/bash.py b/src/rlm/tools/bash.py index 90cf929a..fb49b272 100644 --- a/src/rlm/tools/bash.py +++ b/src/rlm/tools/bash.py @@ -112,6 +112,9 @@ def bash_outcome( with ( subprocess.Popen( ["bash", "-c", command], + # The caller's stdin may be its terminal, where a background group stops on + # a read, or its own protocol stream; a command reads end-of-file instead. + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd or None, From 89ad82606618429d1a28d8c256bc1bf9139c3d1a Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 1 Nov 2026 04:09:05 +0000 Subject: [PATCH 19/21] fix: leave a bash command's background job running at its timeout A job a command put in the background without redirecting its output holds the call open until the timeout, which then killed the command's process group and the job with it. Once bash itself has exited, the call now stops waiting without killing anything, so the job outlives the call. Co-Authored-By: Claude Opus 5.5 --- src/rlm/tools/bash.py | 6 ++++-- tests/test_kernel_recovery.py | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/rlm/tools/bash.py b/src/rlm/tools/bash.py index fb49b272..90537a99 100644 --- a/src/rlm/tools/bash.py +++ b/src/rlm/tools/bash.py @@ -128,7 +128,7 @@ def bash_outcome( try: while selector.get_map(): if interrupt_event is not None and interrupt_event.is_set(): - reason = "[command interrupted; process group terminated]" + reason = "[command interrupted]" break remaining = deadline - time.monotonic() if remaining <= 0: @@ -146,7 +146,9 @@ def bash_outcome( except subprocess.TimeoutExpired: reason = f"Error: command timed out after {timeout}s" finally: - if reason or proc.poll() is None: + # Once bash itself has exited, what still holds its output is a job it put in + # the background, and that job outlives the call. + if proc.poll() is None: with contextlib.suppress(ProcessLookupError): os.killpg(proc.pid, signal.SIGKILL) proc.wait() diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index dfad3dfe..14447f6f 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -504,7 +504,8 @@ async def test_long_output_runs_to_its_end_and_reads_as_whole( async def test_processes_live_as_on_a_terminal(session, tmp_path, split): """A cancelled turn stops its running bash command at once. The session's end stops what is attached to the kernel, like a closing terminal, and leaves a daemon and a - bash command's background job running.""" + bash command's background jobs running, among them one that held the command's + output until its timeout.""" import signal import sys @@ -517,14 +518,22 @@ def loop(name): f"subprocess.Popen(['sh', '-c', {loop('daemon')!r}], start_new_session=True)\n" ) background = f"(sh -c {loop('job')!r}) >/dev/null 2>&1 &" + held = f"sh -c {loop('held')!r} &" client = DummyClient( [ _tool(cell), DummyMessage(tool_calls=[DummyToolCall("bash", {"command": background})]), + DummyMessage(tool_calls=[DummyToolCall("bash", {"command": held})]), DummyMessage(tool_calls=[DummyToolCall("bash", {"command": "sleep 60"})]), ] ) - config = _config().model_copy(update={"builtin_tools": ("ipython", "bash")}) + config = _config() + config = config.model_copy( + update={ + "builtin_tools": ("ipython", "bash"), + "policy": config.policy.model_copy(update={"exec_timeout": 2}), + } + ) if split: config = config.model_copy( update={"execution_command": (sys.executable, "-m", "rlm.execution")} @@ -532,11 +541,11 @@ def loop(name): engine = RLMEngine( client=client, session=session, runtime_config=config, cwd=str(tmp_path) ) - names = ("attached", "daemon", "job") + names = ("attached", "daemon", "job", "held") try: try: turn = asyncio.create_task(engine.prompt("run the commands")) - while len(client.calls) < 3: + while len(client.calls) < 4: await asyncio.sleep(0.1) await asyncio.sleep(0.5) turn.cancel() @@ -551,6 +560,7 @@ def loop(name): assert not (tmp_path / "attached.alive").exists() assert (tmp_path / "daemon.alive").exists() assert (tmp_path / "job.alive").exists() + assert (tmp_path / "held.alive").exists() finally: for name in names: pid = tmp_path / f"{name}.pid" From 4b0f6cf8688f12800b66f3cf11e3f9d5d398786f Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 1 Nov 2026 15:47:16 +0000 Subject: [PATCH 20/21] fix: return a child's history in pages that each fit a frame child.history() went through the supervisor as one reply, which broker and execution frames cap at 16 MiB, so a long child conversation failed to read, locally too, where main read the child's ledger file whole. The supervisor now returns the ledger in pages of at most 8 MiB of JSON from a byte offset, and the handle reads them until the end. The history test reads a child's history past the frame limit, local and split, and checks a printed count: a traceback shows the cell's source, so a marker printed at the end of the cell also appeared when it failed. Co-Authored-By: Claude Opus 5.5 --- src/rlm/agent.py | 12 ++++++-- src/rlm/broker.py | 12 +++++++- src/rlm/history.py | 26 +++++++++++++++++ src/rlm/supervisor.py | 7 +++-- tests/test_kernel_recovery.py | 54 ++++++++++++++++++++++++++--------- 5 files changed, 92 insertions(+), 19 deletions(-) diff --git a/src/rlm/agent.py b/src/rlm/agent.py index 161f646d..36bf9eba 100644 --- a/src/rlm/agent.py +++ b/src/rlm/agent.py @@ -50,8 +50,16 @@ async def history(self) -> History: """Read a fresh snapshot of this agent's local conversation history.""" if not broker.is_active(): return await history(session_dir=self.session_dir) - payload = await broker.agent_request("agent.history", agent_id=self.id) - return History.from_events(self.session_dir, payload) + # The supervisor returns a long ledger in pages that each fit one frame. + events: list[dict] = [] + offset: int | None = 0 + while offset is not None: + page = await broker.agent_request( + "agent.history", agent_id=self.id, offset=offset + ) + events += page["records"] + offset = page["next"] + return History.from_events(self.session_dir, events) async def result(self, *, yield_after: float = 300) -> AgentResult: """Wait up to yield_after seconds for the child to finish, then return its latest state. diff --git a/src/rlm/broker.py b/src/rlm/broker.py index 2e2d48f0..f6d5862f 100644 --- a/src/rlm/broker.py +++ b/src/rlm/broker.py @@ -79,12 +79,21 @@ class BrokerAgentListRequest(TypedDict): class BrokerAgentHandleRequest(TypedDict): __pydantic_config__ = ConfigDict(extra="forbid", strict=True) - op: Literal["agent.info", "agent.cancel", "agent.history"] + op: Literal["agent.info", "agent.cancel"] capability: Annotated[str, Field(min_length=1)] scope_id: Annotated[str, Field(min_length=1)] agent_id: Annotated[str, Field(min_length=1)] +class BrokerAgentHistoryRequest(TypedDict): + __pydantic_config__ = ConfigDict(extra="forbid", strict=True) + op: Literal["agent.history"] + capability: Annotated[str, Field(min_length=1)] + scope_id: Annotated[str, Field(min_length=1)] + agent_id: Annotated[str, Field(min_length=1)] + offset: Annotated[int, Field(ge=0)] + + class BrokerAgentResultRequest(TypedDict): __pydantic_config__ = ConfigDict(extra="forbid", strict=True) op: Literal["agent.result"] @@ -324,6 +333,7 @@ class BrokerSkillRequest(TypedDict): | BrokerAgentGetRequest | BrokerAgentListRequest | BrokerAgentHandleRequest + | BrokerAgentHistoryRequest | BrokerAgentResultRequest | BrokerAgentWaitRequest | BrokerSkillRequest diff --git a/src/rlm/history.py b/src/rlm/history.py index 78d48b56..65c8a085 100644 --- a/src/rlm/history.py +++ b/src/rlm/history.py @@ -9,6 +9,11 @@ from pathlib import Path +HISTORY_PAGE_BYTES = 8 * 1024 * 1024 +"""JSON bytes of ledger records one page of another agent's history holds, beyond its +first record: half a frame, so a page always crosses the broker and execution wire.""" + + def read_records(path: Path) -> Iterator[dict]: with path.open("rb") as stream: for line in stream: @@ -18,6 +23,27 @@ def read_records(path: Path) -> Iterator[dict]: yield json.loads(line) +def read_page( + path: Path, offset: int, limit: int = HISTORY_PAGE_BYTES +) -> tuple[list[dict], int | None]: + """The complete records from byte `offset` that fit `limit` bytes of JSON, at least + one, and the offset of the record after them, or None at the ledger's end.""" + records: list[dict] = [] + size = 0 + with path.open("rb") as stream: + stream.seek(offset) + for line in stream: + if not line.endswith(b"\n"): + break + record = json.loads(line) + size += len(json.dumps(record)) + if records and size > limit: + return records, offset + records.append(record) + offset += len(line) + return records, None + + @dataclass class ContextWindow: index: int diff --git a/src/rlm/supervisor.py b/src/rlm/supervisor.py index b1a7e440..c8df1787 100644 --- a/src/rlm/supervisor.py +++ b/src/rlm/supervisor.py @@ -1238,9 +1238,12 @@ async def _agent_operation(self, request: dict) -> Any: return agents child = self._child(parent, request.get("name_or_id", request.get("agent_id"))) if op == "agent.history": - from rlm.history import read_records + from rlm.history import read_page - return list(read_records(child.session.dir / "messages.jsonl")) + records, offset = read_page( + child.session.dir / "messages.jsonl", request["offset"] + ) + return {"records": records, "next": offset} if op in {"agent.send", "agent.steer"}: if child.status in {"completed", "failed", "cancelled"}: raise RuntimeError("agent is no longer active") diff --git a/tests/test_kernel_recovery.py b/tests/test_kernel_recovery.py index 14447f6f..601bad8e 100644 --- a/tests/test_kernel_recovery.py +++ b/tests/test_kernel_recovery.py @@ -569,36 +569,62 @@ def loop(name): os.kill(int(pid.read_text()), signal.SIGKILL) -async def test_split_history_reaches_the_worker_past_the_frame_limit(session, tmp_path): - """The worker gets the ledger in frame-sized chunks: a prompt larger than one frame - still reaches the kernel's history whole.""" +@pytest.mark.parametrize("split", [False, True], ids=["local", "split"]) +async def test_history_past_the_frame_limit_reads_whole(session, tmp_path, split): + """A history larger than one frame reads whole: the agent's own, which reaches a + worker in frame-sized chunks, and a child's, which the supervisor returns in pages.""" import sys from rlm.execution import MAX_FRAME prompt = "x" * (MAX_FRAME + 1024) - config = _config().model_copy( - update={ - "execution_command": (sys.executable, "-m", "rlm.execution"), - "builtin_tools": ("ipython",), - } + config = _config(max_depth=1).model_copy(update={"builtin_tools": ("ipython",)}) + if split: + config = config.model_copy( + update={"execution_command": (sys.executable, "-m", "rlm.execution")} + ) + + def factory(**kwargs): + return RLMEngine(client=DummyClient([DummyMessage(content="ok")]), **kwargs) + + supervisor = SessionTreeSupervisor( + root_session=session, + runtime_config=config, + cwd=str(tmp_path), + engine_factory=factory, ) client = DummyClient( [ _tool( - "from rlm import history\n" - "h = await history()\n" - f"assert len(h.user_messages()[0]['content']) == {len(prompt)}\n" - "print('SYNCED')" + f""" +import json +from rlm import history +assert len((await history()).user_messages()[0]['content']) == {len(prompt)} +child = await rlm.agent.spawn('answer') +await child.wait(timeout=30) +# A long child conversation: five records of a quarter frame each. +with (child.session_dir / 'messages.jsonl').open('a') as stream: + for _ in range(5): + stream.write(json.dumps({{'type': 'padding', 'text': 'y' * {MAX_FRAME // 4}}}) + '\\n') +h = await child.history() +assert 'answer' in h.user_messages()[0]['content'] +print(f"PADDED {{sum(event['type'] == 'padding' for event in h.events)}}") +""" ), DummyMessage(content="done"), ] ) engine = RLMEngine( - client=client, session=session, runtime_config=config, cwd=str(tmp_path) + client=client, + session=session, + runtime_config=config, + cwd=str(tmp_path), + supervisor=supervisor, ) try: await engine.prompt(prompt) finally: await engine.aclose() - assert "SYNCED" in client.calls[1]["messages"][-1]["content"] + tool = [m for m in client.calls[1]["messages"] if m["role"] == "tool"][-1] + # Printed, not asserted in the cell: a traceback would show the cell's source. + assert "PADDED 5" in tool["content"], tool["content"][-2000:] From 220db7dd7276840bf2b6e61376c6c8172ce03fee Mon Sep 17 00:00:00 2001 From: hallerite Date: Sun, 1 Nov 2026 15:54:41 +0000 Subject: [PATCH 21/21] docs: draw split execution, and say what runs in the workspace Two figures, light and dark: where the loop, credentials and tools run in colocated and split execution, and what one agent step sends over the wire. The README's split section said the supervisor's shell jobs, services and watches return errors in this mode; they run beside the kernels. It also gives the per-call cost and what a closing session stops. Co-Authored-By: Claude Opus 5.5 --- README.md | 38 +++++++++++++++++------ docs/images/split-execution-dark.svg | 1 + docs/images/split-execution-step-dark.svg | 1 + docs/images/split-execution-step.svg | 1 + docs/images/split-execution.svg | 1 + 5 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 docs/images/split-execution-dark.svg create mode 100644 docs/images/split-execution-step-dark.svg create mode 100644 docs/images/split-execution-step.svg create mode 100644 docs/images/split-execution.svg diff --git a/README.md b/README.md index 39da2167..37bc1cda 100644 --- a/README.md +++ b/README.md @@ -553,6 +553,11 @@ Activity subscriptions become `completed` after their target permanently termina ### Separate tool execution + + + Colocated, the agent loop, model credentials and the agent's tools share one machine. Split, the harness keeps the loop, credentials, supervisor and ledger, and a worker in the workspace runs the tools over one authenticated connection. + + The ACP runtime metadata accepts `execution_command` (a trusted argv launching `python -m rlm.execution`) and `execution_cwd` (the task workspace). For example, an orchestrator can start the worker with `docker exec -i @@ -562,8 +567,9 @@ version installed. The orchestrator owns container provisioning and teardown. The engine, model credentials, supervisor, and authoritative conversation logs remain in the harness process. Native tools, persistent IPython kernels, and builtin skills run in the worker environment. Recursive agents inherit the execution destination, -keep separate kernels, and share the task workspace. Communication and recursive -callbacks use worker stdio, so a task container can have networking disabled. +keep separate kernels, and share the task workspace. With the stdio transport, +communication and recursive callbacks use the worker's stdio, so a task container +can have networking disabled. The command itself defines the isolation boundary; launching a local Python worker alone provides process separation, not filesystem isolation. @@ -583,10 +589,24 @@ The token authenticates connections; this transport does not encrypt traffic, so use a private network or an encrypted tunnel. The worker removes its token from the environment before starting tools. The orchestrator owns listener teardown. -Supervisor shell jobs -(`rlm.shell`), services (`rlm.services`) and filesystem watchers -(`rlm.watch.path`) return explicit errors in this mode. `await child.history()` uses the supervisor; the current agent's local -history is a snapshot refreshed before each cell. Worker frames, including history -snapshots and outputs, are limited to 16 MiB. Kernel recovery preserves the existing -no-replay behavior. Worker disconnects fail the session; the orchestrator must stop -remaining task processes when the rollout ends. +Supervisor shell jobs (`rlm.shell`), services (`rlm.services`) and path watches +(`rlm.watch.path`) run in the worker's environment, beside the kernels; the +supervisor keeps their records and inbox events, so the agent gets the same +runtime prompt and API as in local execution. The current agent's history is a +local snapshot refreshed before each cell; `await child.history()` reads a child's +ledger from the supervisor in pages. Frames are limited to 16 MiB: a command or +cell keeps the first and last 512 KiB of its output, and a long ledger syncs in +chunks. Kernel recovery preserves the existing no-replay behavior. Worker +disconnects fail the session. + + + + One agent step in split mode: the harness sends the tool call with the new ledger records, the cell runs in the workspace and calls back for rlm.* operations, and the result returns to the harness. A tool call costs one round trip, each rlm.* call one more. + + +Each tool call costs one round trip between harness and worker, each `rlm.*` +call one more, and `rlm.shell.run` two. + +A session that closes stops what is still attached to its kernels, as a closing +terminal does; daemons and bash background jobs keep running, as they do in local +execution, so the orchestrator stops what remains when the rollout ends. diff --git a/docs/images/split-execution-dark.svg b/docs/images/split-execution-dark.svg new file mode 100644 index 00000000..64e42bc0 --- /dev/null +++ b/docs/images/split-execution-dark.svg @@ -0,0 +1 @@ +Colocatedthe agent's commands run beside the harnessModel APItask VMharnessagent loop · model credentialssupervisor · conversation ledgernano-rlm installagent's toolsIPython kernel · bash · edit · fetchshell jobs · services · watchestask filesone machine, one filesystem, one process treeA command can read the API key, kill orstarve the loop, or edit the ledger.Splitthe harness keeps the loop; the workspace runs the toolsModel APIharness VMagent loopthe model's turnsmodel credentialsnever leave this VMsupervisoragents · inbox · job recordsconversation ledgerauthoritative copyits own image and installworkspace VM · task imagepython -m rlm.executionIPython kernelsone per agent, persistentbash · edit · fetchas the agent userjobs · services · watchesrun here, recorded theretask files · no credentialsgrading runs heretool callresultrlm.* callreplyjob eventsledger recordsone TCP connectiontoken-authenticatedprivate network(or a worker's stdio) diff --git a/docs/images/split-execution-step-dark.svg b/docs/images/split-execution-step-dark.svg new file mode 100644 index 00000000..010b542b --- /dev/null +++ b/docs/images/split-execution-step-dark.svg @@ -0,0 +1 @@ +Model APIharness VMengine · supervisorworkspace VMworker · IPython kernelconversationtool call ipython(code)tool call + new ledger recordscell runsawait rlm.agent.spawn(...)child handleresult: first and last 512 KiB + whole sizenext turn shows its truncation1 round tripper tool call+1 round tripper rlm.* call diff --git a/docs/images/split-execution-step.svg b/docs/images/split-execution-step.svg new file mode 100644 index 00000000..719ca216 --- /dev/null +++ b/docs/images/split-execution-step.svg @@ -0,0 +1 @@ +Model APIharness VMengine · supervisorworkspace VMworker · IPython kernelconversationtool call ipython(code)tool call + new ledger recordscell runsawait rlm.agent.spawn(...)child handleresult: first and last 512 KiB + whole sizenext turn shows its truncation1 round tripper tool call+1 round tripper rlm.* call diff --git a/docs/images/split-execution.svg b/docs/images/split-execution.svg new file mode 100644 index 00000000..b815d8aa --- /dev/null +++ b/docs/images/split-execution.svg @@ -0,0 +1 @@ +Colocatedthe agent's commands run beside the harnessModel APItask VMharnessagent loop · model credentialssupervisor · conversation ledgernano-rlm installagent's toolsIPython kernel · bash · edit · fetchshell jobs · services · watchestask filesone machine, one filesystem, one process treeA command can read the API key, kill orstarve the loop, or edit the ledger.Splitthe harness keeps the loop; the workspace runs the toolsModel APIharness VMagent loopthe model's turnsmodel credentialsnever leave this VMsupervisoragents · inbox · job recordsconversation ledgerauthoritative copyits own image and installworkspace VM · task imagepython -m rlm.executionIPython kernelsone per agent, persistentbash · edit · fetchas the agent userjobs · services · watchesrun here, recorded theretask files · no credentialsgrading runs heretool callresultrlm.* callreplyjob eventsledger recordsone TCP connectiontoken-authenticatedprivate network(or a worker's stdio)