From ab28ece43ac7a724b2c60afb35284217c5b07611 Mon Sep 17 00:00:00 2001 From: rasdani <73563550+rasdani@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:49:16 +0000 Subject: [PATCH] feat(rlm): drop network-dependent prompt guidance on boxes with no network Add RLMHarnessConfig.network_guidance, passed to nano-rlm's execution policy like allow_git. When unset, the harness derives it from the rollout's effective network policy (the runtime config, which already carries the task's network rules): boxes that allow no destinations get network_guidance=false, so nano-rlm stops suggesting `uv pip install` and online lookups; any other box sends nothing and keeps nano-rlm's default. Bump the default nano-rlm ref to the commit that adds the policy field. Co-Authored-By: Claude Opus 5.5 --- tests/unit/harnesses/test_rlm.py | 56 ++++++++++++++++++++++++--- verifiers/v1/harnesses/rlm/harness.py | 16 +++++++- 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/tests/unit/harnesses/test_rlm.py b/tests/unit/harnesses/test_rlm.py index d549d0a24b..9366941aaf 100644 --- a/tests/unit/harnesses/test_rlm.py +++ b/tests/unit/harnesses/test_rlm.py @@ -1,15 +1,26 @@ -"""The rlm harness maps its session's stop reason onto the trace's stop condition.""" +"""The rlm harness's runtime contract and stop-reason mapping.""" + +from types import SimpleNamespace import pytest import verifiers.v1 as vf +from verifiers.v1.configs.runtime import DockerConfig, NetworkPolicyConfig from verifiers.v1.harnesses.rlm.harness import ( + RLM_RUNTIME_METADATA_KEY, RLM_SESSION_METADATA_KEY, RLMHarness, RLMHarnessConfig, ) +def _trace() -> vf.Trace: + return vf.Trace( + agent=vf.AgentInfo(config=vf.AgentConfig()), + task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="q")), + ) + + @pytest.mark.parametrize( "reason,condition", [ @@ -22,10 +33,7 @@ ], ) def test_rlm_stop_reason_survives_completion_and_serialization(reason, condition): - trace = vf.Trace( - agent=vf.AgentInfo(config=vf.AgentConfig()), - task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="q")), - ) + trace = _trace() harness = RLMHarness(RLMHarnessConfig(id="rlm")) metadata = { RLM_SESSION_METADATA_KEY: { @@ -43,3 +51,41 @@ def test_rlm_stop_reason_survives_completion_and_serialization(reason, condition restored = vf.WireTrace.model_validate_json(trace.model_dump_json()) assert restored.stop_condition == condition assert restored.is_truncated is (condition != "agent_completed") + + +def _sent_policy(config: RLMHarnessConfig, runtime_config) -> dict: + runtime = SimpleNamespace(config=runtime_config, env={}) + metadata = RLMHarness(config)._runtime_metadata( + SimpleNamespace(model="m"), _trace(), runtime, "http://endpoint", "s", None + ) + return metadata[RLM_RUNTIME_METADATA_KEY]["policy"] + + +@pytest.mark.parametrize( + "runtime_config", + [ + DockerConfig(allow=[]), + DockerConfig().with_task_network_policy(NetworkPolicyConfig(allow=[])), + ], +) +def test_rlm_drops_network_guidance_for_boxes_with_no_network(runtime_config): + policy = _sent_policy(RLMHarnessConfig(id="rlm"), runtime_config) + assert policy["network_guidance"] is False + + +@pytest.mark.parametrize( + "runtime_config", + [DockerConfig(), DockerConfig(allow=["pypi.org"]), SimpleNamespace(type="local")], +) +def test_rlm_leaves_network_guidance_to_nano_rlm_when_network_reachable( + runtime_config, +): + policy = _sent_policy(RLMHarnessConfig(id="rlm"), runtime_config) + assert "network_guidance" not in policy + + +@pytest.mark.parametrize("value", [True, False]) +def test_rlm_explicit_network_guidance_overrides_runtime_policy(value): + config = RLMHarnessConfig(id="rlm", network_guidance=value) + for runtime_config in (DockerConfig(allow=[]), DockerConfig()): + assert _sent_policy(config, runtime_config)["network_guidance"] is value diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index f124963a62..a9fdf6e813 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -70,7 +70,7 @@ class CompactionConfig(BaseConfig): class RLMHarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" version: str = Field( - default="e4d4407f1fe6f4bd82b2c1da26ccd737627a6543", min_length=1 + default="15513ae8ea3ad007a3d940b30252cb14de5e2caf", min_length=1 ) """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every field this harness puts on the wire, i.e. be at least the default ref.""" @@ -104,6 +104,10 @@ class RLMHarnessConfig(HarnessConfig): """Allow unrestricted Git history queries (e.g. `git log --all`); `None` uses nano-rlm's default (false). Ordinary Git commands remain allowed either way; this is not filesystem or process isolation.""" + network_guidance: bool | None = None + """Keep nano-rlm's prompt guidance that assumes outbound network access (installing + extra kernel packages, the online-solutions warning). `None` turns it off when the + box allows no network destinations and otherwise uses nano-rlm's default (on).""" max_total_turns: PositiveInt | None = None """Tree-total turn budget (one turn = one work-loop model call, any engine); every engine stops before its next call once spent. `None` = uncapped.""" @@ -197,6 +201,7 @@ def _runtime_metadata( "delegation_prompt": self.config.delegation_prompt, "exec_timeout": self.config.exec_timeout, "allow_git": self.config.allow_git, + "network_guidance": self._network_guidance(runtime), "max_total_turns": self.config.max_total_turns, "max_total_tokens": self.config.max_total_tokens or None, "max_tool_output_bytes": self.config.max_tool_output_bytes, @@ -251,6 +256,15 @@ def _runtime_metadata( payload["builtin_tools"] = list(self.config.builtin_tools) return {RLM_RUNTIME_METADATA_KEY: payload} + def _network_guidance(self, runtime: Runtime) -> bool | None: + if self.config.network_guidance is not None: + return self.config.network_guidance + # The rollout's runtime config already carries the task's network policy. + policy = runtime.config + if isinstance(policy, NetworkPolicyConfig) and not policy.allow: + return False + return None + async def prepare_acp( self, ctx: ModelContext,