From 05bcbe59b5ffb397861b1abbd77bae8dcda5393d Mon Sep 17 00:00:00 2001 From: Fares Obeid Date: Mon, 13 Jul 2026 18:14:44 +0000 Subject: [PATCH 1/5] feat(default): auto-compaction via summarize_at_tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds rlm-parity context compaction to the default (bash+edit) harness, matching rlm's engine semantics exactly: trigger on usage.prompt_tokens >= threshold; summary requested with rlm's CHECKPOINT_COMPACTION_PROMPT while tools stay in the request under tool_choice="none" (the prompt renders like a regular turn); the rebuilt list is the system message plus the summary wrapped in rlm's POST_COMPACTION_FRAMING — the original task prompt is dropped and the summary carries the goal. Config mirrors rlm's summarize_at_tokens: fixed int or (lo, hi) per-task draw seeded by task index. Validated in production RL runs (GLM-4.5-Air on scaleswe-v1, 131k contexts). Co-Authored-By: Claude Fable 5 --- verifiers/v1/harnesses/default/harness.py | 41 +++++++++++++++++ verifiers/v1/harnesses/default/program.py | 55 ++++++++++++++++++++--- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/verifiers/v1/harnesses/default/harness.py b/verifiers/v1/harnesses/default/harness.py index 4ace368dfd..d648a7c990 100644 --- a/verifiers/v1/harnesses/default/harness.py +++ b/verifiers/v1/harnesses/default/harness.py @@ -1,7 +1,10 @@ import json import os +import random from pathlib import Path +from pydantic import model_validator + from verifiers.v1.harness import Harness, HarnessConfig from verifiers.v1.clients import ModelContext from verifiers.v1.dialects.chat import message_to_wire @@ -35,6 +38,30 @@ class DefaultHarnessConfig(HarnessConfig): eval environment; the key is handed to the program over argv (like the interception secret) so the agent's `bash` subprocesses don't inherit it.""" + summarize_at_tokens: int | tuple[int, int] | None = None + """Auto-compaction threshold: once the context grows past this many tokens, the program asks + the model to summarize its progress and restarts the message list from the initial prompt plus + that summary. An int is a fixed threshold; a `(lo, hi)` pair draws a per-group threshold + (seeded by the task index, so a task's rollouts share one draw and tasks vary). `None` + disables auto-compaction; ints must be positive.""" + + @model_validator(mode="after") + def validate_limits(self) -> "DefaultHarnessConfig": + value = self.summarize_at_tokens + if isinstance(value, tuple): + lo, hi = value + if lo <= 0 or hi <= 0: + raise ValueError("`summarize_at_tokens` range bounds must be positive.") + if lo > hi: + raise ValueError( + "`summarize_at_tokens` range must be (lo, hi) with lo <= hi." + ) + elif value is not None and value <= 0: + raise ValueError( + "`summarize_at_tokens` must be positive, or None to disable." + ) + return self + class DefaultHarness(Harness[DefaultHarnessConfig]): APPENDS_SYSTEM_PROMPT = True @@ -45,6 +72,17 @@ class DefaultHarness(Harness[DefaultHarnessConfig]): async def setup(self, runtime: Runtime) -> None: await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env) + def summarize_threshold(self, task_idx: int) -> int: + """The resolved auto-compaction threshold: a range draws per-group (seeded by task index, + so a task's rollouts share one threshold). 0 when disabled.""" + value = self.config.summarize_at_tokens + if value is None: + return 0 + if isinstance(value, tuple): + lo, hi = value + return random.Random(task_idx).randint(lo, hi) + return value + async def launch( self, ctx: ModelContext, @@ -72,6 +110,9 @@ async def launch( ] if self.config.edit: args.append("--edit") + threshold = self.summarize_threshold(trace.task.idx) + if threshold: + args.append(f"--summarize-at-tokens={threshold}") if self.config.search: # Resolve the key and keep it OUT of the program env: it's handed to the program over # argv (--serper-key), so popping it here stops the agent's `bash` subprocesses from diff --git a/verifiers/v1/harnesses/default/program.py b/verifiers/v1/harnesses/default/program.py index a76e4aa67e..e7ac55c6d0 100644 --- a/verifiers/v1/harnesses/default/program.py +++ b/verifiers/v1/harnesses/default/program.py @@ -17,6 +17,28 @@ SERPER_URL = "https://google.serper.dev/search" +# rlm-parity compaction prompts (mirrors rlm's engine.py). +CHECKPOINT_COMPACTION_PROMPT = ( + "You are performing a CONTEXT CHECKPOINT COMPACTION. " + "Create a handoff summary for another LLM that will resume the task.\n" + "\n" + "Include:\n" + "- Current progress and key decisions made\n" + "- Important context, constraints, or user preferences\n" + "- What remains to be done (clear next steps)\n" + "- Any critical data, examples, or references needed to continue\n" + "\n" + "Be concise, structured, and focused on helping the next LLM " + "seamlessly continue the work." +) +POST_COMPACTION_FRAMING = ( + "Another language model started to solve this problem and produced " + "a summary of its thinking process. Use this to build on the work " + "that has already been done and avoid duplicating work. Here is " + "the summary produced by the other language model, use the " + "information in this summary to assist with your own analysis:" +) + BASH_TOOL = { "type": "function", "function": { @@ -170,12 +192,20 @@ def run_edit(path: str, old_str: str, new_str: str) -> str: async def chat( - client: AsyncOpenAI, model: str, messages: list[dict], tools: list[dict] + client: AsyncOpenAI, + model: str, + messages: list[dict], + tools: list[dict], + tool_choice: str | None = None, ): - completion = await client.chat.completions.create( - model=model, messages=messages, tools=tools or None - ) - return completion.choices[0].message + """One completion; returns (message, prompt tokens) — `usage.prompt_tokens` is the + context size this turn conditioned on (rlm's compaction trigger).""" + kwargs = {"model": model, "messages": messages, "tools": tools or None} + if tools and tool_choice is not None: + kwargs["tool_choice"] = tool_choice + completion = await client.chat.completions.create(**kwargs) + usage = completion.usage + return completion.choices[0].message, usage.prompt_tokens if usage else 0 async def connect_mcp(stack: AsyncExitStack, config: dict) -> tuple[list[dict], dict]: @@ -248,6 +278,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--edit", action="store_true") parser.add_argument("--search", action="store_true") parser.add_argument("--serper-key", default="") + parser.add_argument("--summarize-at-tokens", type=int, default=0) return parser.parse_args() @@ -284,8 +315,11 @@ async def main() -> None: messages.extend(initial) elif args.prompt: messages.append({"role": "user", "content": args.prompt}) + # Compaction keeps only the system message (rlm semantics: the original task + # prompt is dropped; the summary carries the goal). + system_messages = [m for m in messages if m.get("role") == "system"] while True: - message = await chat(client, args.model, messages, tools) + message, context_tokens = await chat(client, args.model, messages, tools) messages.append(message.model_dump(exclude_none=True)) if not message.tool_calls: break @@ -338,6 +372,15 @@ async def main() -> None: messages.append( {"role": "tool", "tool_call_id": call.id, "content": content} ) + # Compact after the turn's tool results land, so the summary sees them (rlm + # semantics: tools stay in the request with tool_choice="none" so the prompt + # renders like a regular turn; tool calls in the reply are ignored; the rebuilt + # list is system + framed summary, dropping the original prompt). + if args.summarize_at_tokens and context_tokens >= args.summarize_at_tokens: + messages.append({"role": "user", "content": CHECKPOINT_COMPACTION_PROMPT}) + summary, _ = await chat(client, args.model, messages, tools, tool_choice="none") + framed = POST_COMPACTION_FRAMING + "\n\n" + (summary.content or "") + messages = system_messages + [{"role": "user", "content": framed}] if __name__ == "__main__": From 2bcc6af90768714437535420d7d0f9d6965e8d63 Mon Sep 17 00:00:00 2001 From: Fares Obeid Date: Mon, 13 Jul 2026 18:14:44 +0000 Subject: [PATCH 2/5] fix(runtimes): user-scope the rate-limiter directory /tmp/vf-rate-limiters is a fixed path; on multi-user hosts another user's bucket file makes every tunnel fail with Permission denied (observed killing 100% of container-runtime rollouts on a shared SLURM node). Suffix the directory with the username. Co-Authored-By: Claude Fable 5 --- verifiers/v1/runtimes/limiters.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/runtimes/limiters.py b/verifiers/v1/runtimes/limiters.py index 3a97100bae..116bc2a806 100644 --- a/verifiers/v1/runtimes/limiters.py +++ b/verifiers/v1/runtimes/limiters.py @@ -9,11 +9,14 @@ import asyncio import fcntl +import getpass import os import tempfile import time -_LIMITER_DIR = os.path.join(tempfile.gettempdir(), "vf-rate-limiters") +# User-scoped: a fixed shared path breaks on multi-user hosts (another user's bucket +# file -> Permission denied on every tunnel). +_LIMITER_DIR = os.path.join(tempfile.gettempdir(), f"vf-rate-limiters-{getpass.getuser()}") class CreationLimiter: From 1f6ca914c8b0d383d10d7a3c981ef7cd1a8c1cdd Mon Sep 17 00:00:00 2001 From: Fares Obeid Date: Tue, 14 Jul 2026 00:44:41 +0000 Subject: [PATCH 3/5] harden getuser() for passwd-less container UIDs (review) Co-Authored-By: Claude Fable 5 --- verifiers/v1/runtimes/limiters.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/runtimes/limiters.py b/verifiers/v1/runtimes/limiters.py index 116bc2a806..251903ebab 100644 --- a/verifiers/v1/runtimes/limiters.py +++ b/verifiers/v1/runtimes/limiters.py @@ -15,8 +15,13 @@ import time # User-scoped: a fixed shared path breaks on multi-user hosts (another user's bucket -# file -> Permission denied on every tunnel). -_LIMITER_DIR = os.path.join(tempfile.gettempdir(), f"vf-rate-limiters-{getpass.getuser()}") +# file -> Permission denied on every tunnel). getuser() can raise in containers running +# under an arbitrary UID with no passwd entry — fall back to the numeric UID. +try: + _user = getpass.getuser() +except Exception: + _user = str(os.getuid()) +_LIMITER_DIR = os.path.join(tempfile.gettempdir(), f"vf-rate-limiters-{_user}") class CreationLimiter: From a40342722fea6bc4993d72a5da3a0042d5358851 Mon Sep 17 00:00:00 2001 From: fares Date: Tue, 14 Jul 2026 14:47:42 +0000 Subject: [PATCH 4/5] revert: drop user-scoped rate-limiter directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts 2bcc6af90 and 1f6ca914c — keep this PR scoped to auto-compaction. Co-Authored-By: Claude Fable 5 --- verifiers/v1/runtimes/limiters.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/verifiers/v1/runtimes/limiters.py b/verifiers/v1/runtimes/limiters.py index 251903ebab..3a97100bae 100644 --- a/verifiers/v1/runtimes/limiters.py +++ b/verifiers/v1/runtimes/limiters.py @@ -9,19 +9,11 @@ import asyncio import fcntl -import getpass import os import tempfile import time -# User-scoped: a fixed shared path breaks on multi-user hosts (another user's bucket -# file -> Permission denied on every tunnel). getuser() can raise in containers running -# under an arbitrary UID with no passwd entry — fall back to the numeric UID. -try: - _user = getpass.getuser() -except Exception: - _user = str(os.getuid()) -_LIMITER_DIR = os.path.join(tempfile.gettempdir(), f"vf-rate-limiters-{_user}") +_LIMITER_DIR = os.path.join(tempfile.gettempdir(), "vf-rate-limiters") class CreationLimiter: From 85f56fbec18b6323ce492e04508c6d856c12626e Mon Sep 17 00:00:00 2001 From: fares Date: Tue, 14 Jul 2026 14:55:37 +0000 Subject: [PATCH 5/5] style: ruff format Co-Authored-By: Claude Fable 5 --- verifiers/v1/harnesses/default/program.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/harnesses/default/program.py b/verifiers/v1/harnesses/default/program.py index e7ac55c6d0..4188147b8d 100644 --- a/verifiers/v1/harnesses/default/program.py +++ b/verifiers/v1/harnesses/default/program.py @@ -377,8 +377,12 @@ async def main() -> None: # renders like a regular turn; tool calls in the reply are ignored; the rebuilt # list is system + framed summary, dropping the original prompt). if args.summarize_at_tokens and context_tokens >= args.summarize_at_tokens: - messages.append({"role": "user", "content": CHECKPOINT_COMPACTION_PROMPT}) - summary, _ = await chat(client, args.model, messages, tools, tool_choice="none") + messages.append( + {"role": "user", "content": CHECKPOINT_COMPACTION_PROMPT} + ) + summary, _ = await chat( + client, args.model, messages, tools, tool_choice="none" + ) framed = POST_COMPACTION_FRAMING + "\n\n" + (summary.content or "") messages = system_messages + [{"role": "user", "content": framed}]