From 998cfe32cba724cb8b2119af3cf8a53e3c89caaf Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 2 Sep 2026 09:19:57 +0000 Subject: [PATCH 1/3] refactor(v1): one install-under-lock helper for the harnesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every third-party harness installed its program the same way — mkdir, take a lock, run "[ ready ] || (install)", raise with stderr[-500:] on failure — with three different locks (flock||lockf, a hand-rolled symlink spinlock, bare flock), nine copies of the pinned `version` field, and six copies of an rm -rf cleanup. - harnesses/utils/install.py: ensure_installed(directory, install, env, label, ready=None, lock=None, shell=("sh", "-c")) and remove_dir() - configs/harness.py: PinnedVersion, the constrained version type the nine harness configs now use for their `version` default - claude_code, codex, kimi_code, pool, openclaw, prime_agent, pi, rlm and node.ensure_node install through the helper; claude_code, codex, openclaw, prime_agent and hermes_agent clean up through it Intended behaviour changes, all in the lock/error path: pi and the shared Node installer use flock||lockf instead of the symlink spinlock (flock releases on holder death, which is what the spinlock's dead-owner check emulated); rlm gains the lockf fallback; install failures report stdout when stderr is empty; cleanup failures share one message shape. Install scripts, ready checks, environments and shells are unchanged (verified by replaying every setup()/cleanup() against a recording runtime on both branches). Co-Authored-By: Claude Fable 5.1 --- verifiers/v1/configs/harness.py | 4 ++ verifiers/v1/harnesses/claude_code/harness.py | 36 +++++---------- verifiers/v1/harnesses/codex/harness.py | 34 +++++--------- .../v1/harnesses/hermes_agent/harness.py | 13 ++---- verifiers/v1/harnesses/kimi_code/harness.py | 22 ++++----- .../v1/harnesses/mini_swe_agent/harness.py | 6 +-- verifiers/v1/harnesses/node.py | 23 ++++------ verifiers/v1/harnesses/openclaw/harness.py | 44 ++++++------------ verifiers/v1/harnesses/pi/harness.py | 29 ++++-------- verifiers/v1/harnesses/pool/harness.py | 25 +++++------ verifiers/v1/harnesses/prime_agent/harness.py | 26 ++++------- verifiers/v1/harnesses/rlm/harness.py | 16 ++++--- verifiers/v1/harnesses/terminus_2/harness.py | 6 +-- verifiers/v1/harnesses/utils/install.py | 45 +++++++++++++++++++ 14 files changed, 148 insertions(+), 181 deletions(-) create mode 100644 verifiers/v1/harnesses/utils/install.py diff --git a/verifiers/v1/configs/harness.py b/verifiers/v1/configs/harness.py index 068a8169f1..3c6a7d9879 100644 --- a/verifiers/v1/configs/harness.py +++ b/verifiers/v1/configs/harness.py @@ -4,12 +4,16 @@ import os from pathlib import Path +from typing import Annotated from pydantic import ConfigDict, Field, FiniteFloat from pydantic_config import BaseConfig from verifiers.v1.types import ID +PinnedVersion = Annotated[str, Field(pattern=r"^[A-Za-z0-9._+-]+$")] +"""A release/tag a harness pins its program install to.""" + class HarnessConfig(BaseConfig): id: ID = "bash" diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index bba53b297a..3c31679eef 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -1,13 +1,10 @@ """Run Claude Code through the Claude Agent SDK ACP adapter.""" -import shlex - -from pydantic import Field - from verifiers.v1.acp import ACPConfig, ACPHarness from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node +from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -32,7 +29,7 @@ class ClaudeCodeHarnessConfig(HarnessConfig): - version: str = Field(default="2.1.232", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "2.1.232" """Claude Code release to install, pinned for reproducibility.""" @@ -51,25 +48,18 @@ async def setup(self, runtime: Runtime) -> None: acp_bin = ACP_BIN.format(**versions) ready = f"{directory}/.ready" script = ACP_INSTALL.replace("{packages}", packages).replace("{ready}", ready) - ensure = shlex.quote( - f"[ -f {ready} ] && [ -x {claude_bin} ] && [ -x {acp_bin} ] || ({script})" - ) - acp_guarded = ( - f"mkdir -p {directory} && " - f'"$(command -v flock || command -v lockf)" {directory}/install.lock ' - f"sh -c {ensure}" - ) - acp_result = await runtime.run( - ["sh", "-c", acp_guarded], - { + await ensure_installed( + runtime, + directory=directory, + ready=f"[ -f {ready} ] && [ -x {claude_bin} ] && [ -x {acp_bin} ]", + install=script, + env={ **self.config.resolved_env, "VF_CLAUDE_CODE_VERSION": self.config.version, "VF_CLAUDE_ACP_VERSION": ACP_VERSION, }, + label="Claude Agent ACP", ) - if acp_result.exit_code != 0: - detail = (acp_result.stderr or acp_result.stdout).strip()[-500:] - raise RuntimeError(f"Claude Agent ACP install failed: {detail}") await super().setup(runtime) async def prepare_acp( @@ -113,11 +103,7 @@ async def prepare_acp( ) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: - result = await runtime.run(["rm", "-rf", self.config_dir(trace)], {}) - if result.exit_code != 0: - raise RuntimeError( - f"failed to clean up Claude config: {result.stderr.strip()[-500:]}" - ) + await remove_dir(runtime, self.config_dir(trace), "Claude config") @staticmethod def config_dir(trace: Trace) -> str: diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 9690a01cbb..6baee9cdc8 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -4,15 +4,13 @@ import json import logging import re -import shlex from collections import Counter -from pydantic import Field - from verifiers.v1.acp import ACPConfig, ACPHarness from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node +from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -38,7 +36,7 @@ class CodexHarnessConfig(HarnessConfig): - version: str = Field(default="0.147.0", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "0.147.0" """Codex release to install, pinned for reproducibility.""" multi_agent: bool = False """Enable Codex's native multi-agent v2 tools.""" @@ -64,24 +62,18 @@ async def setup(self, runtime: Runtime) -> None: acp_bin = ACP_BIN.format(**versions) ready = f"{directory}/.ready" script = INSTALL.replace("{packages}", packages).replace("{ready}", ready) - ensure = shlex.quote( - f"[ -f {ready} ] && [ -x {codex_bin} ] && [ -x {acp_bin} ] || ({script})" - ) - guarded = ( - f"mkdir -p {directory} && " - f'"$(command -v flock || command -v lockf)" {directory}/install.lock ' - f"sh -c {ensure}" - ) - install = await runtime.run( - ["sh", "-c", guarded], - { + await ensure_installed( + runtime, + directory=directory, + ready=f"[ -f {ready} ] && [ -x {codex_bin} ] && [ -x {acp_bin} ]", + install=script, + env={ **self.config.resolved_env, "VF_CODEX_VERSION": self.config.version, "VF_CODEX_ACP_VERSION": ACP_VERSION, }, + label="codex", ) - if install.exit_code != 0: - raise RuntimeError(f"codex install failed: {install.stderr.strip()[-500:]}") await super().setup(runtime) async def prepare_acp( @@ -112,11 +104,7 @@ async def prepare_acp( ) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: - result = await runtime.run(["rm", "-rf", self.trace_home(trace)], {}) - if result.exit_code != 0: - raise RuntimeError( - f"failed to clean up Codex home: {result.stderr.strip()[-500:]}" - ) + await remove_dir(runtime, self.trace_home(trace), "Codex home") @staticmethod def trace_home(trace: Trace) -> str: diff --git a/verifiers/v1/harnesses/hermes_agent/harness.py b/verifiers/v1/harnesses/hermes_agent/harness.py index 297c6ca261..a335bc0a02 100644 --- a/verifiers/v1/harnesses/hermes_agent/harness.py +++ b/verifiers/v1/harnesses/hermes_agent/harness.py @@ -3,11 +3,10 @@ import json from pathlib import Path -from pydantic import Field - from verifiers.v1.acp import ACPConfig, ACPHarness from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion +from verifiers.v1.harnesses.utils.install import remove_dir from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -16,7 +15,7 @@ class HermesAgentHarnessConfig(HarnessConfig): - version: str = Field(default="0.19.0", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "0.19.0" """Hermes Agent release to install, pinned for reproducibility.""" use_bundled_skill: bool = False """Enable Hermes Agent's bundled skill catalog in addition to uploaded skills.""" @@ -96,8 +95,4 @@ async def prepare_acp( ) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: - result = await runtime.run(["rm", "-rf", f"/tmp/vf-hermes/{trace.id}"], {}) - if result.exit_code: - raise RuntimeError( - f"failed to clean up Hermes home: {result.stderr.strip()[-500:]}" - ) + await remove_dir(runtime, f"/tmp/vf-hermes/{trace.id}", "Hermes home") diff --git a/verifiers/v1/harnesses/kimi_code/harness.py b/verifiers/v1/harnesses/kimi_code/harness.py index c3d6b0140b..603a06a1e3 100644 --- a/verifiers/v1/harnesses/kimi_code/harness.py +++ b/verifiers/v1/harnesses/kimi_code/harness.py @@ -1,15 +1,14 @@ """Run Kimi Code's native ACP server against interception.""" import logging -import shlex from typing import Literal import tomli_w -from pydantic import Field from verifiers.v1.acp import ACPConfig, ACPHarness from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion +from verifiers.v1.harnesses.utils.install import ensure_installed from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -39,7 +38,7 @@ class KimiCodeHarnessConfig(HarnessConfig): - version: str = Field(default="0.36.0", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "0.36.0" """Kimi Code release to install, pinned for reproducibility.""" transport: Literal["chat_completions", "responses", "anthropic_messages"] = ( "chat_completions" @@ -58,16 +57,13 @@ async def setup(self, runtime: Runtime) -> None: "kimi-code: ensuring Kimi Code %s is installed", self.config.version ) script = INSTALL.replace("{version}", self.config.version) - guarded = ( - "mkdir -p /tmp/vf-kimi-code && " - '"$(command -v flock || command -v lockf)" ' - f"/tmp/vf-kimi-code/install.lock sh -c {shlex.quote(script)}" + await ensure_installed( + runtime, + directory="/tmp/vf-kimi-code", + install=script, + env={}, + label="Kimi Code", ) - install = await runtime.run(["sh", "-c", guarded], {}) - if install.exit_code != 0: - raise RuntimeError( - f"Kimi Code install failed: {install.stderr.strip()[-500:]}" - ) await super().setup(runtime) async def prepare_acp( diff --git a/verifiers/v1/harnesses/mini_swe_agent/harness.py b/verifiers/v1/harnesses/mini_swe_agent/harness.py index a510226f4a..767d937fb9 100644 --- a/verifiers/v1/harnesses/mini_swe_agent/harness.py +++ b/verifiers/v1/harnesses/mini_swe_agent/harness.py @@ -1,9 +1,7 @@ from pathlib import Path -from pydantic import Field - from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData @@ -13,7 +11,7 @@ class MiniSWEAgentHarnessConfig(HarnessConfig): - version: str = Field(default="2.4.6", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "2.4.6" """mini-swe-agent release to install, pinned for reproducibility.""" diff --git a/verifiers/v1/harnesses/node.py b/verifiers/v1/harnesses/node.py index 6f2146ee3a..5fdf80aadf 100644 --- a/verifiers/v1/harnesses/node.py +++ b/verifiers/v1/harnesses/node.py @@ -1,5 +1,4 @@ -import shlex - +from verifiers.v1.harnesses.utils.install import ensure_installed from verifiers.v1.runtimes import Runtime NODE_DIR = "/var/tmp/vf-node" @@ -39,16 +38,12 @@ async def ensure_node(runtime: Runtime) -> None: """Install the shared Node runtime used by ACP adapter harnesses.""" - lock = f"{NODE_DIR}.install.lock" - guarded = ( - f'until ln -s "$$" {lock} 2>/dev/null; do ' - f"owner=$(readlink {lock}); " - f'if ! kill -0 "$owner" 2>/dev/null; then ' - f'[ "$(readlink {lock})" != "$owner" ] || rm -f {lock}; fi; ' - f"sleep 0.1; done; " - f'trap \'[ "$(readlink {lock})" != "$$" ] || rm -f {lock}\' EXIT; ' - f"sh -c {shlex.quote(INSTALL)}" + # The install replaces NODE_DIR wholesale, so the lock lives beside it. + await ensure_installed( + runtime, + directory=NODE_DIR, + lock=f"{NODE_DIR}.install.lock", + install=INSTALL, + env={"VF_NODE_VERSION": NODE_VERSION}, + label="Node.js", ) - result = await runtime.run(["sh", "-c", guarded], {"VF_NODE_VERSION": NODE_VERSION}) - if result.exit_code != 0: - raise RuntimeError(f"Node.js install failed: {result.stderr.strip()[-500:]}") diff --git a/verifiers/v1/harnesses/openclaw/harness.py b/verifiers/v1/harnesses/openclaw/harness.py index 8af4b4bb7b..493654b2ac 100644 --- a/verifiers/v1/harnesses/openclaw/harness.py +++ b/verifiers/v1/harnesses/openclaw/harness.py @@ -4,13 +4,11 @@ import json import logging import secrets -import shlex - -from pydantic import Field from verifiers.v1.acp import ACPConfig, ACPHarness from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion +from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -120,7 +118,7 @@ class OpenClawHarnessConfig(HarnessConfig): - version: str = Field(default="2026.8.1", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "2026.8.1" """OpenClaw release to install, pinned for reproducibility.""" use_bundled_skill: bool = True """Enable OpenClaw's bundled skill catalog in addition to uploaded harness skills.""" @@ -143,39 +141,29 @@ async def setup(self, runtime: Runtime) -> None: ready_path = f"{self._staged_skills_dir}/.ready" ready = await runtime.run(["test", "-f", ready_path], {}) if ready.exit_code != 0: - cleared = await runtime.run( - ["rm", "-rf", self._staged_skills_dir], {} + await remove_dir( + runtime, self._staged_skills_dir, "OpenClaw skills" ) - if cleared.exit_code != 0: - raise RuntimeError( - "failed to clear OpenClaw skills: " - f"{cleared.stderr.strip()[-500:]}" - ) await self.install_skills(runtime, self._staged_skills_dir) await runtime.write(ready_path, b"") directory = OPENCLAW_DIR.format(version=self.config.version) binary = OPENCLAW_BIN.format(version=self.config.version) + logger.info("openclaw: ensuring OpenClaw %s is installed", self.config.version) # Borrowed runtimes can share this cache, so one filesystem lock owns both # installation and the transcript adjustment. - guarded = ( - f"mkdir -p {shlex.quote(directory)} && " - f'"$(command -v flock || command -v lockf)" ' - f"{shlex.quote(f'{directory}/install.lock')} " - f"bash -o pipefail -c {shlex.quote(SETUP)}" - ) - logger.info("openclaw: ensuring OpenClaw %s is installed", self.config.version) - result = await runtime.run( - ["sh", "-c", guarded], - { + await ensure_installed( + runtime, + directory=directory, + install=SETUP, + env={ **self.config.resolved_env, "VF_OPENCLAW_BIN": binary, "VF_OPENCLAW_DIR": directory, "VF_OPENCLAW_VERSION": self.config.version, }, + label="OpenClaw", + shell=("bash", "-o", "pipefail", "-c"), ) - if result.exit_code != 0: - detail = (result.stderr or result.stdout).strip()[-500:] - raise RuntimeError(f"OpenClaw setup failed: {detail}") await super().setup(runtime) async def prepare_acp( @@ -277,8 +265,4 @@ async def prepare_acp( async def cleanup(self, trace: Trace, runtime: Runtime) -> None: state_dir = f".vf-openclaw/{trace.id}" - result = await runtime.run(["rm", "-rf", state_dir], {}) - if result.exit_code != 0: - raise RuntimeError( - f"failed to clean up OpenClaw state: {result.stderr.strip()[-500:]}" - ) + await remove_dir(runtime, state_dir, "OpenClaw state") diff --git a/verifiers/v1/harnesses/pi/harness.py b/verifiers/v1/harnesses/pi/harness.py index b1403009ee..944b54399a 100644 --- a/verifiers/v1/harnesses/pi/harness.py +++ b/verifiers/v1/harnesses/pi/harness.py @@ -5,12 +5,11 @@ import shlex from typing import Literal -from pydantic import Field - from verifiers.v1.acp import ACPConfig, ACPHarness from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node +from verifiers.v1.harnesses.utils.install import ensure_installed from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -46,7 +45,7 @@ class PiHarnessConfig(HarnessConfig): - version: str = Field(default="0.84.1", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "0.84.1" """Pi release to install, pinned for reproducibility.""" transport: Literal["chat_completions", "responses", "anthropic_messages"] = ( "chat_completions" @@ -69,27 +68,17 @@ async def setup(self, runtime: Runtime) -> None: self.config.version, ACP_VERSION, ) - lock = f"{PI_DIR}/install.lock" - guarded = ( - f"mkdir -p {PI_DIR} && " - f'until ln -s "$$" {lock} 2>/dev/null; do ' - f"owner=$(readlink {lock}); " - f'if ! kill -0 "$owner" 2>/dev/null; then ' - f'[ "$(readlink {lock})" != "$owner" ] || rm -f {lock}; fi; ' - f"sleep 0.1; done; " - f'trap \'[ "$(readlink {lock})" != "$$" ] || rm -f {lock}\' EXIT; ' - f"sh -c {shlex.quote(INSTALL)}" - ) - install = await runtime.run( - ["sh", "-c", guarded], - { + await ensure_installed( + runtime, + directory=PI_DIR, + install=INSTALL, + env={ "VF_PI_VERSION": self.config.version, "VF_PI_MCP_VERSION": MCP_VERSION, "VF_PI_ACP_VERSION": ACP_VERSION, }, + label="pi", ) - if install.exit_code != 0: - raise RuntimeError(f"pi install failed: {install.stderr.strip()[-500:]}") await super().setup(runtime) async def prepare_acp( diff --git a/verifiers/v1/harnesses/pool/harness.py b/verifiers/v1/harnesses/pool/harness.py index 84258d6577..43a9cd487b 100644 --- a/verifiers/v1/harnesses/pool/harness.py +++ b/verifiers/v1/harnesses/pool/harness.py @@ -3,11 +3,10 @@ import json import shlex -from pydantic import Field - from verifiers.v1.acp import ACPConfig, ACPHarness from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion +from verifiers.v1.harnesses.utils.install import ensure_installed from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -28,7 +27,7 @@ class PoolHarnessConfig(HarnessConfig): - version: str = Field(default="1.0.15", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "1.0.15" """Pool release to install, pinned for reproducibility.""" @@ -44,17 +43,15 @@ async def setup(self, runtime: Runtime) -> None: script = INSTALL.replace("{version}", self.config.version).replace( "{dir}", directory ) - ensure = shlex.quote(f"[ -x {binary} ] || ({script})") - # Cache the pinned binary across local rollouts; Linux has flock, macOS has lockf. - guarded = ( - f"mkdir -p {directory} && " - f'"$(command -v flock || command -v lockf)" {directory}/install.lock ' - f"bash -o pipefail -c {ensure}" + await ensure_installed( + runtime, + directory=directory, + ready=f"[ -x {binary} ]", + install=script, + env=self.config.resolved_env, + label="Pool", + shell=("bash", "-o", "pipefail", "-c"), ) - result = await runtime.run(["sh", "-c", guarded], self.config.resolved_env) - if result.exit_code != 0: - detail = (result.stderr or result.stdout).strip()[-500:] - raise RuntimeError(f"Pool install failed: {detail}") await super().setup(runtime) async def prepare_acp( diff --git a/verifiers/v1/harnesses/prime_agent/harness.py b/verifiers/v1/harnesses/prime_agent/harness.py index dcdf7cda6d..25b4de367f 100644 --- a/verifiers/v1/harnesses/prime_agent/harness.py +++ b/verifiers/v1/harnesses/prime_agent/harness.py @@ -10,6 +10,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node +from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -157,26 +158,19 @@ async def setup(self, runtime: Runtime) -> None: await self.install_skills(runtime, SKILLS_DIR) await ensure_node(runtime) logger.info("prime-agent: ensuring commit %s is installed", self.config.commit) - lock = f"{PRIME_AGENT_DIR}/install.lock" - guarded = ( - f"mkdir -p {PRIME_AGENT_DIR} && " - f'"$(command -v flock || command -v lockf)" {lock} ' - f"sh -c {shlex.quote(INSTALL)}" - ) - result = await runtime.run( - ["sh", "-c", guarded], - { + await ensure_installed( + runtime, + directory=PRIME_AGENT_DIR, + install=INSTALL, + env={ **self.config.resolved_env, "VF_PRIME_AGENT_DIR": PRIME_AGENT_DIR, "VF_PRIME_AGENT_GITHUB_RELEASE_URL": GITHUB_RELEASE_URL, "PRIME_AGENT_COMMIT": self.config.commit, "PRIME_AGENT_RELEASE_VERSION": PRIME_AGENT_VERSION, }, + label="prime-agent", ) - if result.exit_code != 0: - raise RuntimeError( - f"prime-agent install failed: {result.stderr.strip()[-500:]}" - ) await super().setup(runtime) async def prepare_acp( @@ -285,11 +279,7 @@ async def prepare_acp( async def cleanup(self, trace: Trace, runtime: Runtime) -> None: root = self._root(trace) - removed = await runtime.run(["rm", "-rf", root], {}) - if removed.exit_code != 0: - raise RuntimeError( - f"prime-agent state cleanup failed: {removed.stderr.strip()[-500:]}" - ) + await remove_dir(runtime, root, "prime-agent state") def _bin(self) -> str: return f"{PRIME_AGENT_DIR}/{self.config.commit}/bin/prime-agent" diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index fd81436faa..29e9366009 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -18,6 +18,7 @@ from verifiers.v1.acp import ACPConfig, ACPHarness, ACPTurn, JsonObject from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.harnesses.utils.install import ensure_installed from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -130,16 +131,17 @@ async def setup(self, runtime: Runtime) -> None: f"touch {ready})" ) logger.info("rlm: ensuring rlm is installed (version=%s)", self.config.version) - ensure = shlex.quote(f"[ -f {ready} ] && [ -x {binary} ] || ({install})") - guarded = ( - f"mkdir -p {directory} && flock {directory}/install.lock sh -c {ensure}" - ) env = self.config.resolved_env.copy() extra_uv_args = env.get("RLM_EXTRA_UV_ARGS", "") env["RLM_EXTRA_UV_ARGS"] = f"{extra_uv_args} --with mcp~=1.28".strip() - result = await runtime.run(["sh", "-c", guarded], env) - if result.exit_code != 0: - raise RuntimeError(f"rlm install failed: {result.stderr.strip()[-500:]}") + await ensure_installed( + runtime, + directory=directory, + ready=f"[ -f {ready} ] && [ -x {binary} ]", + install=install, + env=env, + label="rlm", + ) await super().setup(runtime) def _runtime_metadata( diff --git a/verifiers/v1/harnesses/terminus_2/harness.py b/verifiers/v1/harnesses/terminus_2/harness.py index e54588e17f..c200efbf32 100644 --- a/verifiers/v1/harnesses/terminus_2/harness.py +++ b/verifiers/v1/harnesses/terminus_2/harness.py @@ -1,10 +1,8 @@ import logging from pathlib import Path -from pydantic import Field - from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion from verifiers.v1.harness import Harness from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData @@ -15,7 +13,7 @@ class Terminus2HarnessConfig(HarnessConfig): - version: str = Field(default="0.21.0", pattern=r"^[A-Za-z0-9._+-]+$") + version: PinnedVersion = "0.21.0" """Harbor release to install, pinned for reproducibility.""" diff --git a/verifiers/v1/harnesses/utils/install.py b/verifiers/v1/harnesses/utils/install.py new file mode 100644 index 0000000000..142d8bd7cb --- /dev/null +++ b/verifiers/v1/harnesses/utils/install.py @@ -0,0 +1,45 @@ +"""Host-side helpers for installing a harness's program into a runtime.""" + +import shlex + +from verifiers.v1.runtimes import Runtime + +# Linux has flock, macOS/BSD has lockf; either releases the lock when its holder dies. +_LOCKER = '"$(command -v flock || command -v lockf)"' + + +async def ensure_installed( + runtime: Runtime, + *, + directory: str, + install: str, + env: dict[str, str], + label: str, + ready: str | None = None, + lock: str | None = None, + shell: tuple[str, ...] = ("sh", "-c"), +) -> None: + """Run `install` in `runtime` under a lock, so concurrent rollouts sharing the runtime + install once and the rest wait. `ready` is a shell test that skips the install when it + already holds. The lock is `directory/install.lock` unless the install replaces + `directory` itself, in which case pass a `lock` path beside it. `shell` runs the script + (e.g. `bash -o pipefail -c` for pipelines).""" + lock = lock or f"{directory}/install.lock" + script = f"{ready} || ({install})" if ready else install + guarded = ( + f"mkdir -p {shlex.quote(directory)} && {_LOCKER} {shlex.quote(lock)} " + f"{shlex.join(shell)} {shlex.quote(script)}" + ) + result = await runtime.run(["sh", "-c", guarded], env) + if result.exit_code != 0: + detail = (result.stderr or result.stdout).strip()[-500:] + raise RuntimeError(f"{label} install failed: {detail}") + + +async def remove_dir(runtime: Runtime, path: str, label: str) -> None: + """Delete `path` in `runtime`; a failure names `label` (what the path held).""" + result = await runtime.run(["rm", "-rf", path], {}) + if result.exit_code != 0: + raise RuntimeError( + f"failed to clean up {label}: {result.stderr.strip()[-500:]}" + ) From 303f9823c7377cfb0ab575cbf06dd1b27dd65b72 Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 2 Sep 2026 18:25:23 +0000 Subject: [PATCH 2/3] refactor(v1): fall back to a symlink spinlock when flock and lockf are absent Keeps the install guard dependency-free, as the Pi and Node installers were before the shared helper. The fallback also reaps a lock that is not a symlink (left by flock/lockf) or whose owner pid is gone; the original spinlock spun forever on a regular lock file because `kill -0 ""` succeeds under busybox ash. Exercised on alpine:latest: flock branch and spinlock branch each serialize two concurrent installs, propagate the install's exit code, reap a dead-owner symlink and a leftover regular file, and wait on a live owner. Co-Authored-By: Claude Fable 5.1 --- verifiers/v1/harnesses/utils/install.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/verifiers/v1/harnesses/utils/install.py b/verifiers/v1/harnesses/utils/install.py index 142d8bd7cb..588b1fe0a2 100644 --- a/verifiers/v1/harnesses/utils/install.py +++ b/verifiers/v1/harnesses/utils/install.py @@ -4,9 +4,6 @@ from verifiers.v1.runtimes import Runtime -# Linux has flock, macOS/BSD has lockf; either releases the lock when its holder dies. -_LOCKER = '"$(command -v flock || command -v lockf)"' - async def ensure_installed( runtime: Runtime, @@ -23,12 +20,25 @@ async def ensure_installed( install once and the rest wait. `ready` is a shell test that skips the install when it already holds. The lock is `directory/install.lock` unless the install replaces `directory` itself, in which case pass a `lock` path beside it. `shell` runs the script - (e.g. `bash -o pipefail -c` for pipelines).""" - lock = lock or f"{directory}/install.lock" + (e.g. `bash -o pipefail -c` for pipelines). + + The lock is `flock` (Linux, busybox) or `lockf` (macOS/BSD), both released when the holder + dies; an image with neither falls back to a symlink spinlock that reaps a dead owner.""" + lock = shlex.quote(lock or f"{directory}/install.lock") script = f"{ready} || ({install})" if ready else install + run = f"{shlex.join(shell)} {shlex.quote(script)}" + # A lock that is not a symlink (left behind by flock/lockf) or whose owner pid is gone + # is stale and reaped. `kill -0 ""` succeeds under busybox ash, hence the explicit -z. + spinlock = ( + f'until ln -s "$$" {lock} 2>/dev/null; do owner=$(readlink {lock} 2>/dev/null); ' + f'if [ -z "$owner" ] || ! kill -0 "$owner" 2>/dev/null; then ' + f'[ "$(readlink {lock} 2>/dev/null)" != "$owner" ] || rm -f {lock}; fi; ' + f"sleep 0.1; done; " + f'trap \'[ "$(readlink {lock} 2>/dev/null)" != "$$" ] || rm -f {lock}\' EXIT; {run}' + ) guarded = ( - f"mkdir -p {shlex.quote(directory)} && {_LOCKER} {shlex.quote(lock)} " - f"{shlex.join(shell)} {shlex.quote(script)}" + f"mkdir -p {shlex.quote(directory)} && " + f'if l=$(command -v flock || command -v lockf); then "$l" {lock} {run}; else {spinlock}; fi' ) result = await runtime.run(["sh", "-c", guarded], env) if result.exit_code != 0: From 93a21004fe5ae10547bab54679faa809dacaad29 Mon Sep 17 00:00:00 2001 From: hallerite Date: Wed, 2 Sep 2026 19:30:12 +0000 Subject: [PATCH 3/3] refactor(v1): spinlock owner is pid:starttime; error detail skips blank stderr - The fallback lock's owner token is now `pid:starttime` (starttime read from /proc//stat where readable, pid alone otherwise), so a holder that died and had its pid reused is reaped instead of being waited on until the unrelated process exits. - A whitespace-only stderr no longer hides the diagnostic in stdout. Exercised on alpine:latest with flock hidden from PATH: a live process holding a lock recorded with a stale starttime is reaped in ~1s; the same process with its true identity is waited on; the earlier scenarios (concurrency, exit-code propagation, regular-file and dead-pid locks) hold. Co-Authored-By: Claude Fable 5.1 --- verifiers/v1/harnesses/utils/install.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/verifiers/v1/harnesses/utils/install.py b/verifiers/v1/harnesses/utils/install.py index 588b1fe0a2..70288d0e09 100644 --- a/verifiers/v1/harnesses/utils/install.py +++ b/verifiers/v1/harnesses/utils/install.py @@ -23,18 +23,28 @@ async def ensure_installed( (e.g. `bash -o pipefail -c` for pipelines). The lock is `flock` (Linux, busybox) or `lockf` (macOS/BSD), both released when the holder - dies; an image with neither falls back to a symlink spinlock that reaps a dead owner.""" + dies; an image with neither falls back to a symlink spinlock whose owner is recorded as + `pid:starttime` so a dead holder is reaped even if its pid was reused.""" lock = shlex.quote(lock or f"{directory}/install.lock") script = f"{ready} || ({install})" if ready else install run = f"{shlex.join(shell)} {shlex.quote(script)}" - # A lock that is not a symlink (left behind by flock/lockf) or whose owner pid is gone - # is stale and reaped. `kill -0 ""` succeeds under busybox ash, hence the explicit -z. + # The owner token is pid:starttime (starttime from /proc where readable), so a reused pid + # does not pass as the live holder. A lock that is not a symlink (left behind by + # flock/lockf) or whose owner is gone is stale and reaped; `kill -0 ""` succeeds under + # busybox ash, hence the explicit -z. + ident = ( + 'ident() { if [ -r "/proc/$1/stat" ] && s=$(sed "s/^.*) //" "/proc/$1/stat" ' + '| cut -d" " -f20) && [ -n "$s" ]; then echo "$1:$s"; else echo "$1"; fi; }; ' + "me=$(ident $$); " + ) spinlock = ( - f'until ln -s "$$" {lock} 2>/dev/null; do owner=$(readlink {lock} 2>/dev/null); ' - f'if [ -z "$owner" ] || ! kill -0 "$owner" 2>/dev/null; then ' + f"{ident}" + f'until ln -s "$me" {lock} 2>/dev/null; do owner=$(readlink {lock} 2>/dev/null); ' + f'pid=${{owner%%:*}}; if [ -z "$owner" ] || ! kill -0 "$pid" 2>/dev/null ' + f'|| [ "$(ident "$pid")" != "$owner" ]; then ' f'[ "$(readlink {lock} 2>/dev/null)" != "$owner" ] || rm -f {lock}; fi; ' f"sleep 0.1; done; " - f'trap \'[ "$(readlink {lock} 2>/dev/null)" != "$$" ] || rm -f {lock}\' EXIT; {run}' + f'trap \'[ "$(readlink {lock} 2>/dev/null)" != "$me" ] || rm -f {lock}\' EXIT; {run}' ) guarded = ( f"mkdir -p {shlex.quote(directory)} && " @@ -42,7 +52,7 @@ async def ensure_installed( ) result = await runtime.run(["sh", "-c", guarded], env) if result.exit_code != 0: - detail = (result.stderr or result.stdout).strip()[-500:] + detail = (result.stderr.strip() or result.stdout.strip())[-500:] raise RuntimeError(f"{label} install failed: {detail}")