fix: free runtime resources on exit/interrupt

verifiers/v1/runtimes/__init__.py

@@ -11,7 +11,7 @@
 
 from pydantic import Field
 
-from verifiers.v1.runtimes.base import ProgramResult, Runtime
+from verifiers.v1.runtimes.base import ProgramResult, Runtime, register
 from verifiers.v1.runtimes.docker import DockerConfig, DockerRuntime
 from verifiers.v1.runtimes.prime import PrimeConfig, PrimeRuntime
 from verifiers.v1.runtimes.subprocess import SubprocessConfig, SubprocessRuntime
@@ -23,10 +23,13 @@
 
 def make_runtime(config: RuntimeConfig) -> Runtime:
     if isinstance(config, PrimeConfig):
-        return PrimeRuntime(config)
-    if isinstance(config, DockerConfig):
-        return DockerRuntime(config)
-    return SubprocessRuntime(config)
+        runtime: Runtime = PrimeRuntime(config)
+    elif isinstance(config, DockerConfig):
+        runtime = DockerRuntime(config)
+    else:
+        runtime = SubprocessRuntime(config)
+    register(runtime)
+    return runtime
 
 
 __all__ = [

verifiers/v1/runtimes/base.py

@@ -5,9 +5,13 @@
 depend only on this contract, so they stay runtime-agnostic.
 """
 
+import asyncio
+import atexit
+import contextlib
 import hashlib
 import shlex
 import uuid
+import weakref
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 
@@ -42,12 +46,49 @@ class ProgramResult:
     stderr: str
 
 
+# `stop()` frees a runtime's external resource on the normal path (the rollout's `finally`).
+# A Ctrl-C / SIGTERM can cancel that `finally` mid-teardown, so runtimes are tracked in
+# `_LIVE` and freed by a *synchronous* `atexit` hook (`cleanup`) — sync because the event
+# loop is gone at interpreter shutdown. SIGKILL runs none of this.
+_LIVE: "weakref.WeakSet[Runtime]" = weakref.WeakSet()
+_atexit_armed = False
+
+
+def register(runtime: "Runtime") -> None:
+    """Track a runtime so the atexit hook can free it if a signal cuts its `finally` short.
+    Weak, so a finished rollout's runtime drops out on its own; arms the hook once."""
+    global _atexit_armed
+    _LIVE.add(runtime)
+    if not _atexit_armed:
+        _atexit_armed = True
+        atexit.register(cleanup_at_exit)
+
+
+def cleanup_at_exit() -> None:
+    """Synchronously free any runtime still live at interpreter shutdown — a Ctrl-C /
+    SIGTERM cancelled its `finally` mid-teardown. Sync on purpose (the event loop is gone);
+    best-effort and idempotent (a clean `stop` already ran it)."""
+    for runtime in list(_LIVE):
+        with contextlib.suppress(Exception):
+            runtime.cleanup()
+
+
 class Runtime(ABC):
     @abstractmethod
     async def start(self) -> None:
         """Provision execution (workspace / container / sandbox). Use `expose` to turn a
         host port into a URL the program can reach."""
 
+    def cleanup(self) -> None:
+        """Synchronously free the provisioned resource — best-effort and idempotent. The
+        source of truth for teardown: usable from the atexit backstop where async machinery
+        is dead, and run off the event loop by `stop` on the normal path. Default no-op."""
+
+    async def stop(self) -> None:
+        """Free the provisioned resource on the normal path, off the event loop. Override
+        only for teardown that must be async (e.g. a remote API call)."""
+        await asyncio.to_thread(self.cleanup)
+
     async def expose(self, port: int) -> str:
         """A base URL the program (inside this runtime) can use to reach a host service
         on localhost `port` — the interception endpoint and host-side tool servers both
@@ -56,9 +97,6 @@ async def expose(self, port: int) -> str:
         tunnel the port."""
         return f"http://127.0.0.1:{port}"
 
-    async def stop(self) -> None:
-        """Tear down any provisioned resources. Default no-op."""
-
     @abstractmethod
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
         """Run `argv` (with the interception env vars `env`) to completion."""
@@ -117,7 +155,7 @@ async def run_uv_script(
         number of distinct scripts. Published via a unique temp + atomic `mv`, so
         concurrent rollouts writing the same content never race a half-written read."""
         data = script.encode() if isinstance(script, str) else script
-        path = f"/tmp/v1-scripts/{hashlib.sha256(data).hexdigest()}.py"
+        path = f"/tmp/vf-scripts/{hashlib.sha256(data).hexdigest()}.py"
         tmp = f"{path}.{uuid.uuid4().hex}.tmp"
         await self.write(tmp, data)
         await self.run(

verifiers/v1/runtimes/docker.py

@@ -8,6 +8,7 @@
 import contextlib
 import logging
 import shlex
+import subprocess
 import uuid
 from pathlib import PurePosixPath
 from typing import Literal
@@ -84,7 +85,7 @@ async def start(self) -> None:
             raise RuntimeError(
                 f"docker runtime selected but the Docker daemon is not reachable: {detail}{hint}"
             )
-        self._container = f"v1-{uuid.uuid4().hex[:12]}"
+        self._container = f"vf-{uuid.uuid4().hex[:12]}"
         limits: list[str] = []
         if self.config.cpu_cores is not None:
             limits += ["--cpus", str(self.config.cpu_cores)]
@@ -183,12 +184,17 @@ async def write(self, path: str, data: bytes) -> None:
                 f"write {path!r}: {stderr.decode(errors='replace').strip()}"
             )
 
-    async def stop(self) -> None:
+    def cleanup(self) -> None:
         if self._container is None or self._stopped:
             return
         self._stopped = (
             True  # idempotency guard; keep `_container` so the name still shows
         )
         logger.debug("docker: removing container %s", self._container)
         with contextlib.suppress(Exception):
-            await docker("rm", "--force", self._container)
+            subprocess.run(
+                ["docker", "rm", "--force", self._container],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=30,
+            )

verifiers/v1/runtimes/prime.py

@@ -81,7 +81,7 @@ async def start(self) -> None:
         try:
             sandbox = await self._client.create(
                 CreateSandboxRequest(
-                    name="v1-program",
+                    name="vf-program",
                     docker_image=self.config.image,
                     network_access=self.config.network_access,
                     vm=self.config.vm,
@@ -188,6 +188,22 @@ async def write(self, path: str, data: bytes) -> None:
         except Exception as e:
             raise ProgramError(f"write {path!r}: {e}") from e
 
+    def cleanup(self) -> None:
+        # Synchronous atexit backstop (the async client can't run once the loop is gone):
+        # stop the already-sync tunnels and delete the sandbox via the sync client, so the
+        # costly resource isn't left to its max-lifetime. Idempotent — the async `stop`
+        # deletes it on the normal path, and a second delete just 404s (suppressed).
+        for tunnel in self._tunnels:
+            with contextlib.suppress(Exception):
+                tunnel.sync_stop()
+        self._tunnels = []
+        if self._sandbox_id is not None:
+            from prime_sandboxes import SandboxClient
+            from prime_sandboxes.core import APIClient
+
+            with contextlib.suppress(Exception):
+                SandboxClient(APIClient()).delete(self._sandbox_id)
+
     async def stop(self) -> None:
         # Best-effort, idempotent teardown: each step is independent so one failure
         # never skips the sandbox delete (the costly resource). Runs from the

verifiers/v1/runtimes/subprocess.py

@@ -1,14 +1,15 @@
 """Local subprocess runtime: run the program on the host; server on localhost.
 
-Each rollout gets a fresh, unique `/tmp/v1-*` workspace (created on `start`,
-removed on `stop`) used as the program's cwd, so concurrent local rollouts are
-isolated and trivially cleaned up. Relative `read`/`write` paths resolve against it.
+Each rollout gets a fresh, unique `/tmp/vf-*` workspace (created on `start`, removed on
+`stop`/`cleanup`) used as the program's cwd, so concurrent local rollouts are isolated
+and trivially cleaned up. Relative `read`/`write` paths resolve against it.
 """
 
 import asyncio
 import contextlib
 import os
 import shutil
+import signal
 import tempfile
 from pathlib import Path
 from typing import Literal
@@ -42,7 +43,7 @@ def descriptor(self) -> str | None:
         return self.workdir.name if self.workdir else None
 
     async def start(self) -> None:
-        self.workdir = Path(tempfile.mkdtemp(prefix="v1-", dir="/tmp"))
+        self.workdir = Path(tempfile.mkdtemp(prefix="vf-", dir="/tmp"))
 
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:
         full_env = {k: v for k, v in os.environ.items() if "API_KEY" not in k.upper()}
@@ -89,10 +90,10 @@ async def write(self, path: str, data: bytes) -> None:
         target.parent.mkdir(parents=True, exist_ok=True)
         await asyncio.to_thread(target.write_bytes, data)
 
-    async def stop(self) -> None:
+    def cleanup(self) -> None:
         for proc in self._background:
-            with contextlib.suppress(ProcessLookupError):
-                proc.terminate()
+            with contextlib.suppress(ProcessLookupError, OSError):
+                os.kill(proc.pid, signal.SIGTERM)
         self._background = []
         if self.workdir is not None:
-            await asyncio.to_thread(shutil.rmtree, self.workdir, True)
+            shutil.rmtree(self.workdir, ignore_errors=True)