swe_env/anyswe: address @ffrujeri review on #1572 (fixes + gold baseline)

fern/versions/latest/pages/agent-server/swe-environment.mdx

@@ -0,0 +1,70 @@
+---
+title: "SWE Environment"
+description: "Run any Gym agent inside a SWE-bench task container and grade its patch, on docker or apptainer."
+position: 3
+---
+
+# SWE Environment
+
+The **SWE environment** lets you evaluate any Gym agent on software-engineering benchmarks
+(SWE-bench Verified / Multilingual, SWE-bench-ext, R2E-Gym, SWE-rebench) by dropping the agent
+*inside* the task's container, extracting its `git diff`, and grading that patch against the
+instance's tests. It is harness-agnostic (the same flow runs hermes, claude_code, or openclaw)
+and sandbox-agnostic (docker or apptainer).
+
+It has two pieces:
+
+- **`responses_api_agents/anyswe_agent`** — the thin runner (an Agent Server). It provisions one
+  sandbox from the task image, runs the configured inner agent inside it via a generic
+  dynamic-loader runner, extracts the patch (`git add -A && git diff --cached`), and grades it.
+- **`responses_api_agents/swe_env`** — the shared library the runner builds on (not a server):
+  - **`harnesses/`** — per-dataset-family recipes: build the sandbox spec, materialize the patch,
+    run the evaluation, and grade the log host-side with the official per-repo parser.
+  - **`sandbox.py`** — async sandbox lifecycle (`AsyncSweEnvironment`, `acquire_sandbox`) over a
+    `nemo_gym.sandbox` provider, with always-teardown.
+  - **`self_drive.py`** — provision a writable sandbox, inject a sandbox-reachable model endpoint,
+    run the agent, and extract the diff.
+  - **`verify_task.py`** — grade a patch **inline** in a fresh sandbox (no separate verifier
+    server), returning a mask-aware reward.
+
+## Selecting the sandbox provider
+
+The same task runs on either backend via `sandbox_provider`. **docker** pulls the per-instance
+image on demand (no pre-build); **apptainer** runs a pre-built local `.sif`.
+
+```yaml
+# docker (default): images pull on demand from the registry
+anyswe_hermes:
+  responses_api_agents:
+    anyswe_agent:
+      container_formatter: "docker://swebench/sweb.eval.x86_64.{instance_id}"
+      sandbox_provider: {docker: {}}
+
+# apptainer: point the formatter at a local .sif and select the provider
+      container_formatter: "data/sifs/{instance_id}.sif"
+      sandbox_provider: {apptainer: {}}
+```
+
+<Note>
+Grading is host-side (flat): the eval log is parsed with swebench's official per-repo parser, so
+it runs on any exec-capable provider — docker, apptainer, or opensandbox. The provider only changes
+*where* the eval script runs, never the verdict: docker and apptainer resolve the **identical**
+SWE-bench Verified gold set (493/500 each, verified). See the in-tree
+`responses_api_agents/swe_env/README.md` for the gold-patch baseline and how to reproduce it.
+</Note>
+
+## Running it
+
+```bash
+# Prepare a dataset of tasks (optionally pre-build .sif images for apptainer)
+python responses_api_agents/anyswe_agent/prepare.py --limit 100
+
+# Start the servers, then collect rollouts
+ng_run "+config_paths=[responses_api_agents/anyswe_agent/configs/anyswe_hermes.yaml,responses_api_models/openai_model/configs/openai_model.yaml]"
+ng_collect_rollouts +agent_name=anyswe_hermes \
+  +input_jsonl_fpath=responses_api_agents/anyswe_agent/data/swebench_verified.jsonl \
+  +output_jsonl_fpath=results/hermes.jsonl +num_samples_in_parallel=16
+```
+
+The resolve rate is `mean/reward` in the run's `aggregate_metrics.json`; a clean run has
+`mean/mask_sample == 0` (infra failures and timeouts are masked out, not scored 0).

nemo_gym/sandbox/providers/docker/provider.py

@@ -54,6 +54,7 @@ def __init__(
         run_args: list[str] | None = None,
         keep_alive_command: str = "sleep infinity",
         concurrency: int = 32,
+        default_exec_timeout_s: int | float | None = 3600,
         **_: Any,
     ) -> None:
         """Configure the Docker sandbox provider.
@@ -70,6 +71,9 @@ def __init__(
                 it alive for subsequent ``exec`` calls.
             concurrency: Maximum number of concurrent ``docker`` CLI subprocesses,
                 bounded by a shared semaphore (matches the apptainer provider).
+            default_exec_timeout_s: Default per-``exec`` timeout (seconds) applied when a
+                caller passes none, so a hung in-container command cannot block a rollout
+                forever (e.g. the git-diff extraction in self_drive). None = no default.
             **_: Additional keyword arguments are accepted and ignored.
 
         Raises:
@@ -83,6 +87,7 @@ def __init__(
         self._run_args = list(run_args or [])
         self._keep_alive = keep_alive_command
         self._semaphore = asyncio.Semaphore(concurrency)
+        self._default_exec_timeout_s = default_exec_timeout_s
 
     async def _run(self, *args: str, timeout_s: int | float | None = None) -> tuple[int, str, str]:
         """Run the ``docker`` CLI with the given arguments and capture output.
@@ -101,12 +106,19 @@ async def _run(self, *args: str, timeout_s: int | float | None = None) -> tuple[
             text using ``errors="replace"``.
         """
         async with self._semaphore:
-            proc = await asyncio.create_subprocess_exec(
-                self._bin,
-                *args,
-                stdout=asyncio.subprocess.PIPE,
-                stderr=asyncio.subprocess.PIPE,
-            )
+            try:
+                proc = await asyncio.create_subprocess_exec(
+                    self._bin,
+                    *args,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                )
+            except FileNotFoundError as exc:
+                # Surface a clear, actionable error instead of a bare FileNotFoundError deep in
+                # create()/exec() (parity with apptainer's _require_apptainer up-front check).
+                raise SandboxCreateError(
+                    f"docker executable {self._bin!r} not found on PATH — install Docker or set docker_bin"
+                ) from exc
             try:
                 out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout_s)
             except (asyncio.TimeoutError, TimeoutError):
@@ -228,15 +240,16 @@ async def exec(
             cwd: Working directory for the command; falls back to the workdir
                 recorded at create time.
             env: Extra environment variables for the command.
-            timeout_s: Optional timeout in seconds; on expiry a result with
-                return code 124 and ``error_type="timeout"`` is returned.
+            timeout_s: Optional timeout in seconds; falls back to the provider's
+                ``default_exec_timeout_s`` when None. On expiry a result with return
+                code 124 and ``error_type="timeout"`` is returned.
             user: User (name or UID) to run as; falls back to the provider's
                 default user.
 
         Returns:
             A ``SandboxExecResult`` with stdout, stderr, return code, and an
-            ``error_type`` of ``"sandbox"`` for docker-level failures (125/126/
-            127 with no stdout), ``"timeout"`` on timeout, or None otherwise.
+            ``error_type`` of ``"sandbox"`` for a docker-daemon failure (rc 125 with
+            no stdout), ``"timeout"`` on timeout, or None otherwise.
         """
         args = ["exec"]
         workdir = cwd or handle.raw.get("workdir")
@@ -248,17 +261,22 @@ async def exec(
         for key, value in (env or {}).items():
             args += ["-e", f"{key}={value}"]
         args += [handle.sandbox_id, "bash", "-c", command]
+        eff_timeout = timeout_s if timeout_s is not None else self._default_exec_timeout_s
         try:
-            rc, out, err = await self._run(*args, timeout_s=timeout_s)
+            rc, out, err = await self._run(*args, timeout_s=eff_timeout)
         except (asyncio.TimeoutError, TimeoutError):
+            # Only the local `docker exec` client is killed here; the in-container process is
+            # reaped when the sandbox is closed (`docker rm -f`), which acquire_sandbox always does.
             return SandboxExecResult(
                 stdout=None,
-                stderr=f"command timed out after {timeout_s}s",
+                stderr=f"command timed out after {eff_timeout}s",
                 return_code=124,
                 error_type="timeout",
             )
-        # docker exec returns 125/126/127 for docker-level failures (container gone, not executable).
-        error_type = "sandbox" if rc in (125, 126, 127) and not out else None
+        # rc 125 is a docker-daemon-level failure (container gone / daemon error). 126 (not
+        # executable) and 127 (command not found) are legitimate *user*-command exit codes when
+        # run via `bash -c`, so only rc 125 with no stdout is classified as an infra failure.
+        error_type = "sandbox" if rc == 125 and not out else None
         return SandboxExecResult(stdout=out, stderr=err, return_code=rc, error_type=error_type)
 
     async def upload_file(self, handle: SandboxHandle, source_path: Path, target_path: str) -> None:
@@ -275,7 +293,7 @@ async def upload_file(self, handle: SandboxHandle, source_path: Path, target_pat
         parent = posixpath.dirname(target_path)
         if parent:
             await self.exec(handle, f"mkdir -p {shlex.quote(parent)}")
-        rc, out, err = await self._run("cp", str(source_path), f"{handle.sandbox_id}:{target_path}")
+        rc, out, err = await self._run("cp", str(source_path), f"{handle.sandbox_id}:{target_path}", timeout_s=300)
         if rc != 0:
             raise RuntimeError(f"docker cp upload failed: {err.strip() or out.strip()}")
 
@@ -292,7 +310,7 @@ async def download_file(self, handle: SandboxHandle, source_path: str, target_pa
         """
         target = Path(target_path)
         target.parent.mkdir(parents=True, exist_ok=True)
-        rc, out, err = await self._run("cp", f"{handle.sandbox_id}:{source_path}", str(target))
+        rc, out, err = await self._run("cp", f"{handle.sandbox_id}:{source_path}", str(target), timeout_s=300)
         if rc != 0:
             raise RuntimeError(f"docker cp download failed: {err.strip() or out.strip()}")
 
@@ -317,7 +335,12 @@ async def close(self, handle: SandboxHandle) -> None:
         Args:
             handle: Handle identifying the container to remove.
         """
-        await self._run("rm", "-f", handle.sandbox_id)
+        # Best-effort + bounded: teardown runs in acquire_sandbox's finally, so a wedged daemon
+        # must not hang (or crash) it after the result is already in hand.
+        try:
+            await self._run("rm", "-f", handle.sandbox_id, timeout_s=120)
+        except Exception:
+            pass
 
     async def aclose(self) -> None:
         """Release provider-level resources; this provider holds none."""

responses_api_agents/anyswe_agent/app.py

@@ -447,7 +447,12 @@ def _setup_params(self, body: NeMoGymResponseCreateParamsNonStreaming) -> Tuple[
         dataset_dir = persistent_dir / "instance_datasets"
         dataset_dir.mkdir(parents=True, exist_ok=True)
         instance_dataset_path = dataset_dir / f"{agent_run_id}.jsonl"
-        instance_dict = json.loads(problem_info["instance_dict"])
+        # Accept instance_dict as a JSON string or an already-parsed dict (mirrors _build_swetask),
+        # so a dict-valued row doesn't raise TypeError and mask the whole run with an opaque error.
+        raw_instance_dict = problem_info["instance_dict"]
+        instance_dict = (
+            json.loads(raw_instance_dict) if isinstance(raw_instance_dict, str) else dict(raw_instance_dict)
+        )
         instance_dict.setdefault("repo_name", instance_dict.get("repo", ""))
         instance_dataset_path.write_text(json.dumps(instance_dict) + "\n")
 
@@ -507,6 +512,10 @@ def _provider(self, params: AnySweInstanceConfig):
             start_args = list(create_cfg.get("extra_start_args") or [])
             if "--writable-tmpfs" not in start_args:
                 start_args.append("--writable-tmpfs")
+            # Isolate from the host $HOME (same reason as the grading sandbox in ``_grading_provider``):
+            # apptainer's default host-home bind leaks host dotfiles/caches into the agent run.
+            if "--no-mount" not in start_args:
+                start_args += ["--no-mount", "home"]
             create_cfg["extra_start_args"] = start_args
             appt["create"] = create_cfg
             return ApptainerProvider(**appt)
@@ -544,6 +553,13 @@ def _grading_provider(self):
         start_args = list(create_cfg.get("extra_start_args") or [])
         if "--writable-tmpfs" not in start_args:
             start_args.append("--writable-tmpfs")
+        # Don't bind-mount the host $HOME into the grading sandbox. apptainer mounts it by default,
+        # leaking host dotfiles/caches into the eval (e.g. ~/.config/matplotlib + the host font cache),
+        # which changes test outcomes vs docker -- matplotlib image-comparison tests fail on the host
+        # fonts even for the gold patch. (Scoped to --no-mount home: --cleanenv / --no-mount tmp,bind-paths
+        # are too aggressive and break the eval's conda/PATH env, producing an empty test log.)
+        if "--no-mount" not in start_args:
+            start_args += ["--no-mount", "home"]
         create_cfg["extra_start_args"] = start_args
         appt["create"] = create_cfg
         return {"apptainer": appt}

responses_api_agents/anyswe_agent/configs/anyswe_openclaw.yaml

@@ -42,11 +42,10 @@ anyswe_openclaw:
                   api: openai-completions
 
       container_formatter: data/sifs/{instance_id}.sif
+      sandbox_provider: {apptainer: {}}
       swebench_agent_timeout: 2100
       swebench_tests_timeout: 1800
-      apptainer_memory_limit_mb: 32768
       concurrency: 2
-      skip_eval: false
 
       datasets:
         - name: example