[rollout-observability][7/7] Add SWE agent rollout observations

fern/versions/latest/pages/reference/trajectory-capabilities.mdx

@@ -62,6 +62,7 @@ For C1, C2, and C4, `V` evaluates the correlated Gym Model Server path; direct-p
 | `non_executing_simple_agent` | V | V | X | V | X | X | X |
 | `openclaw_agent` | V | V | X | O | V | V | V |
 | `opencode_agent` | V | V | X | V | V | V | V |
+| `opencode_sandboxed_agent` | V | V | X | V | V | V | V |
 | `osworld_agent` | O | O | X | O | X | X | X |
 | `pi_agent` | V | V | X | V | V | V | V |
 | `proof_refinement_agent` | V | V | X | V | X | X | X |
@@ -70,7 +71,8 @@ For C1, C2, and C4, `V` evaluates the correlated Gym Model Server path; direct-p
 | `simple_agent` | V | V | O | V | O | X | V |
 | `speed_bench_agent` | V | V | X | V | X | X | X |
 | `stirrup_agent` | O | O | X | O | X | X | X |
-| `swe_agents` | X | X | X | X | X | X | X |
+| `swe_agents` / OpenCode (legacy) | V | V | X | V | X | X | V |
+| `swe_agents` / OpenHands (legacy) | X | X | X | O | X | X | V |
 | `tau2` | V | V | X | V | X | X | X |
 | `tool_simulation_agent` | V | V | X | V | X | X | X |
 | `toolsandbox_agent` | V | V | X | V | X | X | X |
@@ -81,4 +83,8 @@ CVDP Simple path, Finance and Remote aggregate usage, LabBench image redaction,
 omitted raised failures in Simple-derived agents, and Stirrup calls outside its policy path. The matrix reports producer output,
 not schema capacity.
 OpenClaw coverage includes its standalone resource-server path and the PinchBench sandbox benchmark path.
+OpenCode coverage includes both the standalone producer and the decoupled `opencode_sandboxed_agent` path. The decoupled
+path composes agent observations with verifier-sandbox evidence returned by its resources server. Legacy OpenHands retains
+one cumulative root conversation, so its model-visible history remains partial; its pinned fork does not route model calls
+through a rollout-prefixed Gym Model Server endpoint.
 C7 requires standardized agent-side trajectory evidence; model HTTP capture alone does not satisfy it.

resources_servers/swebench/app.py

@@ -17,12 +17,12 @@
 from glob import glob
 from pathlib import Path
 from shutil import rmtree
-from time import time
+from time import monotonic, time
 from traceback import format_exc
 from typing import Any, Dict, Optional, Tuple
 
 from fastapi import Request
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from swebench.harness.run_evaluation import make_test_spec
 from swebench.harness.test_spec.test_spec import LATEST, TestSpec
 
@@ -36,6 +36,7 @@
     SimpleResourcesServer,
 )
 from nemo_gym.global_config import get_global_config_dict
+from nemo_gym.rollout_observability import SandboxObservation
 from nemo_gym.sandbox import AsyncSandbox, SandboxResources, SandboxSpec
 from nemo_gym.sandbox.config import resolve_provider_config, resolve_provider_metadata
 from nemo_gym.server_utils import SESSION_ID_KEY
@@ -105,6 +106,10 @@ class SWEBenchVerifyResponse(BaseVerifyResponse):
 
     log_dir: str
 
+    verifier_sandbox_observation: Optional[SandboxObservation] = Field(
+        default=None, exclude_if=lambda value: value is None
+    )
+
 
 # @bxyu-nvidia: This is a wrapper that can be passed directly to a very lightly modified version of `run_instance`
 # The method is almost identical to the original, just with async awaits rather than sync.
@@ -114,18 +119,26 @@ class DockerContainer(BaseModel):
     instance_id: str
 
     _inner_container: AsyncSandbox
+    _eval_return_code: Optional[int] = None
+    _sandbox_error_type: Optional[str] = None
 
     async def exec_run(
         self,
         command: str,
         workdir: Optional[str] = None,
         user: Optional[str] = None,
     ) -> ExecResult:
-        res = await self._inner_container.exec(
-            command=command,
-            cwd=workdir,
-            user=user,
-        )
+        try:
+            res = await self._inner_container.exec(
+                command=command,
+                cwd=workdir,
+                user=user,
+            )
+        except Exception as exc:
+            self._sandbox_error_type = self._sandbox_error_type or type(exc).__name__
+            raise
+        if res.error_type is not None:
+            self._sandbox_error_type = self._sandbox_error_type or res.error_type
 
         return ExecResult(
             exit_code=res.return_code,
@@ -143,6 +156,9 @@ async def exec_run_with_timeout(self, command: str, timeout: int) -> Tuple[str,
                 # AsyncSandbox.exec takes timeout_s, not docker-py's timeout.
                 timeout_s=timeout,
             )
+            self._eval_return_code = res.return_code if res.error_type is None else None
+            if res.error_type is not None:
+                self._sandbox_error_type = res.error_type
             timed_out = False
 
             stdout = res.stdout or ""
@@ -153,7 +169,11 @@ async def exec_run_with_timeout(self, command: str, timeout: int) -> Tuple[str,
         except TimeoutError:
             # Gym Sandbox API will throw a timeout error on actual timeout.
             timed_out = True
+            self._sandbox_error_type = "TimeoutError"
             test_output = ""
+        except Exception as exc:
+            self._sandbox_error_type = type(exc).__name__
+            raise
 
         return (test_output, timed_out, time() - start_time)
 
@@ -162,14 +182,41 @@ async def copy(self, src: Path, dest: Path) -> None:
             data = src.read_text()
             src.write_text(patch_swebench_multilingual_golden_patch_pass(data, self.instance_id))
 
-        await self._inner_container.upload(local_path=src, remote_path=str(dest))
+        try:
+            await self._inner_container.upload(local_path=src, remote_path=str(dest))
+        except Exception as exc:
+            self._sandbox_error_type = self._sandbox_error_type or type(exc).__name__
+            raise
 
     async def cleanup(self) -> None:
         try:
             await self._inner_container.stop()
-        except:
+        except Exception as exc:
+            self._sandbox_error_type = self._sandbox_error_type or type(exc).__name__
             print("Failed to stop verification sandbox", format_exc(), file=sys.stderr)
 
+    def observation(self, *, wall_time_s: float, evaluation_completed: bool) -> SandboxObservation:
+        handle = self._inner_container._handle
+        normalized_error = self._sandbox_error_type.lower() if isinstance(self._sandbox_error_type, str) else ""
+        if "timeout" in normalized_error:
+            outcome = "timeout"
+        elif self._sandbox_error_type is not None:
+            outcome = "sandbox_error"
+        elif evaluation_completed:
+            outcome = "completed"
+        else:
+            outcome = "failed"
+
+        return SandboxObservation(
+            role="verifier",
+            provider=handle.provider_name if handle is not None else None,
+            sandbox_id=handle.sandbox_id if handle is not None else None,
+            outcome=outcome,
+            exit_code=self._eval_return_code,
+            wall_time_s=wall_time_s,
+            error_type=self._sandbox_error_type,
+        )
+
 
 # TODO @bxyu-nvidia: Eventually once the sandbox server infra is ready, these seed_session types need to upgrade to pass a sandbox spec.
 # They can possibly even omitted once this graduates to core infra.
@@ -283,9 +330,9 @@ async def verify(self, request: Request, body: SWEBenchVerifyRequest) -> SWEBenc
 
         test_spec = self._make_test_spec(body)
 
-        start_time = time()
+        verifier_sandbox_lifecycle_started_at = monotonic()
         eval_sandbox = await self._create_sandbox(test_spec)
-        eval_sandbox_start_time_taken = time() - start_time
+        eval_sandbox_start_time_taken = monotonic() - verifier_sandbox_lifecycle_started_at
 
         model_patch = ""
         if self.config.is_verifying_golden_patch:
@@ -323,6 +370,16 @@ async def verify(self, request: Request, body: SWEBenchVerifyRequest) -> SWEBenc
             rewrite_reports=False,
         )
         patch_verification_time_taken = time() - start_time
+        verifier_sandbox_wall_time_s = monotonic() - verifier_sandbox_lifecycle_started_at
+
+        try:
+            verifier_sandbox_observation = mock_container.observation(
+                wall_time_s=verifier_sandbox_wall_time_s,
+                evaluation_completed=res["completed"],
+            )
+        except Exception:
+            verifier_sandbox_observation = None
+            print("Failed to build verification sandbox observation", format_exc(), file=sys.stderr)
 
         log_dir = Path(__file__).parent / "logs/run_evaluation" / run_id
 
@@ -347,6 +404,7 @@ async def verify(self, request: Request, body: SWEBenchVerifyRequest) -> SWEBenc
             model_patch=model_patch or None,
             test_output=test_output,
             log_dir=str(log_dir),
+            verifier_sandbox_observation=verifier_sandbox_observation,
         )
 
 

resources_servers/swebench/tests/test_app.py

@@ -12,13 +12,36 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from pathlib import Path
 from unittest.mock import AsyncMock, MagicMock
 
+import pytest
 from fastapi.testclient import TestClient
 from pytest import MonkeyPatch
 
+from nemo_gym.sandbox import SandboxExecResult, SandboxHandle
 from nemo_gym.server_utils import ServerClient
-from resources_servers.swebench.app import SwebenchResourcesServer, SwebenchResourcesServerConfig
+from resources_servers.swebench.app import (
+    DockerContainer,
+    SwebenchResourcesServer,
+    SwebenchResourcesServerConfig,
+    SWEBenchVerifyResponse,
+)
+
+
+def make_sandbox(
+    *,
+    exec_result: SandboxExecResult | None = None,
+    exec_error: Exception | None = None,
+    upload_error: Exception | None = None,
+    stop_error: Exception | None = None,
+) -> MagicMock:
+    sandbox = MagicMock()
+    sandbox._handle = SandboxHandle(sandbox_id="sandbox-123", provider_name="test-provider", raw=None)
+    sandbox.exec = AsyncMock(return_value=exec_result, side_effect=exec_error)
+    sandbox.upload = AsyncMock(side_effect=upload_error)
+    sandbox.stop = AsyncMock(side_effect=stop_error)
+    return sandbox
 
 
 class TestApp:
@@ -37,8 +60,10 @@ def test_sanity(self, monkeypatch: MonkeyPatch) -> None:
 
         client = TestClient(app)
 
+        eval_sandbox = make_sandbox()
         monkeypatch.setattr(
-            "resources_servers.swebench.app.SwebenchResourcesServer._create_sandbox", AsyncMock(start=AsyncMock())
+            "resources_servers.swebench.app.SwebenchResourcesServer._create_sandbox",
+            AsyncMock(return_value=eval_sandbox),
         )
         monkeypatch.setattr(
             "resources_servers.swebench.app.run_instance",
@@ -77,3 +102,130 @@ def test_sanity(self, monkeypatch: MonkeyPatch) -> None:
             },
         )
         assert res.status_code == 200
+        observation = res.json()["verifier_sandbox_observation"]
+        assert observation.pop("wall_time_s") >= 0
+        assert observation == {
+            "kind": "sandbox",
+            "role": "verifier",
+            "provider": "test-provider",
+            "sandbox_id": "sandbox-123",
+            "outcome": "completed",
+            "exit_code": None,
+            "cpu_time_s": None,
+            "peak_memory_mib": None,
+            "resource_usage_source": None,
+            "error_type": None,
+        }
+
+    def test_unobserved_response_omits_optional_field(self) -> None:
+        response = SWEBenchVerifyResponse.model_construct(verifier_sandbox_observation=None)
+
+        assert "verifier_sandbox_observation" not in response.model_dump()
+
+    async def test_eval_exit_code_is_observed_without_treating_failed_tests_as_sandbox_failure(self) -> None:
+        sandbox = make_sandbox(exec_result=SandboxExecResult(stdout="test output", stderr=None, return_code=7))
+        container = DockerContainer(id="run-id", instance_id="instance-id")
+        container._inner_container = sandbox
+
+        test_output, timed_out, _ = await container.exec_run_with_timeout("/bin/bash /eval.sh", timeout=60)
+        observation = container.observation(wall_time_s=3.5, evaluation_completed=True)
+
+        assert test_output == "test output"
+        assert timed_out is False
+        assert observation.outcome == "completed"
+        assert observation.exit_code == 7
+        assert observation.wall_time_s == 3.5
+
+    async def test_timeout_is_observed_without_changing_harness_timeout_behavior(self) -> None:
+        sandbox = make_sandbox(
+            exec_result=SandboxExecResult(
+                stdout=None,
+                stderr="backend failed",
+                return_code=125,
+                error_type="sandbox",
+            )
+        )
+        container = DockerContainer(id="run-id", instance_id="instance-id")
+        container._inner_container = sandbox
+
+        await container.exec_run("git apply patch.diff")
+        sandbox.exec.side_effect = TimeoutError("timed out")
+        test_output, timed_out, _ = await container.exec_run_with_timeout("/bin/bash /eval.sh", timeout=60)
+        observation = container.observation(wall_time_s=60.0, evaluation_completed=False)
+
+        assert test_output == ""
+        assert timed_out is True
+        assert observation.outcome == "timeout"
+        assert observation.exit_code is None
+        assert observation.error_type == "TimeoutError"
+
+    async def test_runtime_error_is_observed_and_still_propagates(self) -> None:
+        sandbox = make_sandbox(exec_error=RuntimeError("Sandbox was OOM-killed"))
+        container = DockerContainer(id="run-id", instance_id="instance-id")
+        container._inner_container = sandbox
+
+        with pytest.raises(RuntimeError, match="OOM-killed"):
+            await container.exec_run_with_timeout("/bin/bash /eval.sh", timeout=60)
+
+        observation = container.observation(wall_time_s=1.0, evaluation_completed=False)
+        assert observation.outcome == "sandbox_error"
+        assert observation.error_type == "RuntimeError"
+        assert observation.exit_code is None
+
+    @pytest.mark.parametrize(
+        ("error_type", "expected_outcome"),
+        [("sandbox", "sandbox_error"), ("TimeoutError", "timeout")],
+    )
+    async def test_provider_error_does_not_report_sentinel_as_process_exit_code(
+        self, error_type: str, expected_outcome: str
+    ) -> None:
+        sandbox = make_sandbox(
+            exec_result=SandboxExecResult(stdout=None, stderr="backend failed", return_code=125, error_type=error_type)
+        )
+        container = DockerContainer(id="run-id", instance_id="instance-id")
+        container._inner_container = sandbox
+
+        _, timed_out, _ = await container.exec_run_with_timeout("/bin/bash /eval.sh", timeout=60)
+        observation = container.observation(wall_time_s=1.0, evaluation_completed=False)
+
+        assert timed_out is False
+        assert observation.outcome == expected_outcome
+        assert observation.error_type == error_type
+        assert observation.exit_code is None
+
+    async def test_pre_eval_provider_error_is_observed(self) -> None:
+        sandbox = make_sandbox(
+            exec_result=SandboxExecResult(stdout=None, stderr="backend failed", return_code=125, error_type="sandbox")
+        )
+        container = DockerContainer(id="run-id", instance_id="instance-id")
+        container._inner_container = sandbox
+
+        await container.exec_run("git apply patch.diff")
+        observation = container.observation(wall_time_s=1.0, evaluation_completed=False)
+
+        assert observation.outcome == "sandbox_error"
+        assert observation.error_type == "sandbox"
+        assert observation.exit_code is None
+
+    async def test_upload_error_is_observed(self, tmp_path: Path) -> None:
+        sandbox = make_sandbox(upload_error=RuntimeError("upload failed"))
+        container = DockerContainer(id="run-id", instance_id="instance-id")
+        container._inner_container = sandbox
+
+        with pytest.raises(RuntimeError, match="upload failed"):
+            await container.copy(tmp_path / "patch.diff", Path("/tmp/patch.diff"))
+
+        observation = container.observation(wall_time_s=1.0, evaluation_completed=False)
+        assert observation.outcome == "sandbox_error"
+        assert observation.error_type == "RuntimeError"
+
+    async def test_cleanup_error_is_fail_open_and_observed(self) -> None:
+        sandbox = make_sandbox(stop_error=RuntimeError("stop failed"))
+        container = DockerContainer(id="run-id", instance_id="instance-id")
+        container._inner_container = sandbox
+
+        await container.cleanup()
+        observation = container.observation(wall_time_s=2.0, evaluation_completed=True)
+
+        assert observation.outcome == "sandbox_error"
+        assert observation.error_type == "RuntimeError"