From cae7ae8129e0ca7df26311f80054886b9eca6153 Mon Sep 17 00:00:00 2001 From: adil-a Date: Tue, 23 Jun 2026 07:11:16 +0000 Subject: [PATCH] feat(swe_agents): drive OpenHands through the decoupled swe_env env + verifier (#1249) Wires the OpenHands swe_agents harness onto the swe_env library: provisions one working sandbox, self-drives, extracts the patch, and scores it through the swe_env verifier over HTTP. Relocates the SWE-bench-Ext parser into the shared swe_env package and deletes the duplicate copy under swe_agents. Adds a reference end-to-end driver and a documented SWE-bench Verified example. Co-Authored-By: Claude Opus 4.8 Signed-off-by: adil-a --- responses_api_agents/swe_agents/README.md | 47 +- responses_api_agents/swe_agents/app.py | 1640 ++++++----------- .../swe_agents/configs/swe_env_base.yaml | 39 + .../configs/swebench_multi_tools.yaml | 26 +- .../configs/swebench_openhands.yaml | 25 +- .../configs/swebench_openhands_training.yaml | 34 +- .../scripts/openhands_decoupled_rollout.py | 267 +++ .../swe_agents/swe_bench_ext/__init__.py | 0 .../swe_agents/swe_bench_ext/frameworks.py | 174 -- .../swe_agents/swe_bench_ext/parsing.py | 1606 ---------------- .../swe_agents/swe_bench_ext/utils.py | 166 -- .../swe_agents/swe_env_adapter.py | 415 +++++ .../swe_agents/tests/test_app.py | 1010 ++-------- .../swe_agents/tests/test_swe_env_adapter.py | 359 ++++ 14 files changed, 1945 insertions(+), 3863 deletions(-) create mode 100644 responses_api_agents/swe_agents/configs/swe_env_base.yaml create mode 100644 responses_api_agents/swe_agents/scripts/openhands_decoupled_rollout.py delete mode 100644 responses_api_agents/swe_agents/swe_bench_ext/__init__.py delete mode 100644 responses_api_agents/swe_agents/swe_bench_ext/frameworks.py delete mode 100644 responses_api_agents/swe_agents/swe_bench_ext/parsing.py delete mode 100644 responses_api_agents/swe_agents/swe_bench_ext/utils.py create mode 100644 responses_api_agents/swe_agents/swe_env_adapter.py create mode 100644 responses_api_agents/swe_agents/tests/test_swe_env_adapter.py diff --git a/responses_api_agents/swe_agents/README.md b/responses_api_agents/swe_agents/README.md index 8a8b7efc7b..a0e9d8afc1 100644 --- a/responses_api_agents/swe_agents/README.md +++ b/responses_api_agents/swe_agents/README.md @@ -2,7 +2,9 @@ A unified Responses-API wrapper that runs LLM-driven agents against real-world software-engineering benchmarks (SWE-bench and friends), executes the proposed patch inside the dataset's evaluation harness, and returns trajectories + a binary "resolved" reward suitable for both evaluation and RL training. -The entrypoint is [`app.py`](app.py), which exposes a `SWEBenchWrapper` (a `SimpleResponsesAPIAgent`) over HTTP. Each `responses` request takes one dataset instance, runs an agent inside an Apptainer container, runs the matching evaluation harness in a second container, and returns the trajectory plus reward. +The entrypoint is [`app.py`](app.py), which exposes a `SWEBenchWrapper` (a `SimpleResponsesAPIAgent`) over HTTP. Each `responses` request takes one dataset instance, provisions a single working sandbox for the agent through the shared `swe_env` library, lets the agent self-drive to produce a patch, scores that patch through the `swe_env` verifier in its own fresh sandbox, and returns the trajectory plus reward. + +For a runnable, validated end-to-end example (provision → self-drive → extract patch → verify → reward), see [End-to-end SWE-bench Verified](#end-to-end-swe-bench-verified). --- @@ -281,6 +283,49 @@ python responses_api_agents/swe_agents/client.py --- +## End-to-end SWE-bench Verified + +A minimal, self-contained run of one SWE-bench Verified instance through OpenHands and the decoupled `swe_env` verifier on a single machine with Docker. The driver [`scripts/openhands_decoupled_rollout.py`](scripts/openhands_decoupled_rollout.py) performs the full pipeline: provision the official SWE-bench image as a sandbox, let OpenHands self-drive, extract the patch from `output.jsonl`, and grade it in a fresh verifier sandbox. + +### 1. Serve a model + +Any OpenAI-compatible endpoint works. A local vLLM (the tool-call flags are required because OpenHands sends `tool_choice=auto`): + +```bash +docker run -d --name swe-vllm --gpus '"device=0"' \ + -v ~/.cache/huggingface:/root/.cache/huggingface -p 8000:8000 --ipc=host \ + vllm/vllm-openai:latest \ + --model Qwen/Qwen2.5-Coder-32B-Instruct-AWQ \ + --enable-auto-tool-choice --tool-call-parser hermes \ + --max-model-len 32768 --gpu-memory-utilization 0.92 +``` + +### 2. Run one instance end-to-end + +```bash +python responses_api_agents/swe_agents/scripts/openhands_decoupled_rollout.py \ + --instance astropy__astropy-13453 \ + --model Qwen/Qwen2.5-Coder-32B-Instruct-AWQ \ + --model-host 127.0.0.1 --model-port 8000 --max-iter 30 +``` + +The driver prints each stage and the final reward: + +``` +[provision] (9s) +[launch] OpenHands run_infer.sh (RUNTIME=local) ... +[patch] bytes +=== astropy__astropy-13453: resolved= patch_applied= error_kind=<...> REWARD=<0.0|1.0> === +``` + +`reward` is `1.0` when the agent's patch makes every `FAIL_TO_PASS` and `PASS_TO_PASS` test pass, else `0.0`; the reward depends on the model's coding ability, while the surrounding pipeline (provision → self-drive → extract → grade) is what this example exercises. To confirm the grading half independently, pass a known-good patch — the gold patch for `astropy__astropy-13453` grades to `reward=1.0` through the same verifier path. + +### Running at scale + +For a whole-dataset sweep through the Responses-API server stack rather than the single-instance driver, use the multi-server flow in [Quick Start](#quick-start) plus [Batch evaluation / data collection](#batch-evaluation--data-collection). To validate grading across the full set with gold patches (no model needed), see the verifier's [`resources_servers/swe_env/README.md`](../../resources_servers/swe_env/README.md). + +--- + ## Batch evaluation / data collection ```bash diff --git a/responses_api_agents/swe_agents/app.py b/responses_api_agents/swe_agents/app.py index f7a54acfdf..e5c677324a 100644 --- a/responses_api_agents/swe_agents/app.py +++ b/responses_api_agents/swe_agents/app.py @@ -11,6 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Agent harness server for SWE-bench-style software-engineering evaluation tasks. + +This module wires up a Responses API agent that runs OpenHands inside a per-task +sandbox to produce a code patch, then scores the patch by POSTing it to a separate +verifier server. It defines the agent configuration, per-instance configuration, +dataset/harness processors for the supported benchmark formats, the Ray worker that +drives a single task, and the FastAPI server that exposes the agent endpoints. +""" + import asyncio import glob import importlib.util @@ -24,7 +33,6 @@ import time import uuid from asyncio import Semaphore -from asyncio.subprocess import Process from contextlib import contextmanager from pathlib import Path from shutil import rmtree @@ -34,11 +42,8 @@ from typing import Any, Dict, Literal, Optional, Tuple, Union import ray -import tomlkit -from gprof2dot import main as gprof2dot_main from openai.types.responses.function_tool import FunctionTool from pydantic import BaseModel, ConfigDict, Field -from pydot import graph_from_dot_file from nemo_gym import PARENT_DIR from nemo_gym.base_resources_server import ( @@ -56,7 +61,7 @@ NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, ) -from nemo_gym.profiling import Profiler +from nemo_gym.server_utils import get_response_json, raise_for_status from responses_api_models.vllm_model.app import VLLMConverter, split_responses_input_output_items @@ -66,6 +71,8 @@ class AgentPromptOverride(BaseModel): + """Prompt and tool-naming overrides applied to a single agent run.""" + user_prompt_template: Optional[str] = Field( default=None, description="Path to the user prompt template file", @@ -89,6 +96,8 @@ class AgentPromptOverride(BaseModel): class SWEBenchWrapperConfig(BaseResponsesAPIAgentConfig): + """Server-level configuration for the SWE-bench agent harness.""" + model_server: ModelServerRef # Agent framework configuration @@ -131,10 +140,10 @@ class SWEBenchWrapperConfig(BaseResponsesAPIAgentConfig): default=False, description=( "If True, skip the agent run and use the sample's golden patch " - "(instance_dict['patch']) as the model patch. The eval container " - "still runs, so this verifies that the dataset sample actually " - "resolves when its golden patch is applied. Currently supported " - "for dataset_name == 'swe-bench-ext'." + "(instance_dict['patch']) as the model patch. The patch is graded via the " + "decoupled verifier (the same /verify POST the agent path uses), so this " + "verifies that the dataset sample actually resolves when its golden patch is " + "applied. Currently supported for dataset_name == 'swe-bench-ext'." ), ) @@ -153,8 +162,30 @@ class SWEBenchWrapperConfig(BaseResponsesAPIAgentConfig): openhands_should_log: bool = False debug: bool = False + # Retained (default True) for config compatibility; there is a single eval path and no branch to gate. + eval_via_verifier: bool = Field( + default=True, + description=( + "Run OpenHands in a single working sandbox via the decoupled swe_env infra " + "(acquire_sandbox + self-drive + output.jsonl patch extraction) and score the patch by " + "POSTing to the swe_env verifier (verifier_server_name). This is the only supported eval " + "path." + ), + ) + verifier_server_name: Optional[str] = Field( + default=None, + description="Name of the resources_servers/swe_env verifier to POST /verify to when eval_via_verifier=True.", + ) + sandbox_provider: Optional[Dict[str, Any]] = Field( + default=None, + description="Single-key swe_env sandbox provider mapping for the decoupled path " + "(e.g. {'docker': {...}} or {'apptainer': {...}}). Defaults to apptainer when eval_via_verifier=True.", + ) + class SWEBenchWrapperServerConfig(BaseModel): + """Per-run server state computed once at startup (session id, setup dirs, results dir).""" + ng_global_config_dict_str: str model_server_name: str openhands_setup_dir: Path @@ -167,6 +198,8 @@ class SWEBenchWrapperServerConfig(BaseModel): class ExecuteContainerCommandArgs(BaseModel): + """Arguments describing a command to run inside a container and the file it produces.""" + command: str expected_file_pattern: str mode: Union[Literal["agent"], Literal["eval"]] @@ -174,6 +207,12 @@ class ExecuteContainerCommandArgs(BaseModel): class SWEBenchWrapperInstanceConfig(SWEBenchWrapperServerConfig, SWEBenchWrapperConfig): + """Fully resolved configuration for a single task instance. + + Combines the server-level config and per-run server state with the per-instance + problem info, paths, resolved prompt overrides, and timing inputs used by the worker. + """ + metrics_fpath: Path problem_info: Dict[str, Any] body: NeMoGymResponseCreateParamsNonStreaming @@ -187,7 +226,8 @@ class SWEBenchWrapperInstanceConfig(SWEBenchWrapperServerConfig, SWEBenchWrapper output_for_eval_mounted_path: Path output_for_eval_path: Path model_patch_path: Path - container: str + # Not populated; the image is resolved via _resolve_image_name. Kept Optional/None for config compatibility. + container: Optional[str] = None eval_dir_in_openhands: str openhands_config_file_path: str agent_script_path: Path @@ -206,7 +246,7 @@ class SWEBenchWrapperInstanceConfig(SWEBenchWrapperServerConfig, SWEBenchWrapper resolved_diversify_tool_names: Optional[bool] = False resolved_camel_case_tool_names: Optional[bool] = False - # Set later + # Optional fields that are not populated. Kept Optional/None to avoid config churn for callers that set them. eval_command: Optional[ExecuteContainerCommandArgs] = None eval_apptainer_command_str: Optional[str] = None agent_command: Optional[ExecuteContainerCommandArgs] = None @@ -222,6 +262,8 @@ def instance_id(self) -> str: class SWEBenchMetrics(BaseModel): + """Per-task outcome and timing metrics persisted and reported for a run.""" + resolved: Optional[bool] = None patch_exists: Optional[bool] = None model_patch: Optional[str] = None @@ -247,6 +289,8 @@ class SWEBenchMetrics(BaseModel): class SWEBenchVerifyResponse(SWEBenchMetrics, BaseVerifyResponse): + """Verify response combining the reward/response payload with task metrics and config.""" + instance_config: SWEBenchWrapperInstanceConfig @@ -256,6 +300,8 @@ class SWEBenchVerifyResponse(SWEBenchMetrics, BaseVerifyResponse): class BaseDatasetHarnessProcessor(BaseModel): + """Base class for dataset- and harness-specific setup and post-run processing.""" + config: SWEBenchWrapperConfig | SWEBenchWrapperInstanceConfig ######################################## @@ -267,13 +313,30 @@ def parent_dir(self) -> Path: return Path(__file__).parent def _run_setup_command(self, command: str) -> None: + """Run a setup shell command and assert it exits successfully. + + Args: + command: The shell command to execute. + """ process = Popen(command, shell=True) return_code = process.wait() assert return_code == 0, f"Command failed: {command}" @contextmanager def _setup_directory_lock(self, setup_dir: Path, label: str): - """Cross-node lock using mkdir (atomic on Lustre/NFS, unlike fcntl.flock).""" + """Acquire a cross-node directory lock for the duration of the context. + + Uses an atomic ``mkdir`` as the lock primitive so it works on shared filesystems + where advisory file locks are unreliable. Polls until the lock is acquired, breaking + a stale lock that is older than the threshold, and removes the lock on exit. + + Args: + setup_dir: The directory whose setup is being guarded; the lock lives beside it. + label: Human-readable name used in log messages. + + Yields: + None: Control once the lock is held. + """ lock_dir = setup_dir.parent lock_dir.mkdir(parents=True, exist_ok=True) lock_path = lock_dir / f".{setup_dir.name}.lockdir" @@ -309,22 +372,34 @@ def _setup_directory_lock(self, setup_dir: Path, label: str): finally: shutil.rmtree(lock_path, ignore_errors=True) - # Setup method is sync for now since there's been no need to concurrently set up + # Setup method is sync since there's been no need to concurrently set up. def setup(self) -> Path: - pass + """Set up the dataset or harness for this processor. - def get_run_command(self) -> ExecuteContainerCommandArgs: + Returns: + Path: The setup directory. The base implementation does nothing. + """ pass def postprocess_after_run(self, report_file: Path) -> None: - pass + """Post-process the run output, typically producing or rewriting the report file. - def _get_command_sleep_until_predictions_file(self) -> str: - return f"until [ -f {self.config.output_for_eval_mounted_path} ]; do sleep 5; done" + Args: + report_file: Path to the report file to read and/or write. The base + implementation does nothing. + """ + pass class SweBenchDatasetProcessor(BaseDatasetHarnessProcessor): + """Dataset processor for SWE-bench tasks.""" + def setup(self) -> Path: + """Clone and install the SWE-bench harness under a locked setup directory. + + Returns: + Path: The setup directory containing the prepared SWE-bench environment. + """ swebench_repo = "https://github.com/HeyyyyyyG/SWE-bench.git" swebench_commit = "HEAD" @@ -353,48 +428,16 @@ def setup(self) -> Path: return setup_dir - def get_run_command(self) -> ExecuteContainerCommandArgs: - swebench_cmd = ( - f'date +"%s.%N" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} && ' - f"{self._get_command_sleep_until_predictions_file()} && " - # Use pre-built SWE-bench - "cd /swebench_setup/SWE-bench && " - # Set UV environment variables to use the mounted portable directories - f'export UV_INSTALL_DIR="{self.config.swebench_setup_dir}/uv" && ' - f'export UV_PYTHON_INSTALL_DIR="{self.config.swebench_setup_dir}/python" && ' - f'export PATH="{self.config.swebench_setup_dir}/uv/bin:$PATH" && ' - f"ls -lrt /root/dataset && " - # Run with clean environment to avoid venv contamination - # Use the pre-built venv directly with its absolute path - f"env -u VIRTUAL_ENV {self.config.swebench_setup_dir}/SWE-bench/venv/bin/python -m swebench.harness.run_local_evaluation " - f" --predictions_path {self.config.output_for_eval_mounted_path} " - f" --instance_ids {self.config.instance_id} " - f" --timeout {self.config.swebench_tests_timeout} " - f" --dataset_name /root/dataset/data.jsonl " - f" --split {self.config.problem_info['split']} " - f" --run_id {self.config.agent_run_id} && " - f"cp -r logs/run_evaluation/{self.config.agent_run_id} /trajectories_mount/ && " - f"rm -rf logs/run_evaluation/{self.config.agent_run_id} && rm -rf *{self.config.agent_run_id}*" - ) - - # Execute SWE-bench evaluation command - search_path = os.path.join( - self.config.persistent_dir, - self.config.agent_run_id, - "**", - f"{self.config.instance_id}/report.json", - ) - - return ExecuteContainerCommandArgs( - command=swebench_cmd, - expected_file_pattern=search_path, - mode="eval", - timeout=self.config.swebench_tests_timeout + 120, - ) - class SweBenchMultilingualDatasetProcessor(BaseDatasetHarnessProcessor): + """Dataset processor for SWE-bench Multilingual tasks.""" + def setup(self) -> Path: + """Clone and install the SWE-bench Multilingual harness under a locked setup directory. + + Returns: + Path: The setup directory containing the prepared environment. + """ swebench_repo = "https://github.com/Kipok/SWE-bench.git" swebench_commit = "HEAD" @@ -423,48 +466,18 @@ def setup(self) -> Path: return setup_dir - def get_run_command(self) -> ExecuteContainerCommandArgs: - swebench_cmd = ( - f'date +"%s.%N" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} && ' - f"{self._get_command_sleep_until_predictions_file()} && " - # Use pre-built SWE-bench - "cd /swebench_multilingual_setup/SWE-bench_Multilingual && " - # Set UV environment variables to use the mounted portable directories - f'export UV_INSTALL_DIR="{self.config.swebench_multilingual_setup_dir}/uv" && ' - f'export UV_PYTHON_INSTALL_DIR="{self.config.swebench_multilingual_setup_dir}/python" && ' - f'export PATH="{self.config.swebench_multilingual_setup_dir}/uv/bin:$PATH" && ' - f"ls -lrt /root/dataset && " - # Run with clean environment to avoid venv contamination - # Use the pre-built venv directly with its absolute path - f"env -u VIRTUAL_ENV {self.config.swebench_multilingual_setup_dir}/SWE-bench_Multilingual/venv/bin/python -m swebench.harness.run_local_evaluation " - f" --predictions_path {self.config.output_for_eval_mounted_path} " - f" --instance_ids {self.config.instance_id} " - f" --timeout {self.config.swebench_tests_timeout} " - f" --dataset_name /root/dataset/data.jsonl " - f" --split {self.config.problem_info['split']} " - f" --run_id {self.config.agent_run_id} && " - f"cp -r logs/run_evaluation/{self.config.agent_run_id} /trajectories_mount/ && " - f"rm -rf logs/run_evaluation/{self.config.agent_run_id} && rm -rf *{self.config.agent_run_id}*" - ) - # Execute SWE-bench evaluation command - search_path = os.path.join( - self.config.persistent_dir, - self.config.agent_run_id, - "**", - f"{self.config.instance_id}/report.json", - ) +class R2EGymDatasetProcessor(BaseDatasetHarnessProcessor): + """Dataset processor for R2E-Gym tasks.""" - return ExecuteContainerCommandArgs( - command=swebench_cmd, - expected_file_pattern=search_path, - mode="eval", - timeout=self.config.swebench_tests_timeout + 120, - ) + def setup(self) -> Path: + """Clone and install the R2E-Gym harness under a locked setup directory. + Verifies an existing install by importing ``r2egym`` and rebuilds if missing. -class R2EGymDatasetProcessor(BaseDatasetHarnessProcessor): - def setup(self) -> Path: + Returns: + Path: The setup directory containing the prepared R2E-Gym environment. + """ eval_harness_repo = "https://github.com/sdevare-nv/nv-R2E-Gym.git" eval_harness_commit = "local-eval" @@ -501,137 +514,19 @@ def setup(self) -> Path: return setup_dir - def get_run_command(self) -> ExecuteContainerCommandArgs: - r2e_gym_cmd = ( - f'date +"%s.%N" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} && ' - f"{self._get_command_sleep_until_predictions_file()} && " - # Use mounted directory path for cd - "cd /r2egym_setup/R2E-Gym && " - # Set UV environment variables to use the mounted portable directories - f'export UV_INSTALL_DIR="{self.config.r2e_gym_setup_dir}/uv" && ' - f'export UV_PYTHON_INSTALL_DIR="{self.config.r2e_gym_setup_dir}/python" && ' - f'export PATH="{self.config.r2e_gym_setup_dir}/uv/bin:$PATH" && ' - # Run with clean environment to avoid venv contamination - # Use the pre-built venv directly with its absolute path - f"env -u VIRTUAL_ENV {self.config.r2e_gym_setup_dir}/R2E-Gym/venv/bin/python src/r2egym/agenthub/run/run_local_evaluation.py " - f" --predictions_path {self.config.output_for_eval_mounted_path} " - f" --instance_id {self.config.instance_id} " - f" --timeout {self.config.swebench_tests_timeout} " - f" --dataset /root/dataset/data.jsonl " - f" --output_dir /trajectories_mount/eval-outputs/{self.config.agent_run_id}" - ) - - search_path = os.path.join( - self.config.persistent_dir, - "eval-outputs", - self.config.agent_run_id, - "report.json", - ) - - return ExecuteContainerCommandArgs( - command=r2e_gym_cmd, - expected_file_pattern=search_path, - mode="eval", - timeout=self.config.swebench_tests_timeout + 120, - ) - class NVInternalDatasetProcessor(BaseDatasetHarnessProcessor): - def get_run_command(self) -> ExecuteContainerCommandArgs: - instance_dict = json.loads(self.config.problem_info["instance_dict"]) - base_dockerfile = instance_dict.get("base_dockerfile", "") - instance_dockerfile = instance_dict.get("instance_dockerfile", "") - - env_lines = [] - for line in (base_dockerfile + "\n" + instance_dockerfile).split("\n"): - line = line.strip() - if line.startswith("ENV "): - # Convert ENV KEY=VALUE or ENV KEY VALUE to export KEY="VALUE" - export_line = line.replace("ENV ", "export ", 1) - # Handle both Docker ENV formats: - # 1. ENV KEY=VALUE (with equals) - # 2. ENV KEY VALUE (space-separated) - if "=" in export_line: - # Format: export KEY=VALUE -> normalize spaces around = - export_line = re.sub(r"\s*=\s*", "=", export_line) - else: - # Format: export KEY VALUE -> convert to export KEY="VALUE" - parts = export_line.split(None, 2) # Split into at most 3 parts - if len(parts) >= 3: # export KEY VALUE - key = parts[1] - value = parts[2] - export_line = f'export {key}="{value}"' - - env_lines.append(export_line) - - env_exports = "\n".join(env_lines) - - # Get repo setup command - repo_cmd = instance_dict.get("before_repo_set_cmd", "").strip() - if repo_cmd: - repo_cmd = repo_cmd.split("\n")[-1] - - # Get test files - test_files_str = instance_dict.get("selected_test_files_to_run", "[]") - if isinstance(test_files_str, str): - test_files = ",".join(eval(test_files_str)) - else: - test_files = ",".join(test_files_str) - - run_script = instance_dict["run_script.sh"] - parsing_script = instance_dict["parsing_script.py"] - run_script_path = self.config.persistent_dir / "run_script.sh" - parsing_script_path = self.config.persistent_dir / "parsing_script.py" - with open(run_script_path, "w") as f: - f.write(run_script) - with open(parsing_script_path, "w") as f: - f.write(parsing_script) - - cmd = f"""#!/bin/bash -set -e - -date +\"%s.%N\" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} + """Dataset processor for the nv-internal dataset format.""" -{self._get_command_sleep_until_predictions_file()} - -{env_exports} - -# Apply patch -cd /app -git reset --hard {instance_dict.get("base_commit", "")} -git checkout {instance_dict.get("base_commit", "")} - -# Apply patch with rejection to handle conflicts -git apply --ignore-space-change --ignore-whitespace --reject -v /root/patch.diff || true - -# Setup repository -{repo_cmd} - -# Run tests -bash /root/run_script.sh {test_files} > /root/stdout.log 2> /root/stderr.log || true - -# Parse results -python /root/parsing_script.py /root/stdout.log /root/stderr.log /root/output.json - -# Move outputs to the mounted directory -mkdir -p /trajectories_mount/eval_results -cp /root/output.json /trajectories_mount/eval_results/output.json -""" - - search_path = os.path.join( - self.config.persistent_dir, - "eval_results", - "output.json", - ) + def postprocess_after_run(self, report_file: Path) -> None: + """Grade the test results and overwrite the report file with a resolution summary. - return ExecuteContainerCommandArgs( - command=cmd, - expected_file_pattern=search_path, - mode="eval", - timeout=self.config.swebench_tests_timeout, - ) + Reads the fail-to-pass and pass-to-pass test sets from the instance, checks whether + they all passed, and writes a report keyed by instance id. - def postprocess_after_run(self, report_file: Path) -> None: + Args: + report_file: Path to the report file containing raw test results; rewritten in place. + """ instance_dict = json.loads(self.config.problem_info["instance_dict"]) fail_to_pass_str = instance_dict.get("fail_to_pass_select", instance_dict.get("fail_to_pass", "[]")) @@ -673,6 +568,17 @@ def check_tests_passed( f2p: set[str], p2p: set[str], ) -> bool: + """Check whether every required test passed. + + Args: + test_results: Parsed test results, a mapping with a ``tests`` list of + ``{"name", "status"}`` entries. + f2p: Set of fail-to-pass test names that must pass. + p2p: Set of pass-to-pass test names that must pass. + + Returns: + bool: True if all required tests passed, otherwise False. + """ if not test_results: return False @@ -687,6 +593,17 @@ def check_tests_passed( def _load_rebench_log_parsers(rebench_repo_dir: Path): + """Dynamically import the SWE-rebench log-parsers module from a checked-out repo. + + Temporarily prepends the repo directories to ``sys.path`` so the module's own imports + resolve, loads it from its file location, and restores ``sys.path`` afterward. + + Args: + rebench_repo_dir: Path to the checked-out SWE-rebench repository. + + Returns: + module: The loaded log-parsers module. + """ lp_path = rebench_repo_dir / "lib" / "agent" / "log_parsers.py" if not lp_path.exists(): lp_path = rebench_repo_dir / "agent" / "log_parsers.py" @@ -711,7 +628,14 @@ def _load_rebench_log_parsers(rebench_repo_dir: Path): class SWERebenchDatasetProcessor(BaseDatasetHarnessProcessor): + """Dataset processor for SWE-rebench tasks.""" + def setup(self) -> Path: + """Clone the SWE-rebench repository under a locked setup directory. + + Returns: + Path: The setup directory containing the prepared SWE-rebench environment. + """ setup_dir = self.parent_dir / "swe_rebench_setup" with self._setup_directory_lock(setup_dir, "SWE-rebench"): @@ -734,6 +658,14 @@ def setup(self) -> Path: @staticmethod def _normalize_test_name(name: str) -> str: + """Strip trailing timing annotations from a test name and trim whitespace. + + Args: + name: The raw test name as parsed from the test output. + + Returns: + str: The normalized test name. + """ _REBENCH_TIMING_NORMALIZE_RES = [ re.compile(r"\s*\[\s*\d+(?:\.\d+)?\s*(?:ms|s)\s*\]\s*$", re.IGNORECASE), re.compile(r"\s+in\s+\d+(?:\.\d+)?\s+(?:msec|sec)\b", re.IGNORECASE), @@ -743,96 +675,17 @@ def _normalize_test_name(name: str) -> str: name = pattern.sub("", name) return name.strip() - def get_run_command(self) -> ExecuteContainerCommandArgs: - instance_dict = json.loads(self.config.problem_info["instance_dict"]) - install_config = instance_dict.get("install_config", {}) - test_cmds = install_config.get("test_cmd", []) - if isinstance(test_cmds, str): - test_cmds = [test_cmds] - install_cmds = install_config.get("install", []) - if isinstance(install_cmds, str): - install_cmds = [install_cmds] - # log_parser_name = install_config.get("log_parser", "") - - repo = instance_dict.get("repo", "") - repo_name = repo.split("/")[1] if "/" in repo else repo - - test_patch = instance_dict.get("test_patch", "") - test_patch_path = self.config.persistent_dir / "test_patch.diff" - test_patch_path.write_text(test_patch) - - fail_to_pass = instance_dict.get("FAIL_TO_PASS", []) - pass_to_pass = instance_dict.get("PASS_TO_PASS", []) - if isinstance(fail_to_pass, str): - fail_to_pass = json.loads(fail_to_pass) - if isinstance(pass_to_pass, str): - pass_to_pass = json.loads(pass_to_pass) - - # Write test metadata to files to avoid exceeding OS argument length limits - eval_meta_dir = self.config.persistent_dir / "eval_meta" - eval_meta_dir.mkdir(parents=True, exist_ok=True) - # Pre-normalize all expected test names so the in-container eval script - # can compare directly without duplicating the normalization regexes. - norm_fail_to_pass = sorted(self._normalize_test_name(n) for n in fail_to_pass) - norm_pass_to_pass = sorted(self._normalize_test_name(n) for n in pass_to_pass) - (eval_meta_dir / "expected_passed.json").write_text( - json.dumps(sorted(set(norm_fail_to_pass + norm_pass_to_pass))) - ) - (eval_meta_dir / "fail_to_pass.json").write_text(json.dumps(norm_fail_to_pass)) - (eval_meta_dir / "pass_to_pass.json").write_text(json.dumps(norm_pass_to_pass)) - - install_block = "\n".join(install_cmds) if install_cmds else "" - test_block = "\n".join(test_cmds) - - cmd = f"""#!/bin/bash -set -e - -date +\"%s.%N\" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} - -{self._get_command_sleep_until_predictions_file()} - -cd /{repo_name} -git reset --hard HEAD - -# Apply model patch -git apply --reject --recount --ignore-space-change --whitespace=nowarn /root/patch.diff || true - -# Apply test patch -git apply --reject --recount --ignore-space-change --whitespace=nowarn /root/test_patch.diff || true - -# Run install commands (non-fatal, some may fail harmlessly) -set +e -{install_block} -set -e - -# Run tests and write output to bind-mounted path (parsed on host, no python3 needed) -mkdir -p /trajectories_mount/eval_results -set +e -( -{test_block} -) > /trajectories_mount/eval_results/test_output.log 2>&1 -TEST_EXIT=$? -set -e - -printf '{{"_test_completed": true, "exit_code": %d}}\\n' $TEST_EXIT \ - > /trajectories_mount/eval_results/report.json -""" - - search_path = os.path.join( - self.config.persistent_dir, - "eval_results", - "report.json", - ) + def postprocess_after_run(self, report_file: Path) -> None: + """Parse SWE-rebench test output on the host and write a resolution report. - return ExecuteContainerCommandArgs( - command=cmd, - expected_file_pattern=search_path, - mode="eval", - timeout=self.config.swebench_tests_timeout, - ) + Loads the instance-specific log parser, normalizes the parsed test names, compares + them against the expected fail-to-pass and pass-to-pass sets, and writes a report + keyed by instance id. Parsing on the host avoids requiring Python inside the container. - def postprocess_after_run(self, report_file: Path) -> None: - """Parse test output on the host (avoids needing python3 inside the container).""" + Args: + report_file: Path to the report file to write; the sibling ``test_output.log`` + in the same directory supplies the raw test output. + """ report_path = Path(report_file) test_output_path = report_path.parent / "test_output.log" @@ -903,128 +756,18 @@ def postprocess_after_run(self, report_file: Path) -> None: class SweBenchExtDatasetProcessor(BaseDatasetHarnessProcessor): """Dataset processor for SWE-Bench-Ext format tasks.""" - def _get_instance_dict(self) -> dict: - raw = self.config.problem_info.get("instance_dict", "{}") - if isinstance(raw, str): - return json.loads(raw) - return raw - - def get_run_command(self) -> ExecuteContainerCommandArgs: - from responses_api_agents.swe_agents.swe_bench_ext.frameworks import ( - get_framework_config, - get_test_command_with_output, - ) - - inst = self._get_instance_dict() - - base_command = inst.get("test_command", "") - base_commit = inst.get("base_commit", "") - test_patch = inst.get("test_patch", "") - test_framework = inst.get("test_framework", "") - - # Write test patch to persistent_dir (mounted into container) - test_patch_path = self.config.persistent_dir / "test_patch.diff" - test_patch_path.write_text(test_patch) - - # Write eval metadata for host-side postprocessing - fail_to_pass = inst.get("FAIL_TO_PASS", inst.get("fail_to_pass", [])) - pass_to_pass = inst.get("PASS_TO_PASS", inst.get("pass_to_pass", [])) - if isinstance(fail_to_pass, str): - fail_to_pass = json.loads(fail_to_pass) - if isinstance(pass_to_pass, str): - pass_to_pass = json.loads(pass_to_pass) - - eval_meta_dir = self.config.persistent_dir / "eval_meta" - eval_meta_dir.mkdir(parents=True, exist_ok=True) - (eval_meta_dir / "fail_to_pass.json").write_text(json.dumps(fail_to_pass)) - (eval_meta_dir / "pass_to_pass.json").write_text(json.dumps(pass_to_pass)) - (eval_meta_dir / "test_framework.txt").write_text(test_framework) - - reset_cmd = f"git reset --hard {base_commit}" if base_commit else "" - - # Use lighthouse to add structured output flags (--json, --junitxml, etc.) - # This is the same transformation swe_bench_ext_agent/task.py applies. - test_cmd = get_test_command_with_output(base_command, test_framework) - config = get_framework_config(test_framework, base_command) - result_file = config.get("result_file") - - # Build the result file dump block (mirrors task.py's generate_test_run_script) - result_file_block = "" - if result_file: - if "*" in result_file: - result_file_block = f""" -echo "<<>>" -for f in {result_file}; do - if [ -f "$f" ]; then - echo "=== FILE: $f ===" - cat "$f" - echo "" - fi -done 2>/dev/null || true -echo "<<>>" -""" - else: - result_file_block = f""" -echo "<<>>" -if [ -f "{result_file}" ]; then - cat "{result_file}" -fi -echo "<<>>" -""" - - cmd = f"""#!/bin/bash -set -o pipefail - -date +\"%s.%N\" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} - -{self._get_command_sleep_until_predictions_file()} - -# Try common repo locations in the container -cd /testbed 2>/dev/null || cd /workspace/repo 2>/dev/null || cd /app 2>/dev/null || true - -# Reset to base commit if specified -{reset_cmd} - -# Apply model patch (agent output or golden patch) -git apply --reject --recount --ignore-space-change --ignore-whitespace /root/patch.diff || true - -# Apply test patch (adds/modifies test files) -git apply --reject --recount --ignore-space-change --ignore-whitespace /root/test_patch.diff || true - -# Run tests with structured output and capture to log -mkdir -p /trajectories_mount/eval_results /workspace/test-results -set +e -( -echo "<<>>" -{test_cmd} -test_exit_code=$? -{result_file_block} -echo "<<>>" -exit $test_exit_code -) > /trajectories_mount/eval_results/test_output.log 2>&1 -TEST_EXIT=$? -set -e - -printf '{{"_test_completed": true, "exit_code": %d}}\\n' $TEST_EXIT \ - > /trajectories_mount/eval_results/report.json -""" - - search_path = os.path.join( - self.config.persistent_dir, - "eval_results", - "report.json", - ) + def postprocess_after_run(self, report_file: Path) -> None: + """Parse SWE-Bench-Ext test output on the host and write a resolution report. - return ExecuteContainerCommandArgs( - command=cmd, - expected_file_pattern=search_path, - mode="eval", - timeout=self.config.swebench_tests_timeout, - ) + Reads the test framework and the fail-to-pass and pass-to-pass sets from the + instance's eval metadata, parses the raw test output, and writes a report keyed by + instance id. Parsing on the host avoids requiring Python inside the container. - def postprocess_after_run(self, report_file: Path) -> None: - """Parse test output on the host using lighthouse's parsing library.""" - from responses_api_agents.swe_agents.swe_bench_ext.utils import parse_and_check_tests + Args: + report_file: Path to the report file to write; the sibling ``test_output.log`` + in the same directory supplies the raw test output. + """ + from responses_api_agents.swe_env.parsing import parse_and_check_tests report_path = Path(report_file) test_output_path = report_path.parent / "test_output.log" @@ -1062,7 +805,14 @@ def postprocess_after_run(self, report_file: Path) -> None: class OpenHandsHarnessProcessor(BaseDatasetHarnessProcessor): + """Harness processor that installs the OpenHands agent framework.""" + def setup(self) -> Path: + """Clone and install the OpenHands agent framework under a locked setup directory. + + Returns: + Path: The setup directory containing the prepared OpenHands install. + """ setup_dir = self.parent_dir / "swe_openhands_setup" with self._setup_directory_lock(setup_dir, "OpenHands"): @@ -1088,162 +838,6 @@ def setup(self) -> Path: return setup_dir - def get_run_command(self) -> ExecuteContainerCommandArgs: - data_point = self.config.problem_info - agent_run_id = self.config.agent_run_id - - agent_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), "configs/oh_config.toml") - - # Add parameters to config.toml - # TODO(sugam): is there a better way to do this? - with open(agent_config, "r") as f: - config = tomlkit.parse(f.read()) - - config["llm"]["model"] |= { - "model": self.config.body.model, - "base_url": "", # May need to populate this - "temperature": self.config.inference_params["temperature"], - "top_p": self.config.inference_params["top_p"], - } - - config_str = tomlkit.dumps(config) - - eval_dir_in_openhands = self.config.eval_dir_in_openhands - local_dataset_path = "/root/dataset/data.jsonl" - config_file_path = self.config.openhands_config_file_path - - assert self.config.openhands_setup_dir is not None, "OpenHands setup directory is not set" - - if self.config.debug: - profiling_cmd = f"export NG_PROFILING_DIR={self.config.profiling_mounted_dir} && " - else: - profiling_cmd = "" - - if self.config.openhands_should_log: - log_cmd = "export LOG_LEVEL=DEBUG && export LOG_TO_FILE=true && export NG_OPENHANDS_SHOULD_LOG=true && " - else: - log_cmd = ( - "export LOG_LEVEL=CRITICAL && " - "export DEBUG=False && " - "export DEBUG_LLM=False && " - "export LOG_TO_FILE=False && " - "export LOG_ALL_EVENTS=False && " - "export DEBUG_RUNTIME=False && " - ) - - if data_point["dataset_name"] == "nv-internal-1" or data_point["dataset_name"] == "swe-bench-ext": - crypto_fix_cmd = ( - "_crypto_fix_dir=$(mktemp -d /tmp/crypto_fix_XXXXXX) && " - "/openhands_setup/OpenHands/.venv/bin/python -m pip install " - " --target=$_crypto_fix_dir " - " --index-url https://pypi.org/simple " - " --trusted-host pypi.org --trusted-host files.pythonhosted.org " - " --only-binary :all: " - " --no-deps --no-cache-dir " - " --quiet " - " 'cryptography<43' && " - "export PYTHONPATH=$_crypto_fix_dir:${PYTHONPATH:-} &&" - ) - else: - crypto_fix_cmd = "" - - if self.config.resolved_diversify_tool_names: - diversify_tool_names_cmd = "export DIVERSIFY_TOOL_NAMES=true &&" - else: - diversify_tool_names_cmd = "" - - if self.config.resolved_camel_case_tool_names: - camel_case_tool_names_cmd = "export CAMEL_CASE_TOOL_NAMES=true &&" - else: - camel_case_tool_names_cmd = "" - - workspace_check_cmd = "" - - agent_main_cmd = ( - f"{workspace_check_cmd}" - # Add miniforge bin to PATH (for tmux, node, poetry, etc.) - "mkdir -p /tmp/ && " - "export PATH=/openhands_setup/miniforge3/bin:$PATH && " - # Setup tmux socket (OpenHands requirement) - "uid=$(id -ru 2>/dev/null || id -u) && " - "export TMUX_TMPDIR=/tmp && " - "export TMUX=/tmp/tmux-$uid/default && " - "mkdir -p /tmp/tmux-$uid && " - "chown $uid:$uid /tmp/tmux-$uid || true && " - "chmod 700 /tmp/tmux-$uid && " - "tmux -S /tmp/tmux-$uid/default start-server || true && " - "cp /openhands_setup/miniforge3/bin/jq /usr/local/bin/jq 2>/dev/null || true && " - # Use pre-built OpenHands - "cd /openhands_setup/OpenHands && " - "export RUNTIME=local && " - f'date +"%s.%N" > {self.config.generation_apptainer_spinup_timestamp_mounted_fpath} && ' - f"{log_cmd}" - f"{profiling_cmd}" - f"export NEMO_GYM_METRICS_FPATH={self.config.base_mounted_dir}/nemo_gym_metrics.json && " - f"export NEMO_GYM_CONFIG_DICT={self.config.ng_global_config_dict_str} && " - f"export NEMO_GYM_MODEL_SERVER_NAME={self.config.model_server_name} &&" - "export VIRTUAL_ENV=/openhands_setup/OpenHands/.venv && " - "export PATH=$PATH:/openhands_setup/OpenHands/.venv/bin && " - # CRITICAL: Configure poetry to only use the OpenHands venv (ignore external venvs) - "export POETRY_VIRTUALENVS_IN_PROJECT=true && " - "export POETRY_VIRTUALENVS_CREATE=false && " - "export POETRY_VIRTUALENVS_PATH=/openhands_setup/OpenHands && " - f"export TMUX_MEMORY_LIMIT={self.config.apptainer_memory_limit_mb} && " - f"export COMMAND_EXEC_TIMEOUT={self.config.command_exec_timeout} && " - f"{crypto_fix_cmd}" - f"{diversify_tool_names_cmd}" - f"{camel_case_tool_names_cmd}" - f"echo {shlex.quote(config_str)} >{config_file_path} && " - # f" export EVAL_OUTPUT_DIR={eval_dir_in_openhands} && " - f"./evaluation/benchmarks/swe_bench/scripts/run_infer.sh " - f" llm.model " # name of llm config section in config.toml - f" {self.config.agent_framework_commit} " # openhands commit - f" {self.config.resolved_agent_cls} " # agent - f" 0 " # Note: this is eval limit which randomly chooses an instance from the dataset - f" {self.config.agent_max_turns} " # max agent iterations - f" 1 " # number of workers - f" {data_point['dataset_name']} " # dataset name - f" {data_point['split']} " # dataset split - f" {eval_dir_in_openhands} " - f" {data_point['instance_id']} " - f" {local_dataset_path} " - f" {config_file_path}" - ) - - if self.config.resolved_user_prompt_template is not None: - agent_main_cmd += " /openhands_setup/OpenHands/user_prompt.j2 " - if self.config.resolved_user_prompt_template is not None: - agent_main_cmd += " /openhands_setup/OpenHands/system_prompt.j2 " - agent_main_cmd += " /openhands_setup/OpenHands/system_prompt_long_horizon.j2 " - - agent_script_name = f"agent_script_{agent_run_id}.sh" - agent_script_path = self.config.persistent_dir / agent_script_name - with open(agent_script_path, "w") as f: - f.write("#!/bin/bash\nset -e\n") - f.write(agent_main_cmd) - f.flush() - os.fsync(f.fileno()) - - agent_timeout_seconds = self.config.swebench_agent_timeout - openhands_cmd = ( - f"timeout --signal=TERM --kill-after=30 {agent_timeout_seconds} " - f"bash /trajectories_mount/{agent_script_name}" - ) - - search_path = os.path.join( - self.config.openhands_setup_dir / "OpenHands" / eval_dir_in_openhands, - "**", - "output.jsonl", - ) - - # Execute OpenHands command - return ExecuteContainerCommandArgs( - command=openhands_cmd, - expected_file_pattern=search_path, - mode="agent", - timeout=self.config.swebench_agent_timeout + 60, - ) - ######################################## # START Ray worker logic @@ -1251,6 +845,15 @@ def get_run_command(self) -> ExecuteContainerCommandArgs: def _classify_agent_error(err: Optional[str]) -> Optional[str]: + """Classify an agent error message into a coarse failure-mode category. + + Args: + err: The agent error message, or None/empty if the agent finished cleanly. + + Returns: + Optional[str]: One of ``"max_iteration"``, ``"context_window"``, + ``"stuck_in_loop"``, or ``"other"``; None when there is no error. + """ if not err: return None s = str(err) @@ -1263,6 +866,53 @@ def _classify_agent_error(err: Optional[str]) -> Optional[str]: return "other" +def _resolve_image_name(container_formatter: "str | list[str]", instance_id: str) -> str: + """Resolve a sandbox image name from a container formatter template. + + Substitutes the instance id into the template (replacing ``__`` with ``_1776_`` and + lowercasing) and strips a leading ``docker://`` scheme. For the default docker SWE-bench + formatter this yields a Docker Hub image name; apptainer/.sif resolution is owned by the provider. + + Args: + container_formatter: A template string, or a list whose first element is used, + optionally containing the ``{instance_id}`` placeholder. + instance_id: The task instance id to substitute into the template. + + Returns: + str: The resolved image name with any ``docker://`` prefix removed. + """ + fmt = container_formatter[0] if isinstance(container_formatter, list) else container_formatter + if "{instance_id}" in fmt: + fmt = fmt.format(instance_id=instance_id.replace("__", "_1776_").lower()) + return fmt[len("docker://") :] if fmt.startswith("docker://") else fmt + + +def _should_mask_sample( + resolved: bool, + agent_error_kind: Optional[str], + eval_timed_out: bool, + agent_timed_out: bool, +) -> bool: + """Decide whether to mask this sample from the GRPO gradient. + + A sample is masked when: the patch passed eval but the agent did not actually submit + (max-turns or context window), so the reward is accidental; the final eval timed out; + or the agent itself timed out on wall-clock. + + Args: + resolved: Whether the task was scored as resolved. + agent_error_kind: The classified agent failure mode, or None if the agent finished cleanly. + eval_timed_out: Whether evaluation timed out. + agent_timed_out: Whether the agent timed out. + + Returns: + bool: True if the sample should be masked. + """ + return bool( + (resolved and agent_error_kind in ("max_iteration", "context_window")) or eval_timed_out or agent_timed_out + ) + + @ray.remote( scheduling_strategy="SPREAD", runtime_env={ @@ -1271,7 +921,18 @@ def _classify_agent_error(err: Optional[str]) -> Optional[str]: num_cpus=0.1, ) def runner_ray_remote(params_dict: dict[str, Any]) -> Optional[Path]: - # For some reason Ray may not pick up the proper model fields if we don't rebuild the model here. Very strange. + """Ray remote entrypoint that runs a single task instance in a worker. + + Validates the instance config from the serialized dict and drives the OpenHands agent + for that task. + + Args: + params_dict: Serialized ``SWEBenchWrapperInstanceConfig`` for the task. + + Returns: + Optional[Path]: The report file path if one is produced, otherwise None. + """ + # Ray may not pick up the proper model fields unless the models are rebuilt here. SWEBenchWrapperInstanceConfig.model_rebuild(force=True) RunOpenHandsAgent.model_rebuild(force=True) @@ -1283,6 +944,15 @@ def runner_ray_remote(params_dict: dict[str, Any]) -> Optional[Path]: def update_metrics(metrics_fpath: Path, update_dict: Dict[str, Any]) -> None: + """Merge non-null metric values into the JSON metrics file on disk. + + Reads the existing metrics, drops null entries from both the existing and update + dicts, merges them (update values winning), and writes the result back. + + Args: + metrics_fpath: Path to the JSON metrics file to read and rewrite. + update_dict: Metric values to merge in; null values are ignored. + """ with metrics_fpath.open() as f: existing_dict = json.loads(f.read()) @@ -1305,18 +975,25 @@ def update_metrics(metrics_fpath: Path, update_dict: Dict[str, Any]) -> None: # return data -class ActiveContainerCommand(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) - - process: Process - log_file: Any - log_file_path: Path - - class RunOpenHandsAgent(BaseModel): + """Drives a single OpenHands agent run for one task instance and collects its output.""" + config: SWEBenchWrapperInstanceConfig def _openhands_dir_copy_from_host(self, output_file_path: Optional[str]) -> Optional[str]: + """Copy an OpenHands run's output and latest LLM completion off the eval directory. + + Locates the run's ``output.jsonl`` (falling back to the most recent one under the eval + directory) and copies it to the configured prediction path, copies the latest LLM + completion into the trajectories tree, then removes the eval directory and config file. + + Args: + output_file_path: Path to the run output file, relative to the eval directory or + absolute; if falsy, no output is copied. + + Returns: + Optional[str]: The destination path of the copied output, or None if none was copied. + """ data_point = self.config.problem_info eval_dir_in_openhands = self.config.eval_dir_in_openhands config_file_path = self.config.openhands_config_file_path @@ -1360,204 +1037,143 @@ def _openhands_dir_copy_from_host(self, output_file_path: Optional[str]) -> Opti return dest_output - async def _start_container_command( - self, command: ExecuteContainerCommandArgs, apptainer_cmd: str - ) -> ActiveContainerCommand: - # Stream output to log file as it appears - logs_dir = self.config.persistent_dir / "apptainer_logs" - logs_dir.mkdir(exist_ok=True) - log_file_path = logs_dir / f"{self.config.instance_id}_{command.mode}.log" - log_file = open(log_file_path, "w") - - process = await asyncio.create_subprocess_shell(apptainer_cmd, stdout=log_file, stderr=log_file) + async def process_single_datapoint(self) -> Optional[Path]: + """Run the agent (or substitute the golden patch) for this task instance. - return ActiveContainerCommand(process=process, log_file=log_file, log_file_path=log_file_path) + The agent runs in a single working sandbox, self-drives, and persists its patch and + agent metrics; the eval and reward happen later in ``run()`` via the verifier POST. When + ``verify_golden_patch`` is set, the sample's gold patch is substituted instead of running + the agent. - async def _finish_container_command( - self, active_command: ActiveContainerCommand, command: ExecuteContainerCommandArgs - ) -> str: - data_point = self.config.problem_info + Returns: + Optional[Path]: Always None; the patch is persisted to the metrics file rather than + returned as a report file. + """ + if self.config.verify_golden_patch: + return await self._run_golden_patch_verification() - try: - # Wait for completion with timeout - await asyncio.wait_for(active_command.process.communicate(), timeout=command.timeout) - except asyncio.TimeoutError: - if active_command.process.returncode is None: - active_command.process.kill() - await active_command.process.wait() - raise ValueError("Command timed out") - finally: - active_command.log_file.close() + return await self._run_decoupled_agent() - if active_command.process.returncode != 0: - raise RuntimeError( - f"Command failed with return code {active_command.process.returncode}. " - f"Logs:\n{active_command.log_file_path.read_text(errors='replace')}" - ) + async def _run_decoupled_agent(self) -> Optional[Path]: + """Provision one working sandbox, self-drive OpenHands, and persist the extracted patch. - # Look for the expected file - pred_files = glob.glob(command.expected_file_pattern, recursive=True) - - if len(pred_files) == 1: - return pred_files[0] - elif len(pred_files) > 1: - latest_file = max(pred_files, key=os.path.getmtime) - print( - f"Multiple outputs found for {data_point['instance_id']} " - f"({len(pred_files)}). Using latest: {latest_file}", - flush=True, - ) - return latest_file - else: - raise ValueError( - f"Expected exactly one file matching {command.expected_file_pattern} for {data_point['instance_id']}, " - f"found {len(pred_files)}." - ) + Builds the task and launch command, stages the agent config and dataset files, provisions + the sandbox via the swe_env infra, runs OpenHands locally inside it, and records the + extracted patch and agent-side metrics (timing, error classification, timeout). The + eval and reward happen in ``run()`` via a POST to the verifier. - async def _kill_active_command(self, active_command: ActiveContainerCommand) -> None: - if active_command.process.returncode is None: - active_command.process.kill() - await active_command.process.wait() - active_command.log_file.close() + Returns: + Optional[Path]: Always None; the patch is persisted to the metrics file. + """ + from responses_api_agents.swe_agents.swe_env_adapter import ( + build_openhands_launch_command, + openhands_config_toml, + provision_and_collect, + ) + from responses_api_agents.swe_env.harness import SweTask - async def process_single_datapoint(self) -> Optional[Path]: - if self.config.verify_golden_patch: - return await self._run_golden_patch_verification() + def _as_list(v): + if isinstance(v, str): + try: + return json.loads(v) + except json.JSONDecodeError: + return [v] + return v or [] - instance_id = self.config.instance_id - if self.config.debug: - profiler = Profiler(name=instance_id, base_profile_dir=self.config.profiling_mounted_dir) - profiler.start() + data_point = self.config.problem_info + instance_dict = json.loads(data_point["instance_dict"]) + setup_dir = str(self.config.openhands_setup_dir) + gym_root = str(Path(setup_dir).resolve().parents[2]) metrics = SWEBenchMetrics(ray_queue_time=time.time() - self.config.ray_queue_timestamp) - metrics.openhands_run_time = -time.time() - metrics.generation_apptainer_spinup_time = metrics.openhands_run_time - metrics.final_eval_apptainer_spinup_time = metrics.openhands_run_time - openhands_active_command = await self._start_container_command( - self.config.agent_command, self.config.agent_apptainer_command_str + # Provider: explicit config, else docker with the Gym repo bind-mounted at its host path + # (resolves OpenHands' venv abs-symlinks + the nemo_gym editable install) + host network. + provider = self.config.sandbox_provider or { + "docker": {"network": "host", "run_args": ["-v", f"{gym_root}:{gym_root}:ro"]} + } + task = SweTask( + instance_id=self.config.instance_id, + image=_resolve_image_name(self.config.container_formatter, self.config.instance_id), + base_commit=instance_dict.get("base_commit", "") or "", + repo_workdir="/testbed", + test_command="", + model_patch="", + test_patch=instance_dict.get("test_patch", "") or "", + fail_to_pass=_as_list(instance_dict.get("FAIL_TO_PASS")), + pass_to_pass=_as_list(instance_dict.get("PASS_TO_PASS")), + benchmark=data_point["dataset_name"], + split=data_point.get("split", "test"), + metadata={"ttl_s": self.config.swebench_agent_timeout + 600, "ready_timeout_s": 900}, ) - eval_active_command = await self._start_container_command( - self.config.eval_command, self.config.eval_apptainer_command_str + launch = build_openhands_launch_command( + setup_dir=setup_dir, + instance_id=self.config.instance_id, + dataset_name=data_point["dataset_name"], + split=data_point.get("split", "test"), + ng_config_dict_quoted=self.config.ng_global_config_dict_str, + model_server_name=self.config.model_server_name, + agent_cls=self.config.resolved_agent_cls, + max_iter=self.config.agent_max_turns, + command_exec_timeout=self.config.command_exec_timeout, + tmux_memory_limit_mb=self.config.apptainer_memory_limit_mb, ) + stage_files = { + "/root/config.toml": openhands_config_toml( + self.config.body.model, + temperature=self.config.inference_params.get("temperature", 0.0), + top_p=self.config.inference_params.get("top_p", 1.0), + ), + "/root/dataset/data.jsonl": json.dumps(instance_dict), + } try: - out_file_in_eval = await self._finish_container_command( - openhands_active_command, self.config.agent_command + result = await provision_and_collect( + task, + provider=provider, + agent_launch_command=launch, + stage_files=stage_files, + patch_output_glob="/root/eval_results", + agent_timeout_s=self.config.swebench_agent_timeout, ) - out_file = self._openhands_dir_copy_from_host(output_file_path=out_file_in_eval) - except Exception as e: - print(f"Agent command failed for {instance_id}: {e}", flush=True) - try: - self._openhands_dir_copy_from_host(output_file_path=None) - except Exception: - pass - await self._kill_active_command(eval_active_command) + patch = result.get("patch") or None + if patch and not patch.endswith("\n"): + patch += "\n" metrics.openhands_run_time += time.time() - metrics.patch_exists = False - metrics.final_eval_apptainer_spinup_time = None - # Detect wall-clock agent timeout: openhands_run_time (elapsed since start) - # reached or exceeded the configured swebench_agent_timeout. - metrics.agent_timed_out = ( + metrics.model_patch = patch + metrics.patch_exists = bool(patch) + metrics.agent_error_kind = _classify_agent_error(result.get("agent_error")) + # The environment does not raise on agent timeout; it returns error_type="timeout" + # instead, so the except below never fires for a timed-out agent. Recover the timeout + # signal from the provision result's error_type, with a wall-clock fallback, so the + # sample is masked correctly. + metrics.agent_timed_out = result.get("error_type") == "timeout" or ( metrics.openhands_run_time is not None and metrics.openhands_run_time >= self.config.swebench_agent_timeout ) - update_metrics(self.config.metrics_fpath, metrics.model_dump()) - if self.config.debug: - profiler.stop() - return None - - generation_apptainer_spinup_timestamp = float( - self.config.generation_apptainer_spinup_timestamp_fpath.read_text() - ) - metrics.generation_apptainer_spinup_time += generation_apptainer_spinup_timestamp - metrics.openhands_run_time += time.time() - - with open(out_file, "r") as f: - out_dict = json.loads(f.read().strip()) - - metrics.agent_error_kind = _classify_agent_error(out_dict.get("error")) - - patch = out_dict["test_result"]["git_patch"] or None - patch = patch + "\n" if patch and not patch.endswith("\n") else patch - metrics.model_patch = patch - - # Create file in the SWE-bench evaluation format - self.config.output_for_eval_path.parent.mkdir(parents=True, exist_ok=True) - with self.config.output_for_eval_path.open("w") as f: - f.write( - json.dumps( - { - "model_name_or_path": out_dict["metadata"]["llm_config"]["model"], - "instance_id": out_dict["instance_id"], - "model_patch": patch, - "oh_time_metrics": out_dict["metrics"], - } - ) - ) - - # Dump out dot and png files from profiling on OpenHands level - if self.config.debug: - try: - profiling_name = "openhands" - callgrind_path = self.config.profiling_dir / f"{profiling_name}.callgrind" - callgrind_dotfile_path = self.config.profiling_dir / f"{profiling_name}.dot" - callgrind_graph_path = self.config.profiling_dir / f"{profiling_name}.png" - - gprof2dot_main( - argv=f"--format=callgrind --output={callgrind_dotfile_path} -e 5 -n 5 {callgrind_path}".split() - ) - - (graph,) = graph_from_dot_file(callgrind_dotfile_path) - graph.write_png(callgrind_graph_path) - except Exception as e: - print(f"Error dumping profiling files: {e}", flush=True) - - if not patch: + except Exception as e: # noqa: BLE001 + print(f"Decoupled agent run failed for {self.config.instance_id}: {e}", flush=True) + metrics.openhands_run_time += time.time() metrics.patch_exists = False - metrics.final_eval_apptainer_spinup_time = None - - await self._kill_active_command(eval_active_command) - - update_metrics(self.config.metrics_fpath, metrics.model_dump()) - return - - with open(self.config.model_patch_path, "w") as f: - f.write(patch) - - metrics.final_eval_time = -time.time() - try: - report_file = await self._finish_container_command(eval_active_command, self.config.eval_command) - except Exception as e: - print(f"Eval command failed for {instance_id}: {e}", flush=True) - metrics.final_eval_time += time.time() - metrics.patch_exists = True - # Detect wall-clock eval timeout: final_eval_time (elapsed since eval start) - # reached or exceeded the configured swebench_tests_timeout. - metrics.eval_timed_out = ( - metrics.final_eval_time is not None and metrics.final_eval_time >= self.config.swebench_tests_timeout + metrics.agent_timed_out = ( + metrics.openhands_run_time is not None + and metrics.openhands_run_time >= self.config.swebench_agent_timeout ) - update_metrics(self.config.metrics_fpath, metrics.model_dump()) - if self.config.debug: - profiler.stop() - return None - - final_eval_apptainer_spinup_timestamp = float( - self.config.final_eval_apptainer_spinup_timestamp_fpath.read_text() - ) - metrics.final_eval_apptainer_spinup_time += final_eval_apptainer_spinup_timestamp - metrics.final_eval_time += time.time() - - metrics.patch_exists = True update_metrics(self.config.metrics_fpath, metrics.model_dump()) + return None - if self.config.debug: - profiler.stop() + async def _run_golden_patch_verification(self) -> Optional[Path]: + """Skip the agent run and persist the sample's golden patch for verification. - return report_file + Writes the sample's gold patch (``instance_dict['patch']``) as the worker's + ``model_patch`` in the metrics file, exactly where ``_run_decoupled_agent`` would leave an + agent patch, so it is later graded by the verifier POST. Currently supported only for the + ``swe-bench-ext`` dataset. - async def _run_golden_patch_verification(self) -> Optional[Path]: + Returns: + Optional[Path]: Always None; the gold patch is persisted to the metrics file. + """ instance_id = self.config.instance_id dataset_name = self.config.problem_info.get("dataset_name") # TODO(sugam): add support for other datasets @@ -1576,45 +1192,11 @@ async def _run_golden_patch_verification(self) -> Optional[Path]: metrics = SWEBenchMetrics(ray_queue_time=time.time() - self.config.ray_queue_timestamp) metrics.model_patch = golden_patch metrics.patch_exists = True - - # Write golden patch where the agent would have written the model patch. - self.config.output_for_eval_path.parent.mkdir(parents=True, exist_ok=True) - with self.config.output_for_eval_path.open("w") as f: - f.write( - json.dumps( - { - "model_name_or_path": "golden_patch_verification", - "instance_id": instance_id, - "model_patch": golden_patch, - } - ) - ) - with open(self.config.model_patch_path, "w") as f: - f.write(golden_patch) - - metrics.final_eval_apptainer_spinup_time = -time.time() - metrics.final_eval_time = -time.time() - - eval_active_command = await self._start_container_command( - self.config.eval_command, self.config.eval_apptainer_command_str - ) - try: - report_file = await self._finish_container_command(eval_active_command, self.config.eval_command) - except Exception as e: - print(f"Golden-patch eval failed for {instance_id}: {e}", flush=True) - metrics.final_eval_time += time.time() - update_metrics(self.config.metrics_fpath, metrics.model_dump()) - return None - - final_eval_apptainer_spinup_timestamp = float( - self.config.final_eval_apptainer_spinup_timestamp_fpath.read_text() - ) - metrics.final_eval_apptainer_spinup_time += final_eval_apptainer_spinup_timestamp - metrics.final_eval_time += time.time() - + # No agent ran, so there is no agent error to classify (mask re-join stays clean). + metrics.agent_error_kind = None update_metrics(self.config.metrics_fpath, metrics.model_dump()) - return report_file + return None ######################################## @@ -1623,6 +1205,8 @@ async def _run_golden_patch_verification(self) -> Optional[Path]: class SWEBenchWrapper(SimpleResponsesAPIAgent): + """Responses API agent server that runs OpenHands on SWE-bench-style tasks and scores patches.""" + config: SWEBenchWrapperConfig _sem: Optional[Semaphore] = None @@ -1636,6 +1220,14 @@ class SWEBenchWrapper(SimpleResponsesAPIAgent): ######################################## def model_post_init(self, context: Any) -> None: + """Initialize per-run server state, run dataset/harness setup, and build helpers. + + Generates a run session id and results directory, runs the OpenHands and dataset + processor setups, and creates the concurrency semaphore and vLLM converter. + + Args: + context: The Pydantic post-init context passed through to the superclass. + """ run_session_id = f"{int(time.time() * 1000)}_{str(uuid.uuid4())[:8]}" workspace_root = Path(__file__).parent self._swe_bench_wrapper_server_config = SWEBenchWrapperServerConfig( @@ -1660,8 +1252,19 @@ def model_post_init(self, context: Any) -> None: ######################################## def get_openhands_trajectory_from_completions(self, trajectories_dir: Path, instance_id: str) -> tuple: - """ - This reads the trajectories directly dumped by OpenHands. + """Read the trajectory and tools from the LLM completion files dumped by OpenHands. + + Loads the most recent completion file for the instance, appends the final assistant + message (with any token-id and log-prob fields) to its message list, and returns the + messages together with the tool definitions. + + Args: + trajectories_dir: Directory containing the per-instance trajectory output. + instance_id: The task instance id whose completions are read. + + Returns: + tuple: A ``(messages, tools)`` pair; both are empty lists when no completion + files are found. """ messages, tools = [], [] @@ -1699,258 +1302,15 @@ def get_openhands_trajectory_from_completions(self, trajectories_dir: Path, inst # START Main methods ######################################## - def _find_container(self, data_point: dict) -> str: - """Find the container file using multiple strategies (Exact match > Fuzzy match). + def _resolve_absolute_path(self, path: Optional[str]) -> Optional[str]: + """Resolve a possibly relative path against the package parent directory. - Strategies: - 1. Replace "__" with "_1776_" (Original case, then Lowercase) - 2. Replace "__" with "_s_" (Original case, then Lowercase) - 3. Fuzzy search directory for .sif files matching above patterns. + Args: + path: A relative or absolute path string, or None. Returns: - str: Path to the container file. - - Raises: - FileNotFoundError: If no matching container file is found. + Optional[str]: The absolute path string, or None if no path was given. """ - instance_id = data_point["instance_id"] - container_formatters = data_point["container_formatter"] - - if isinstance(container_formatters, str): - container_formatters = [container_formatters] - - if "SWE-rebench" in data_point["dataset_name"]: - for container_formatter in container_formatters: - # Exact match: {instance_id}.sif (e.g. badges__shields-4557.sif) - container_path = container_formatter.format(instance_id=instance_id) - if os.path.exists(container_path): - return container_path - - # Fuzzy match: glob for files containing the instance_id - container_dir = os.path.dirname(container_formatter.format(instance_id="dummy")) - for pattern in [ - f"{instance_id}*.sif", - f"*{instance_id}*.sif", - ]: - matches = glob.glob(os.path.join(container_dir, pattern)) - if matches: - return matches[0] - raise FileNotFoundError( - f"No SIF found for SWE-rebench instance {instance_id}. " - f"Searched directories: {[os.path.dirname(cf.format(instance_id='dummy')) for cf in container_formatters]}" - ) - - if "R2E-Gym" in data_point["dataset_name"]: - instance_id_modified = re.sub( - r"[^_]+__([^-]+)-", lambda m: m.group(1).lower() + "_final_", data_point["instance_id"] - ) - for container_formatter in container_formatters: - container_name = container_formatter.format(instance_id=instance_id_modified) - if os.path.exists(container_name): - # print(f"container found: {container_name}", flush=True) - # print(f"container formatter: {container_formatter}", flush=True) - return container_name - - replacements = ["_1776_", "_s_"] - - # Generate all candidate IDs in order of priority - candidate_ids = [instance_id] - for replacement in replacements: - replaced_id = instance_id.replace("__", replacement) - candidate_ids.append(replaced_id) - candidate_ids.append(replaced_id.lower()) - - # Phase 1: Exact Matches - try all container formatters - for container_formatter in container_formatters: - for candidate_id in candidate_ids: - path = container_formatter.format(instance_id=candidate_id) - if os.path.exists(path): - return path - - # Phase 2: Fuzzy Search - try all container formatters - search_terms = [instance_id, instance_id.lower()] + candidate_ids - - for container_formatter in container_formatters: - # Define the default fallback path (Strategy 1, original case) - fallback_path = container_formatter.format(instance_id=instance_id.replace("__", replacements[0])) - container_dir = os.path.dirname(fallback_path) - - if os.path.exists(container_dir): - for term in search_terms: - pattern = os.path.join(container_dir, f"*{term}*.sif") - matches = glob.glob(pattern) - if matches: - return matches[0] - else: - if self.config.debug: - print(f"Container directory {container_dir} does not exist", flush=True) - - # Phase 3: Fallback - tried_paths = [] - for container_formatter in container_formatters: - for candidate_id in candidate_ids: - tried_paths.append(container_formatter.format(instance_id=candidate_id)) - - raise FileNotFoundError( - f"No container file found for instance_id {instance_id}. " - f"Tried the following candidate IDs: {candidate_ids}. " - f"Searched in paths: {tried_paths}." - ) - - def _build_apptainer_command( - self, params: SWEBenchWrapperInstanceConfig, command: ExecuteContainerCommandArgs - ) -> str: - dataset_path_to_mount = str(params.instance_dataset_path) - data_point = params.problem_info - - # Fix localhost URLs not working sometimes - container_commands = [] - container_commands.append("echo '127.0.0.1 localhost' >/etc/hosts") - - # Build mount arguments - mount_args = [ - f"--mount type=bind,src={params.persistent_dir},dst=/trajectories_mount", - ] - - openhands_dir = f"{params.openhands_setup_dir}/OpenHands" - mount_args.extend( - [ - # Read-only base mounts (parent first) - f"--mount type=bind,src={openhands_dir},dst=/openhands_setup/OpenHands,ro", - f"--mount type=bind,src={openhands_dir},dst={openhands_dir},ro", - f"--mount type=bind,src={openhands_dir}/.eval_sessions,dst=/openhands_setup/OpenHands/.eval_sessions", - f"--mount type=bind,src={openhands_dir}/.eval_sessions,dst={openhands_dir}/.eval_sessions", - f"--mount type=bind,src={openhands_dir}/logs,dst=/openhands_setup/OpenHands/logs", - f"--mount type=bind,src={openhands_dir}/logs,dst={openhands_dir}/logs", - f"--mount type=bind,src={openhands_dir}/evaluation/oh,dst=/openhands_setup/OpenHands/evaluation/oh", - f"--mount type=bind,src={openhands_dir}/evaluation/oh,dst={openhands_dir}/evaluation/oh", - # Data - f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl", - ] - ) - - if params.resolved_user_prompt_template: - mount_args.append( - f"--mount type=bind,src={params.resolved_user_prompt_template},dst=/openhands_setup/OpenHands/user_prompt.j2" - ) - if params.resolved_system_prompt_template: - mount_args.append( - f"--mount type=bind,src={params.resolved_system_prompt_template},dst=/openhands_setup/OpenHands/system_prompt.j2" - ) - mount_args.append( - f"--mount type=bind,src={params.resolved_system_prompt_template},dst=/openhands_setup/OpenHands/system_prompt_long_horizon.j2" - ) - - miniforge3_path = Path(params.openhands_setup_dir) / "miniforge3" - mount_args.append(f"--mount type=bind,src={miniforge3_path},dst=/openhands_setup/miniforge3,ro") - mount_args.append(f"--mount type=bind,src={miniforge3_path},dst={miniforge3_path},ro") - - # Add SWE-bench setup directory mount if available (for evaluation) - # swe-bench-ext and nv-internal-1 don't use the swebench harness - if command.mode == "eval" and data_point["dataset_name"] not in ("nv-internal-1", "swe-bench-ext"): - # Mount the entire setup directory at both /swebench_setup and its original absolute path - # This is needed because uv venv has hardcoded absolute paths - mount_args.append(f"--mount type=bind,src={params.swebench_setup_dir},dst=/swebench_setup") - mount_args.append(f"--mount type=bind,src={params.swebench_setup_dir},dst={params.swebench_setup_dir}") - - if command.mode == "eval" and "SWE-bench_Multilingual" in data_point["dataset_name"]: - mount_args.append( - f"--mount type=bind,src={params.swebench_multilingual_setup_dir},dst=/swebench_multilingual_setup" - ) - mount_args.append( - f"--mount type=bind,src={params.swebench_multilingual_setup_dir},dst={params.swebench_multilingual_setup_dir}" - ) - - if command.mode == "eval" and data_point["dataset_name"] == "nv-internal-1": - run_script_path = params.persistent_dir / "run_script.sh" - parsing_script_path = params.persistent_dir / "parsing_script.py" - - # Placeholder needed: eval container starts before agent writes the patch - params.model_patch_path.write_text("") - - mount_args.append(f"--mount type=bind,src={run_script_path},dst=/root/run_script.sh") - mount_args.append(f"--mount type=bind,src={parsing_script_path},dst=/root/parsing_script.py") - mount_args.append(f"--mount type=bind,src={params.model_patch_path},dst=/root/patch.diff") - - if command.mode == "eval" and "R2E-Gym" in data_point["dataset_name"]: - # Mount the entire setup directory at both /r2egym_setup and its original absolute path - # This is needed because uv venv has hardcoded absolute paths in its wrappers - # print(f"Mounting R2E-Gym setup directory from: {self.r2e_gym_setup_dir}", flush=True) - mount_args.append(f"--mount type=bind,src={params.r2e_gym_setup_dir},dst=/r2egym_setup") - mount_args.append(f"--mount type=bind,src={params.r2e_gym_setup_dir},dst={params.r2e_gym_setup_dir}") - - if command.mode == "eval" and "SWE-rebench" in data_point["dataset_name"]: - rebench_setup_dir = params.swe_rebench_setup_dir - mount_args.append(f"--mount type=bind,src={rebench_setup_dir},dst=/swe_rebench_setup,ro") - - test_patch_path = params.persistent_dir / "test_patch.diff" - # model_patch_path placeholder needed: eval container starts before agent writes the patch - if not params.model_patch_path.exists(): - params.model_patch_path.write_text("") - mount_args.append(f"--mount type=bind,src={test_patch_path},dst=/root/test_patch.diff") - mount_args.append(f"--mount type=bind,src={params.model_patch_path},dst=/root/patch.diff") - - # Mount eval metadata files explicitly (directory bind mounts may not expose subdirs on Lustre) - eval_meta_dir = params.persistent_dir / "eval_meta" - mount_args.append( - f"--mount type=bind,src={eval_meta_dir / 'expected_passed.json'},dst=/eval_meta/expected_passed.json,ro" - ) - mount_args.append( - f"--mount type=bind,src={eval_meta_dir / 'fail_to_pass.json'},dst=/eval_meta/fail_to_pass.json,ro" - ) - mount_args.append( - f"--mount type=bind,src={eval_meta_dir / 'pass_to_pass.json'},dst=/eval_meta/pass_to_pass.json,ro" - ) - - if command.mode == "eval" and data_point.get("dataset_name") == "swe-bench-ext": - test_patch_path = params.persistent_dir / "test_patch.diff" - if not params.model_patch_path.exists(): - params.model_patch_path.write_text("") - mount_args.append(f"--mount type=bind,src={test_patch_path},dst=/root/test_patch.diff") - mount_args.append(f"--mount type=bind,src={params.model_patch_path},dst=/root/patch.diff") - - if command.mode == "agent" and "R2E-Gym" in data_point["dataset_name"]: - # Remove R2E-Gym test-related files. - for root_dir in ["", "/root", "/testbed"]: - container_commands.append( - # /r2e_tests contains evaluation tests that the agent should not see. - f"rm -rf {root_dir}/r2e_tests && " - # run_tests.sh launches the tests in /r2e_tests, so the agent should not see this either. - # We check that it contains the substring "r2e_tests" - # to avoid accidentally deleting an unrelated file with that name. - f"if grep -qs r2e_tests {root_dir}/run_tests.sh; then rm -rf {root_dir}/run_tests.sh; fi" - ) - container_commands.append(command.command) - combined_command = " && ".join(container_commands) - - script_dir = params.persistent_dir / "container_scripts" - script_dir.mkdir(parents=True, exist_ok=True) - script_path = script_dir / f"{command.mode}_script.sh" - script_path.write_text(combined_command) - container_script_path = f"/container_scripts/{command.mode}_script.sh" - mount_args.append(f"--mount type=bind,src={script_path},dst={container_script_path},ro") - - mount_str = " ".join(mount_args) - - env_args = "" - if "SWE-rebench" in data_point["dataset_name"]: - env_args = "--env _JAVA_OPTIONS=-Djava.net.preferIPv6Addresses=false " - - # Launch Apptainer container and execute the script file - apptainer_cmd = ( - f"apptainer exec --writable-tmpfs --cleanenv --pid --no-mount home,tmp,bind-paths " - f"{env_args}" - f"{mount_str} " - f" {params.container} bash {container_script_path}" - ) - memory_limit_mb = params.apptainer_memory_limit_mb - if memory_limit_mb is not None and memory_limit_mb > 0: - memory_limit_kb = int(memory_limit_mb) * 1024 - apptainer_cmd = f"ulimit -v {memory_limit_kb} && {apptainer_cmd}" - - return apptainer_cmd - - def _resolve_absolute_path(self, path: Optional[str]) -> Optional[str]: if not path: return None p = Path(path) @@ -1961,6 +1321,19 @@ def _resolve_absolute_path(self, path: Optional[str]) -> Optional[str]: def _setup_params( self, body: NeMoGymResponseCreateParamsNonStreaming ) -> Tuple[SWEBenchWrapperInstanceConfig, BaseDatasetHarnessProcessor]: + """Build the per-instance config and select the dataset processor for a request. + + Creates the persistent working directory, writes the instance dataset file, maps + inference parameters from the Responses request, resolves any prompt overrides, and + chooses the dataset processor based on the dataset name. + + Args: + body: The Responses API create-params request carrying the task metadata. + + Returns: + Tuple[SWEBenchWrapperInstanceConfig, BaseDatasetHarnessProcessor]: The resolved + per-instance config and the matching dataset processor. + """ problem_info = body.metadata | {"container_formatter": self.config.container_formatter} instance_id = problem_info.get("instance_id", "unknown") @@ -1999,8 +1372,6 @@ def _setup_params( if value is not None: inference_params[key] = value - container = self._find_container(problem_info) - eval_dir_in_openhands = f"evaluation/oh/{agent_run_id}" openhands_config_file_path = f"/tmp/config_{agent_run_id}.toml" @@ -2029,7 +1400,6 @@ def _setup_params( output_for_eval_path=output_for_eval_path, prediction_path=prediction_path, model_patch_path=persistent_dir / "patch.diff", - container=container, eval_dir_in_openhands=eval_dir_in_openhands, openhands_config_file_path=openhands_config_file_path, agent_script_path=agent_script_path, @@ -2069,16 +1439,22 @@ def _setup_params( else: dataset_processor = SweBenchDatasetProcessor(config=params) - params.eval_command = dataset_processor.get_run_command() - params.eval_apptainer_command_str = self._build_apptainer_command(params, params.eval_command) - - params.agent_command = OpenHandsHarnessProcessor(config=params).get_run_command() - params.agent_apptainer_command_str = self._build_apptainer_command(params, params.agent_command) - params.agent_script = params.agent_script_path.read_text() - + # The agent launch and patch egress are owned by swe_env_adapter; eval is the verifier POST. return params, dataset_processor async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()) -> NeMoGymResponse: + """Handle a Responses request: run the agent for the task and return the response. + + Sets up the per-instance config, persists it, and delegates to the inner handler. On + failure the traceback is written to the persistent directory before the exception is + re-raised. + + Args: + body: The Responses API create-params request carrying the task metadata. + + Returns: + NeMoGymResponse: The response containing the agent trajectory and task metrics. + """ params, dataset_processor = self._setup_params(body) with (params.persistent_dir / "params.json").open("w") as f: @@ -2095,13 +1471,125 @@ async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body() raise e + async def _verify_patch_via_server(self, params: SWEBenchWrapperInstanceConfig) -> Dict[str, Any]: + """POST the worker's patch to the swe_env verifier and return its eval subset. + + Builds a verify request carrying the per-task metadata the verifier reads plus the patch + in ``response.metadata.model_patch``, forwarding the instance's own per-framework test + command and framework when present and falling back to a conda+pytest default otherwise. + The call is bounded by a timeout. On any transport failure it returns a masked subset + (``resolved=False``, ``error_kind='sandbox'``) rather than raising, so the agent always + emits a present (masked) row instead of dropping the rollout. + + Args: + params: The resolved per-instance config holding the patch and task metadata. + + Returns: + Dict[str, Any]: The verifier's eval subset, or a masked subset on failure. + """ + persisted = SWEBenchMetrics.model_validate_json(params.metrics_fpath.read_text()) + patch = persisted.model_patch or "" + instance_dict = json.loads(params.problem_info["instance_dict"]) + + def _as_list(v): + if isinstance(v, str): + try: + return json.loads(v) + except json.JSONDecodeError: + return [v] + return v or [] + + f2p = _as_list(instance_dict.get("FAIL_TO_PASS")) + p2p = _as_list(instance_dict.get("PASS_TO_PASS")) + # Forward the instance's own per-framework eval command and framework when present; only + # fall back to the conda+pytest default when the row ships no test_command. This supports + # both SWE-bench-Verified (no per-row command) and multi-framework swe-bench-ext rows + # (cargo/go/npm/... that carry their own command and framework). + test_framework = instance_dict.get("test_framework", "") or "" + test_command = instance_dict.get("test_command", "") or "" + if not test_command: + nodeids = " ".join("'" + n + "'" for n in f2p + p2p) + test_command = ( + "source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && " + f"python -m pytest -rA {nodeids}" + ) + task_metadata = { + "instance_id": params.instance_id, + "image": _resolve_image_name(params.container_formatter, params.instance_id), + "base_commit": instance_dict.get("base_commit", "") or "", + "repo_workdir": "/testbed", + "test_command": test_command, + "test_framework": test_framework, + "test_patch": instance_dict.get("test_patch", "") or "", + "fail_to_pass": f2p, + "pass_to_pass": p2p, + "benchmark": params.problem_info["dataset_name"], + "split": params.problem_info.get("split", "test"), + } + verify_request = { + "responses_create_params": params.body.model_dump() | {"metadata": task_metadata}, + "response": { + "id": f"swebench-{params.instance_id}", + "created_at": int(time.time()), + "model": params.body.model, + "object": "response", + "output": [], + "metadata": {"model_patch": patch}, + }, + } + + async def _do_verify() -> Dict[str, Any]: + """POST the verify request and return the parsed JSON response. + + Returns: + Dict[str, Any]: The verifier's JSON response body. + """ + verify_response = await self.server_client.post( + server_name=params.verifier_server_name, + url_path="/verify", + json=verify_request, + ) + await raise_for_status(verify_response) + return await get_response_json(verify_response) + + # Bound the whole call (including the client's disconnect-retry loop) so a hung or retried + # verify cannot pin a rollout slot indefinitely. + verify_timeout_s = float(getattr(params, "swebench_tests_timeout", None) or 900) + 900 + try: + return await asyncio.wait_for(_do_verify(), timeout=verify_timeout_s) + except Exception as e: # noqa: BLE001 (incl. asyncio.TimeoutError -> masked, never pins a slot) + print(f"Verifier POST failed for {params.instance_id}: {e}", flush=True) + return {"resolved": False, "error_kind": "sandbox", "patch_exists": bool(patch)} + async def _inner_responses( self, params: SWEBenchWrapperInstanceConfig, dataset_processor: BaseDatasetHarnessProcessor ) -> NeMoGymResponse: + """Run the agent worker, score the patch, decide masking, and build the response. + + Dispatches the task to the Ray worker, grades the resulting patch either via the verifier + POST or by post-processing the worker's report, decides whether to mask the sample from the + GRPO gradient, reconstructs the trajectory and tool definitions, updates the metrics file, + and assembles the response. + + Args: + params: The resolved per-instance config for the task. + dataset_processor: The dataset processor used to post-process worker output. + + Returns: + NeMoGymResponse: The response containing output items, tools, and metrics metadata. + """ maybe_report_file = await runner_ray_remote.remote(params.model_dump()) metrics_to_update = dict() - if maybe_report_file: + if params.eval_via_verifier: + # The worker persisted the patch (no in-worker eval); grade it by POSTing to the + # verifier. The resolved/eval signals feed the same metrics and mask logic below. + eval_subset = await self._verify_patch_via_server(params) + metrics_to_update["resolved"] = bool(eval_subset.get("resolved")) + metrics_to_update["eval_timed_out"] = eval_subset.get("error_kind") == "eval_timeout" + if eval_subset.get("patch_exists") is not None: + metrics_to_update["patch_exists"] = bool(eval_subset.get("patch_exists")) + elif maybe_report_file: dataset_processor.postprocess_after_run(maybe_report_file) report = json.loads(Path(maybe_report_file).read_text()) @@ -2114,20 +1602,14 @@ async def _inner_responses( metrics_to_update["resolved"] = False # Decide whether to mask this sample from the GRPO gradient. - # 1) Patch passed eval but agent did not actually submit (hit max-turns - # or blew the context window) — the reward is accidental. - # 2) Final eval step timed out — reward is unreliable. - # 3) Agent itself timed out (wall-clock) — mask regardless of resolved. persisted_metrics = SWEBenchMetrics.model_validate_json(params.metrics_fpath.read_text()) resolved_now = metrics_to_update.get("resolved", False) agent_error_kind = persisted_metrics.agent_error_kind - eval_timed_out = bool(persisted_metrics.eval_timed_out) + # eval_timed_out may come from a persisted metric or from the verifier POST (in + # metrics_to_update and not yet persisted); prefer the latter when present. + eval_timed_out = bool(metrics_to_update.get("eval_timed_out", persisted_metrics.eval_timed_out)) agent_timed_out = bool(persisted_metrics.agent_timed_out) - if ( - (resolved_now and agent_error_kind in ("max_iteration", "context_window")) - or eval_timed_out - or agent_timed_out - ): + if _should_mask_sample(resolved_now, agent_error_kind, eval_timed_out, agent_timed_out): params.mask_sample = True trajectories_dir = params.persistent_dir / "trajectories" @@ -2162,6 +1644,18 @@ async def _inner_responses( ) async def run(self, body: BaseRunRequest) -> SWEBenchVerifyResponse: + """Run one task end to end under the concurrency limit and return its reward and metrics. + + Acquires the concurrency semaphore, runs the agent via ``responses``, extracts the + trajectory metadata and metrics, and assembles a verify response whose reward is 1.0 when + the task resolved and 0.0 otherwise. + + Args: + body: The run request carrying the Responses create-params for the task. + + Returns: + SWEBenchVerifyResponse: The reward, response, metrics, and resolved instance config. + """ async with self._sem: body.responses_create_params.parallel_tool_calls = True body.responses_create_params.tool_choice = "auto" diff --git a/responses_api_agents/swe_agents/configs/swe_env_base.yaml b/responses_api_agents/swe_agents/configs/swe_env_base.yaml new file mode 100644 index 0000000000..c52469e943 --- /dev/null +++ b/responses_api_agents/swe_agents/configs/swe_env_base.yaml @@ -0,0 +1,39 @@ +# Shared SWE-bench environment configuration (single source of truth). +# +# This file holds the SWE env leaves that are genuinely constant across the +# OpenHands-based SWE-agent configs (swebench_openhands.yaml, +# swebench_multi_tools.yaml, swebench_openhands_training.yaml). Those configs +# pull each leaf in per-key via the `${inherit_from:}` OmegaConf +# directive (see nemo_gym/global_config.py::_recursively_swap_keys), e.g.: +# +# agent_framework_commit: ${inherit_from:swe_env_base.shared.openhands.agent_framework_commit} +# +# `${inherit_from:...}` resolves against the fully-merged global config, so this +# file must be co-loaded at launch. Each consuming config declares it in its own +# `config_paths:`, and load_extra_config_paths() pulls nested config_paths in +# transitively, so users still launch with just their one config (no usability +# regression) -- same pattern as benchmarks/gsm8k/config.yaml chaining to +# resources_servers/math_with_judge/configs/math_with_judge.yaml. +# +# The top-level `swe_env_base` key is intentionally NOT server-shaped (it has no +# responses_api_models / resources_servers / responses_api_agents key) so it is +# ignored by server-instance + almost-server detection and never started. +# +# NOTE: swebench_swe_agent.yaml is deliberately NOT a consumer of this file. +# It uses a different agent framework (the nv-SWE-agent fork, with a different +# agent_framework_repo/commit) and does not define apptainer_memory_limit_mb, +# command_exec_timeout, or swebench_agent_timeout, so it shares none of these +# constants. swebench_tests_timeout also legitimately differs (900 eval vs 1200 +# training) and is therefore left inline in each consuming config, not shared. +swe_env_base: + shared: + # Constants common to every OpenHands-based SWE env block. + apptainer_memory_limit_mb: 32768 + command_exec_timeout: 300 + swebench_agent_timeout: 1800 + # The OpenHands agent-framework fork pinned by the OpenHands configs. These + # are framework-specific (swebench_swe_agent.yaml pins a different fork) and + # so are scoped under `openhands`. + openhands: + agent_framework_repo: https://github.com/sdevare-nv/nv-OpenHands.git + agent_framework_commit: 25bacbc60f7491e562022d6021e155af6b92fccb # pragma: allowlist secret diff --git a/responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml b/responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml index 609a642cc7..32dcaf16db 100644 --- a/responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml +++ b/responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml @@ -1,27 +1,33 @@ # SWE-bench wrapper configuration for OpenHands +# Co-load the shared SWE env constants (single source of truth). This is pulled +# in transitively by load_extra_config_paths, so users still launch with just +# this config. The shared leaves below are referenced from within the +# &swe_agents_config anchor, so swe_agents_val (which merges the anchor via +# `<<: *swe_agents_config`) inherits the same ${inherit_from:...} references. +config_paths: + - responses_api_agents/swe_agents/configs/swe_env_base.yaml -# SWE-bench wrapper configuration for OpenHands swe_agents: responses_api_agents: swe_agents: &swe_agents_config entrypoint: app.py - + # Agent framework configuration agent_framework: openhands agent_config: responses_api_agents/swe_agents/configs/oh_config.toml agent_max_turns: 100 - agent_framework_repo: https://github.com/sdevare-nv/nv-OpenHands.git - agent_framework_commit: 25bacbc60f7491e562022d6021e155af6b92fccb # pragma: allowlist secret - + agent_framework_repo: ${inherit_from:swe_env_base.shared.openhands.agent_framework_repo} + agent_framework_commit: ${inherit_from:swe_env_base.shared.openhands.agent_framework_commit} # pragma: allowlist secret + # Container configuration container_formatter: ??? container_folder_path: null - swebench_agent_timeout: 1800 - swebench_tests_timeout: 900 - apptainer_memory_limit_mb: 32768 - command_exec_timeout: 300 - + swebench_agent_timeout: ${inherit_from:swe_env_base.shared.swebench_agent_timeout} + swebench_tests_timeout: 900 # eval value; training uses 1200 (intentionally NOT shared) + apptainer_memory_limit_mb: ${inherit_from:swe_env_base.shared.apptainer_memory_limit_mb} + command_exec_timeout: ${inherit_from:swe_env_base.shared.command_exec_timeout} + dataset_path: ??? agent_prompt_overrides: diff --git a/responses_api_agents/swe_agents/configs/swebench_openhands.yaml b/responses_api_agents/swe_agents/configs/swebench_openhands.yaml index 4ad834f500..f1edbe7e0b 100644 --- a/responses_api_agents/swe_agents/configs/swebench_openhands.yaml +++ b/responses_api_agents/swe_agents/configs/swebench_openhands.yaml @@ -1,24 +1,31 @@ # SWE-bench wrapper configuration for OpenHands + +# Co-load the shared SWE env constants (single source of truth). This is pulled +# in transitively by load_extra_config_paths, so users still launch with just +# this config. +config_paths: + - responses_api_agents/swe_agents/configs/swe_env_base.yaml + swe_agents: responses_api_agents: swe_agents: entrypoint: app.py - + # Agent framework configuration agent_framework: openhands agent_config: responses_api_agents/swe_agents/configs/oh_config.toml agent_max_turns: 100 - agent_framework_repo: https://github.com/sdevare-nv/nv-OpenHands.git - agent_framework_commit: 25bacbc60f7491e562022d6021e155af6b92fccb # pragma: allowlist secret - + agent_framework_repo: ${inherit_from:swe_env_base.shared.openhands.agent_framework_repo} + agent_framework_commit: ${inherit_from:swe_env_base.shared.openhands.agent_framework_commit} # pragma: allowlist secret + # Container configuration container_formatter: ??? container_folder_path: null - swebench_agent_timeout: 1800 - swebench_tests_timeout: 900 - apptainer_memory_limit_mb: 32768 - command_exec_timeout: 300 - + swebench_agent_timeout: ${inherit_from:swe_env_base.shared.swebench_agent_timeout} + swebench_tests_timeout: 900 # eval value; training uses 1200 (intentionally NOT shared) + apptainer_memory_limit_mb: ${inherit_from:swe_env_base.shared.apptainer_memory_limit_mb} + command_exec_timeout: ${inherit_from:swe_env_base.shared.command_exec_timeout} + dataset_path: ??? # Optional model server reference diff --git a/responses_api_agents/swe_agents/configs/swebench_openhands_training.yaml b/responses_api_agents/swe_agents/configs/swebench_openhands_training.yaml index 69fa78ee1f..60eff871b4 100644 --- a/responses_api_agents/swe_agents/configs/swebench_openhands_training.yaml +++ b/responses_api_agents/swe_agents/configs/swebench_openhands_training.yaml @@ -1,4 +1,12 @@ # SWE-bench wrapper configuration for OpenHands + +# Co-load the shared SWE env constants (single source of truth). This is pulled +# in transitively by load_extra_config_paths, so users still launch with just +# this config. NOTE: swebench_tests_timeout (1200) intentionally differs from +# the eval configs (900) and is therefore left inline in both blocks below. +config_paths: + - responses_api_agents/swe_agents/configs/swe_env_base.yaml + swe_agents_train: responses_api_agents: swe_agents: @@ -7,15 +15,15 @@ swe_agents_train: agent_framework: openhands agent_config: responses_api_agents/swe_agents/configs/oh_config.toml agent_max_turns: 100 - agent_framework_repo: https://github.com/sdevare-nv/nv-OpenHands.git - agent_framework_commit: 25bacbc60f7491e562022d6021e155af6b92fccb # pragma: allowlist secret + agent_framework_repo: ${inherit_from:swe_env_base.shared.openhands.agent_framework_repo} + agent_framework_commit: ${inherit_from:swe_env_base.shared.openhands.agent_framework_commit} # pragma: allowlist secret # Container configuration container_formatter: ??? container_folder_path: null - swebench_agent_timeout: 1800 - swebench_tests_timeout: 1200 - apptainer_memory_limit_mb: 32768 - command_exec_timeout: 300 + swebench_agent_timeout: ${inherit_from:swe_env_base.shared.swebench_agent_timeout} + swebench_tests_timeout: 1200 # training value; eval configs use 900 (intentionally NOT shared) + apptainer_memory_limit_mb: ${inherit_from:swe_env_base.shared.apptainer_memory_limit_mb} + command_exec_timeout: ${inherit_from:swe_env_base.shared.command_exec_timeout} dataset_path: ??? agent_prompt_overrides: # # Codex agent @@ -59,17 +67,17 @@ swe_agents_val: agent_framework: openhands agent_config: responses_api_agents/swe_agents/configs/oh_config.toml agent_max_turns: 100 - agent_framework_repo: https://github.com/sdevare-nv/nv-OpenHands.git - agent_framework_commit: 25bacbc60f7491e562022d6021e155af6b92fccb # pragma: allowlist secret + agent_framework_repo: ${inherit_from:swe_env_base.shared.openhands.agent_framework_repo} + agent_framework_commit: ${inherit_from:swe_env_base.shared.openhands.agent_framework_commit} # pragma: allowlist secret # Container configuration container_formatter: ??? container_folder_path: null - swebench_agent_timeout: 1800 - swebench_tests_timeout: 1200 - apptainer_memory_limit_mb: 32768 - command_exec_timeout: 300 + swebench_agent_timeout: ${inherit_from:swe_env_base.shared.swebench_agent_timeout} + swebench_tests_timeout: 1200 # training value; eval configs use 900 (intentionally NOT shared) + apptainer_memory_limit_mb: ${inherit_from:swe_env_base.shared.apptainer_memory_limit_mb} + command_exec_timeout: ${inherit_from:swe_env_base.shared.command_exec_timeout} dataset_path: ??? - + agent_prompt_overrides: # CodeAct agent diff --git a/responses_api_agents/swe_agents/scripts/openhands_decoupled_rollout.py b/responses_api_agents/swe_agents/scripts/openhands_decoupled_rollout.py new file mode 100644 index 0000000000..a7fae3f117 --- /dev/null +++ b/responses_api_agents/swe_agents/scripts/openhands_decoupled_rollout.py @@ -0,0 +1,267 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reference driver: run OpenHands end-to-end through the swe_env infrastructure. + +Steps: + +1. Provision the agent's working container with the swe_env docker provider + + ``acquire_sandbox`` (host network for model egress; the Gym repo bind-mounted at its + host path so OpenHands' venv abs-symlinks + the ``nemo_gym`` editable install resolve). +2. Let OpenHands self-drive ``RUNTIME=local`` on ``/testbed`` (``--dataset SWE-Gym``). + Egress: the in-tree OpenHands ``CodeActAgent`` is hard-wired to ``NemoGymClient`` -> + ``ServerClient.post(server_name, "/v1/chat/completions")``, so we inject + ``NEMO_GYM_CONFIG_DICT`` (a crafted 3-level ``name.group.module.{host,port}`` map that + routes to a model server) + ``NEMO_GYM_MODEL_SERVER_NAME`` + ``NEMO_GYM_METRICS_FPATH`` + — NOT ``OPENAI_BASE_URL`` (there is no litellm fallback in that fork). +3. Extract the patch from ``output.jsonl[test_result][git_patch]`` (not ``git diff``). +4. Grade it in a separate fresh verifier sandbox via ``verify_task``. + +Prereqs: docker; a vLLM (or Gym model server) reachable at the host/port baked into the +NeMo Gym config; the official SWE-bench image for the instance; OpenHands set up under +swe_openhands_setup/. This is a manual reproduction/integration driver, not a CI test. + +Usage: + .venv/bin/python responses_api_agents/swe_agents/scripts/openhands_decoupled_rollout.py \ + --instance psf__requests-2317 --model Qwen/Qwen2.5-Coder-3B-Instruct \ + --model-host 127.0.0.1 --model-port 8000 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import time +from pathlib import Path + +import responses_api_agents.swe_env.providers # noqa: F401 registers sandbox providers +from nemo_gym.sandbox import SandboxSpec +from resources_servers.swe_env.verify_task import verify_task +from responses_api_agents.swe_agents.swe_env_adapter import run_self_driving +from responses_api_agents.swe_env.harness import SweTask + + +GYM = str(Path(__file__).resolve().parents[3]) +SETUP = f"{GYM}/responses_api_agents/swe_agents/swe_openhands_setup" + + +def _image_for(instance_id: str) -> str: + """Return the official SWE-bench docker image tag for an instance. + + Args: + instance_id: The benchmark instance identifier (e.g. ``psf__requests-2317``). + + Returns: + The fully qualified docker image tag for that instance. + """ + return "swebench/sweb.eval.x86_64." + instance_id.replace("__", "_1776_").lower() + ":latest" + + +def _as_list(v): + """Coerce a value into a list, parsing JSON-encoded strings when possible. + + Args: + v: A list, a JSON-encoded string, a plain string, or ``None``. + + Returns: + The parsed list. A JSON string is decoded; a non-JSON string is wrapped in a + single-element list; ``None`` becomes an empty list. + """ + if isinstance(v, str): + try: + return json.loads(v) + except json.JSONDecodeError: + return [v] + return v or [] + + +def _ng_config_dict(model_host: str, model_port: int) -> str: + """Build the NeMo Gym config dict JSON that routes ``ServerClient`` to a model server. + + Args: + model_host: Host of the model server reachable from the sandbox. + model_port: Port of the model server. + + Returns: + A JSON string of the config dict, where ``ServerClient`` resolves a server name three + levels deep (``cfg[name][group][module]`` -> ``{host, port}``). + """ + # ServerClient resolves server_name 3 levels deep: cfg[name][group][module] -> {host,port}. + cfg = { + "head_server": {"host": "127.0.0.1", "port": 9099}, + "vllm_model": {"responses_api_models": {"vllm_model": {"host": model_host, "port": model_port}}}, + } + return json.dumps(cfg) + + +def _config_toml(model: str, model_host: str, model_port: int) -> str: + """Build the OpenHands ``config.toml`` pointing at a model server's OpenAI endpoint. + + Args: + model: The model identifier to write into the config. + model_host: Host of the model server reachable from the sandbox. + model_port: Port of the model server. + + Returns: + The rendered ``config.toml`` contents as a string. + """ + return ( + "[llm.model]\n" + f'model = "{model}"\n' + f'base_url = "http://{model_host}:{model_port}/v1"\n' + 'api_key = "EMPTY"\n' # pragma: allowlist secret + 'custom_llm_provider = "openai"\n' + "native_tool_calling = false\n" + "temperature = 0.0\n" + "top_p = 1.0\n" + "log_completions = true\n" + 'log_completions_folder = "/root/completions"\n' + ) + + +def _launch_cmd(instance_id: str, model_host: str, model_port: int, max_iter: int) -> str: + """Build the in-sandbox bash that runs OpenHands ``run_infer.sh`` (RUNTIME=local). + + Args: + instance_id: The benchmark instance identifier to run. + model_host: Host of the model server reachable from the sandbox. + model_port: Port of the model server. + max_iter: Maximum agent iterations. + + Returns: + The shell command string to execute inside the sandbox. + """ + ng = json.dumps(_ng_config_dict(model_host, model_port)) # shell-safe quoted JSON literal + return ( + "set -e && " + f"export PATH={SETUP}/miniforge3/bin:$PATH && " + "git config --global --add safe.directory '*' && " # root container, host-owned bind mount + "mkdir -p /root/completions /root/dataset /root/eval_results && " + "uid=$(id -ru 2>/dev/null || id -u) && export TMUX_TMPDIR=/tmp && " + "export TMUX=/tmp/tmux-$uid/default && mkdir -p /tmp/tmux-$uid && chmod 700 /tmp/tmux-$uid && " + "tmux -S /tmp/tmux-$uid/default start-server || true && " + f"cd {SETUP}/OpenHands && export RUNTIME=local && " + "export LOG_LEVEL=INFO && export LOG_TO_FILE=False && export DEBUG=False && " + "export NEMO_GYM_METRICS_FPATH=/root/nemo_gym_metrics.json && echo '{}' > $NEMO_GYM_METRICS_FPATH && " + f"export NEMO_GYM_CONFIG_DICT={ng} && export NEMO_GYM_MODEL_SERVER_NAME=vllm_model && " + f"export VIRTUAL_ENV={SETUP}/OpenHands/.venv && export PATH=$PATH:{SETUP}/OpenHands/.venv/bin && " + "export POETRY_VIRTUALENVS_IN_PROJECT=true && export POETRY_VIRTUALENVS_CREATE=false && " + f"export POETRY_VIRTUALENVS_PATH={SETUP}/OpenHands && " + "export TMUX_MEMORY_LIMIT=8192 && export COMMAND_EXEC_TIMEOUT=300 && export PYTHONDONTWRITEBYTECODE=1 && " + "./evaluation/benchmarks/swe_bench/scripts/run_infer.sh " + f"llm.model '' CodeActAgent 0 {max_iter} 1 SWE-Gym test /root/eval_results " + f"{instance_id} /root/dataset/data.jsonl /root/config.toml" + ) + + +async def main(args): + """Run the full provision, self-drive, extract, and grade pipeline for one instance. + + Loads the instance from SWE-bench Verified, provisions a sandbox, stages the OpenHands + config and instance dataset, runs the agent, extracts the patch from ``output.jsonl``, + then grades it in a fresh verifier sandbox and prints the result. + + Args: + args: Parsed command-line arguments with ``instance``, ``model``, ``model_host``, + ``model_port``, ``max_iter``, and ``timeout`` attributes. + """ + from datasets import load_dataset + + ds = load_dataset("princeton-nlp/SWE-bench_Verified", split="test") + inst = next(r for r in ds if r["instance_id"] == args.instance) + image = _image_for(args.instance) + f2p, p2p = _as_list(inst.get("FAIL_TO_PASS")), _as_list(inst.get("PASS_TO_PASS")) + + # The agent self-drives, then run_self_driving extracts output.jsonl + grades in a fresh sandbox. + nodeids = " ".join("'" + n + "'" for n in f2p + p2p) + test_command = ( + f"source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && python -m pytest -rA {nodeids}" + ) + task = SweTask( + instance_id=args.instance, + image=image, + base_commit=inst["base_commit"], + repo_workdir="/testbed", + test_command=test_command, + test_patch=inst.get("test_patch", ""), + fail_to_pass=f2p, + pass_to_pass=p2p, + benchmark="swe-bench-ext", + metadata={"ttl_s": 3600, "ready_timeout_s": 900}, + ) + + provider = {"docker": {"network": "host", "run_args": ["-v", f"{GYM}:{GYM}:ro"]}} + # Write config.toml + instance dict via a pre-exec; run_self_driving runs the agent then extracts. + from responses_api_agents.swe_env.lifecycle import acquire_sandbox + + # Stage files into the same sandbox the agent uses: the agent needs config.toml + data.jsonl + # present first, so they are written into the live sandbox before launching the agent. + # Path: provision, stage, run agent, extract — done inline. + t0 = time.time() + spec = SandboxSpec(image=image, workdir="/testbed", ttl_s=args.timeout + 600, ready_timeout_s=900) + async with acquire_sandbox(provider, spec, instance_id=args.instance) as env: + print(f"[provision] {env.sandbox_id} ({time.time() - t0:.0f}s)", flush=True) + await env.write_text("/root/dataset/data.jsonl", json.dumps(dict(inst))) + await env.write_text("/root/config.toml", _config_toml(args.model, args.model_host, args.model_port)) + print("[launch] OpenHands run_infer.sh (RUNTIME=local) ...", flush=True) + await env.execute( + _launch_cmd(args.instance, args.model_host, args.model_port, args.max_iter), + cwd=f"{SETUP}/OpenHands", + timeout_s=args.timeout, + ) + from responses_api_agents.swe_agents.swe_env_adapter import _extract_patch_from_output_jsonl + + patch = await _extract_patch_from_output_jsonl(env, "/root/eval_results") + print(f"[patch] {len(patch)} bytes", flush=True) + + report = await verify_task({"docker": {}}, dataclasses_replace(task, patch)) + from responses_api_agents.swe_env.grading import reward_from_report + + print( + f"\n=== {args.instance}: resolved={report.resolved} patch_applied={report.patch_applied} " + f"error_kind={report.error_kind} REWARD={reward_from_report(report)} ===", + flush=True, + ) + + +def dataclasses_replace(task, patch): + """Return a copy of the task with its ``model_patch`` field set to ``patch``. + + Args: + task: The ``SweTask`` to copy. + patch: The unified-diff patch string to set on the copy. + + Returns: + A new ``SweTask`` with ``model_patch`` replaced. + """ + import dataclasses + + return dataclasses.replace(task, model_patch=patch) + + +if __name__ == "__main__": + # run_self_driving is the production entry point; this script stages files + drives the + # pipeline for a manual reproduction. (run_self_driving itself assumes config.toml/data.jsonl + # are baked into the image or the launch command; here they are staged into the live sandbox + # first.) + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--instance", default="psf__requests-2317") + p.add_argument("--model", default="Qwen/Qwen2.5-Coder-3B-Instruct") + p.add_argument("--model-host", default="127.0.0.1") + p.add_argument("--model-port", type=int, default=8000) + p.add_argument("--max-iter", type=int, default=30) + p.add_argument("--timeout", type=int, default=1800) + _ = run_self_driving # referenced for docs; staging path used here + asyncio.run(main(p.parse_args())) diff --git a/responses_api_agents/swe_agents/swe_bench_ext/__init__.py b/responses_api_agents/swe_agents/swe_bench_ext/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/responses_api_agents/swe_agents/swe_bench_ext/frameworks.py b/responses_api_agents/swe_agents/swe_bench_ext/frameworks.py deleted file mode 100644 index 7de570c491..0000000000 --- a/responses_api_agents/swe_agents/swe_bench_ext/frameworks.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test framework output configuration mapping.""" - -from typing import Dict - - -FRAMEWORK_CONFIGS: Dict[str, Dict] = { - "pytest": { - "output_flag": "--junitxml=/workspace/test-results/output.xml", - "result_file": "/workspace/test-results/output.xml", - }, - "unittest": { - "output_flag": "--junitxml=/workspace/test-results/output.xml", - "result_file": "/workspace/test-results/output.xml", - }, - "go": { - "output_flag": "-json", - "result_file": None, - }, - "jest": { - "output_flag": "--json --outputFile=/workspace/test-results/output.json", - "result_file": "/workspace/test-results/output.json", - }, - "vitest": { - "output_flag": "--reporter=json --outputFile=/workspace/test-results/output.json", - "result_file": "/workspace/test-results/output.json", - }, - "mocha": { - "output_flag": "--reporter json --reporter-options output=/workspace/test-results/output.json", - "result_file": "/workspace/test-results/output.json", - }, - "bun": { - "output_flag": None, # Bun doesn't have structured JSON output flag by default - "result_file": None, # Parse from stdout - }, - "junit": { - "output_flag": None, - "result_file": "find:/workspace/repo:*/target/surefire-reports:TEST-*.xml", - }, - "maven": { - "output_flag": None, - "result_file": "find:/workspace/repo:*/target/surefire-reports:TEST-*.xml", - }, - "gtest": { - "output_flag": "--gtest_output=json:/workspace/test-results/output.json", - "result_file": "/workspace/test-results/output.json", - }, - "cargo-nextest": { - "output_flag": None, # Profile is already in test_command - "result_file": None, # JUnit XML is output to repo/junit.xml by profile config - }, - "ctest": { - "output_flag": "--output-on-failure --output-junit /workspace/test-results/output.xml", - "result_file": "/workspace/test-results/output.xml", - }, - "xctest": { - # For SwiftPM with XCTest framework - "output_flag": "--parallel --num-workers=1 --xunit-output /workspace/test-results/output.xml", - "result_file": "/workspace/test-results/output.xml", - }, - "testing": { - # For SwiftPM with new Swift Testing framework (Swift 6+) - "output_flag": "--disable-xctest --parallel --xunit-output /workspace/test-results/output.xml", - "result_file": "/workspace/test-results/output.xml", - }, - "cppunit": { - "output_flag": None, - "result_file": None, - }, - # Lua test frameworks - Tier 1 (Standard XML output) - "busted": { - "output_flag": "--output=junit", - "result_file": "/workspace/test-results/output.xml", - }, - "luaunit": { - "output_flag": "-o junit -n /workspace/test-results/output.xml", - "result_file": "/workspace/test-results/output.xml", - }, - # Lua test frameworks - Tier 2 (Custom parsers) - "telescope": { - "output_flag": None, - "result_file": None, - }, - "lust": { - "output_flag": None, - "result_file": None, - }, - "minitest": { - "output_flag": None, - "result_file": None, - }, - "bespoke_libgeos": { - "output_flag": None, - "result_file": None, - }, - # TAP (Test Anything Protocol) - used by tape, node-tap - "tap": { - "output_flag": None, # TAP outputs to stdout - "result_file": None, # Parse from stdout - }, - "tape": { - "output_flag": None, # tape outputs TAP to stdout - "result_file": None, # Parse from stdout - }, - # Hardhat (Solidity) - uses Mocha under the hood - "hardhat": { - "output_flag": None, # Uses Mocha console reporter by default - "result_file": None, # Parse from stdout - }, -} - - -def get_framework_config(framework: str, test_command: str = "") -> Dict: - """Get configuration for a test framework. - - Args: - framework: Test framework name - test_command: The test command (optional, used to detect Gradle vs Maven) - """ - config = FRAMEWORK_CONFIGS.get( - framework, - { - "output_flag": None, - "result_file": None, - }, - ) - - # Special handling for JUnit: detect Gradle vs Maven from command - if framework == "junit" and test_command: - if "gradlew" in test_command or "gradle " in test_command: - # Gradle uses different output location than Maven - # Use */TEST-*.xml to match both standard Gradle (test/) and Android (testDebugUnitTest/) - config = { - "output_flag": None, - "result_file": "find:/workspace/repo:*/build/test-results*:TEST-*.xml", - } - - # Special handling for xctest: detect Swift Testing vs XCTest from command - # When --disable-xctest is used, the task is using Swift Testing, not XCTest - # Use the 'testing' framework config to avoid adding XCTest-only flags like --num-workers - if framework == "xctest" and test_command: - if "--disable-xctest" in test_command: - config = FRAMEWORK_CONFIGS.get("testing", config) - - return config - - -def get_test_command_with_output(base_command: str, framework: str) -> str: - """ - Add structured output flags to test command. - - Returns: command_with_output_flags - """ - config = get_framework_config(framework, base_command) - output_flag = config.get("output_flag") - - enhanced = f"{base_command} {output_flag}" if output_flag else base_command - - return enhanced diff --git a/responses_api_agents/swe_agents/swe_bench_ext/parsing.py b/responses_api_agents/swe_agents/swe_bench_ext/parsing.py deleted file mode 100644 index 800586adad..0000000000 --- a/responses_api_agents/swe_agents/swe_bench_ext/parsing.py +++ /dev/null @@ -1,1606 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Test parsing utilities for build.py. - -Helper functions for: -- Separating test and gold patches -- Parsing JUnit XML and JSON test outputs -""" - -import json -import re -import xml.etree.ElementTree as ET -from pathlib import Path -from typing import Dict, Optional, Tuple - - -def read_patch(path: Path, skip_binary: bool = False) -> str: - """ - Read the text content of a patch file optionally skipping binary files - """ - parts = split_patch(path, skip_binary=skip_binary) - return "".join([diff for _, diff in parts]) - - -def split_patch(patch_path: Path, skip_binary: bool = False) -> list[Tuple[str, str]]: - """ - Read a patch and partition by file. - - Args: - patch_path (Path) - The patch file to split - skip_binary (bool) - Whether to exclude binary files - - Returns: List of (filename, patch content) tuples - """ - content = patch_path.read_text() - parts = [] - - # Split by file changes (each starts with "diff --git") - file_diffs = re.split(r"(diff --git.*?)(?=diff --git|\Z)", content, flags=re.DOTALL) - - for i in range(0, len(file_diffs), 2): - if i + 1 >= len(file_diffs): - continue - - header = file_diffs[i] - content = file_diffs[i + 1] - full_diff = header + content - - # Extract filename from diff header - file_match = re.search(r"diff --git a/(.*?) b/", full_diff) - if not file_match: - continue - - filepath = file_match.group(1) - - if skip_binary: - binary_match = re.search(r"^GIT binary patch$", full_diff, flags=re.MULTILINE) - if binary_match: - continue - - parts.append((filepath, full_diff)) - - return parts - - -def _parse_embedded_test_results(text_output: str, test_prefix: str = "") -> Dict[str, str]: - """Parse embedded test results from system-out text. - - This handles cases like wolfssl where a single ctest testcase runs many individual tests - and outputs them in a specific format within . - - Expected formats: - - " 1: test_name : passed ( 0.00016)" - - " 2: test_name : failed ( 0.00016)" - - " 3: test_name : skipped" - - "HMAC-MD5 test passed!" - - "RSA test failed!" - - Args: - text_output: The text content from - test_prefix: Prefix to add to test names (usually the testcase name) - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - """ - results = {} - - # Pattern 1: Numbered test format (wolfssl API tests) - # Format: " 1: test_name : passed ( 0.00016)" - numbered_pattern = re.compile( - r"^\s*\d+:\s+([^\s:]+(?:\s+[^\s:]+)*?)\s*:\s*(passed|failed|skipped)", re.MULTILINE | re.IGNORECASE - ) - - for match in numbered_pattern.finditer(text_output): - test_name = match.group(1).strip() - status = match.group(2).lower() - - # Build test ID with prefix - if test_prefix: - test_id = f"{test_prefix}::{test_name}" - else: - test_id = test_name - - if status == "passed": - results[test_id] = "PASSED" - elif status == "failed": - results[test_id] = "FAILED" - elif status == "skipped": - results[test_id] = "SKIPPED" - - # Pattern 2: Unit test format (wolfssl unit tests) - # Format: "HMAC-MD5 test passed!" - # Only match lines that don't contain '---' (separator lines) - # Use [ \t] instead of \s to avoid matching newlines - unit_pattern = re.compile( - r"^([A-Za-z0-9_\-/]+(?:[ \t]+[A-Za-z0-9_\-/]+){0,5}?)[ \t]+test[ \t]+(passed|failed)!", - re.MULTILINE | re.IGNORECASE, - ) - - for match in unit_pattern.finditer(text_output): - test_name = match.group(1).strip() - status = match.group(2).lower() - - # Skip if the test name contains special characters indicating it's not a real test - if "---" in test_name or len(test_name) > 50: - continue - - # Build test ID with prefix - if test_prefix: - test_id = f"{test_prefix}::{test_name}" - else: - test_id = test_name - - if status == "passed": - results[test_id] = "PASSED" - elif status == "failed": - results[test_id] = "FAILED" - - # Pattern 3: FAILURES section (wolfssl API tests) - # Format: "FAILURES:\n 892: test_wolfSSL_CTX_load_verify_locations" - failures_section = re.search(r"FAILURES:\s*\n(.*?)(?:\n\s*End|$)", text_output, re.DOTALL) - if failures_section: - failure_pattern = re.compile(r"^\s*\d+:\s+([^\s:]+(?:\s+[^\s:]+)*)", re.MULTILINE) - for match in failure_pattern.finditer(failures_section.group(1)): - test_name = match.group(1).strip() - if test_prefix: - test_id = f"{test_prefix}::{test_name}" - else: - test_id = test_name - # Mark as failed (this overrides any previous 'passed' if it exists) - results[test_id] = "FAILED" - - return results - - -def parse_junit_xml(xml_content: str) -> Dict[str, str]: - """Parse JUnit XML to extract test results. - - IMPORTANT: We prioritize finding valid test results over detecting errors. - Even if output contains errors (import errors, syntax errors, etc.), if we find valid - XML test results, we parse and return them. We only return None if we're certain - the framework didn't run (no test results + error indicators). - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If test framework failed to run (not the same as tests failing) - """ - results = {} - found_any_xml = False - - # PRIORITY 1 & 2: Try to parse XML documents (pure or mixed with other output) - # Handle multiple concatenated XML documents (from multiple test result files) - # Split by 0: - doc = "", xml_start) - if xml_end > xml_start: - xml_extracted = doc[xml_start : xml_end + len("")] - else: - xml_end = doc.find("", xml_start) - if xml_end > xml_start: - xml_extracted = doc[xml_start : xml_end + len("")] - else: - continue - - try: - tree = ET.fromstring(xml_extracted) - found_any_xml = True - except ET.ParseError: - continue - - # Parse all testcases from this document - for testcase in tree.iter("testcase"): - classname = testcase.get("classname", "") - name = testcase.get("name", "") - test_id = f"{classname}::{name}" if classname else name - - # Check if this testcase has system-out with embedded test results - # This handles cases like wolfssl where a single ctest executable runs many tests - system_out = testcase.find("system-out") - embedded_results = {} - if system_out is not None and system_out.text: - embedded_results = _parse_embedded_test_results(system_out.text, classname or name) - - if embedded_results: - # If we found embedded test results, use those instead of the testcase status - results.update(embedded_results) - elif testcase.find("failure") is not None or testcase.find("error") is not None: - results[test_id] = "FAILED" - elif testcase.find("skipped") is not None: - results[test_id] = "SKIPPED" - else: - results[test_id] = "PASSED" - - # PRIORITY 3: If we found NO valid XML and NO results, check for error indicators - # Only return None if we're certain the framework failed to run - if not found_any_xml and not results: - error_indicators = [ - "ERROR: ", # Generic error marker - "ImportError:", # Python import errors - "ModuleNotFoundError:", # Python module errors - "SyntaxError:", # Python syntax errors - "FAILED ", # Framework failure markers - "INTERNALERROR", # pytest internal errors - "collection errors", # pytest collection errors - "error: ", # Generic error (C++, Swift, etc.) - "fatal error:", # Fatal compilation errors - "cannot find symbol", # Java compilation errors - "error: build had", # Swift build errors (xctest) - "error: terminated", # Swift process crashes (xctest) - ] - has_errors = any(indicator in xml_content for indicator in error_indicators) - # Return None ONLY if: no XML found AND errors present - # Return empty dict if: no XML found AND no errors (rare but valid) - return None if has_errors else results - - return results - - -def parse_go_json(json_output: str) -> Dict[str, str]: - """Parse Go test -json output (newline-delimited JSON). - - IMPORTANT: We prioritize finding valid test results over detecting errors. - Even if output contains errors (module errors, build errors, etc.), if we find valid - test results JSON, we parse and return it. We only return None if we're certain - the tests didn't run (no test results + error indicators). - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If Go tests failed to run (not the same as tests failing) - """ - results = {} - has_valid_json = False - - # PRIORITY 1: Try to parse newline-delimited JSON (valid test output) - for line in json_output.strip().split("\n"): - if not line.strip(): - continue - try: - event = json.loads(line) - has_valid_json = True # Found at least one valid JSON line - action = event.get("Action") - - # Handle test-level events - if "Test" in event and action in ["pass", "fail", "skip"]: - test_name = event.get("Test", "") - if test_name: - package = event.get("Package", "") - test_id = f"{package}::{test_name}" if package else test_name - - if action == "pass": - results[test_id] = "PASSED" - elif action == "fail": - results[test_id] = "FAILED" - elif action == "skip": - results[test_id] = "SKIPPED" - - # Handle package-level failures (no Test field) - elif "Package" in event and "Test" not in event and action == "fail": - package = event.get("Package", "") - test_id = f"{package}::package" - results[test_id] = "FAILED" - - except json.JSONDecodeError: - # PRIORITY 2: Handle plaintext build failures (legitimate failures) - # When tests can't compile/build, Go outputs plaintext "FAIL package [build failed]" - # This is a legitimate test failure, not a parsing error - build_fail_match = re.match(r"^FAIL\s+(\S+)\s+\[build failed\]", line) - if build_fail_match: - package_name = build_fail_match.group(1) - results[package_name] = "FAILED" - has_valid_json = True # Count build failures as valid results - - # PRIORITY 3: If we found NO valid JSON and NO build failures, check for error indicators - if not has_valid_json and not results: - error_indicators = [ - "go: cannot find main module", # Module not found - "can't load package", # Package loading errors - "pattern matches no packages", # No matching packages - "build constraints exclude all Go files", # Build constraints error - ] - has_errors = any(indicator in json_output for indicator in error_indicators) - # Return None ONLY if: no JSON found AND errors present - # Return empty dict if: no JSON found AND no errors (rare but valid) - return None if has_errors else results - - return results - - -def parse_jest_vitest_json(json_output: str) -> Dict[str, str]: - """Parse Jest/Vitest JSON output. - - IMPORTANT: We prioritize finding valid test results over detecting errors. - Even if output contains errors (TypeScript, npm, etc.), if we find valid - test results JSON, we parse and return it. We only return None if we're - certain the framework didn't run (no test results + error indicators). - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If Jest itself failed to run (not the same as tests failing) - """ - results = {} - - # PRIORITY 1: Try to parse as pure JSON (test results take precedence) - try: - data = json.loads(json_output.strip()) - # If we got JSON, check if it has test results (even if errors exist elsewhere in output) - except json.JSONDecodeError: - # PRIORITY 2: Search for JSON markers in mixed output - # Even with errors in output, tests might have run and produced JSON - json_start = json_output.find('{"numFailed') # Jest format - if json_start == -1: - json_start = json_output.find('{"numTotalTest') # Vitest format - if json_start == -1: - json_start = json_output.find('{"test') # Alternative format - if json_start == -1: - # PRIORITY 3: No JSON found - NOW check if there are error indicators - # Only return None if we're sure tests didn't run (no results + errors present) - # NOTE: error_indicators are a LAST RESORT - we prefer finding test results - error_indicators = [ - "error TS", # TypeScript compilation errors (e.g., error TS2307:) - "ELIFECYCLE", # npm script failures - "npm ERR!", # npm errors - "Error: Cannot find module", # Module loading errors (like Mocha) - "SyntaxError:", # JavaScript/TypeScript syntax errors - "Test suite failed to run", # Jest-specific: tests couldn't be loaded - "FAIL ", # Jest failure marker without JSON - ] - has_errors = any(indicator in json_output for indicator in error_indicators) - # Return None ONLY if: no JSON found AND errors present - # Return empty dict if: no JSON found AND no errors (rare but valid) - return None if has_errors else results - - # Try to extract JSON from mixed output - decoder = json.JSONDecoder() - try: - data, _ = decoder.raw_decode(json_output[json_start:]) - except json.JSONDecodeError: - # Could not parse JSON even after finding marker - return None - - # At this point, we have successfully parsed JSON - # Check if this is Jest's error response format (Jest itself failed, not the tests) - # Format: {"error": {"code": 2, "summary": "", "detail": ""}} - # This is a structured error response, NOT test results - if "error" in data and "code" in data.get("error", {}): - # This is an error response from Jest itself, not test results - return None - - # Check if we have the expected test results structure - # If we have testResults, parse it even if tests failed - those are legitimate test results - # Parse test results - if "testResults" in data: - for test_result in data.get("testResults", []): - file_path = test_result.get("name", "") - suite_status = test_result.get("status", "") - assertions = test_result.get("assertionResults", []) - - # Handle suite-level failures (no assertions ran) - if suite_status == "failed" and len(assertions) == 0: - test_id = f"{file_path}::suite" - results[test_id] = "FAILED" - continue - - # Handle individual test assertions - for assertion in assertions: - full_name = assertion.get("fullName", "") - title = assertion.get("title", "") - status = assertion.get("status", "") - test_id = f"{file_path}::{full_name}" if full_name else f"{file_path}::{title}" - - if status == "passed": - results[test_id] = "PASSED" - elif status == "failed": - results[test_id] = "FAILED" - elif status in ["pending", "skipped"]: - results[test_id] = "SKIPPED" - - # If we successfully parsed JSON but found no testResults, that's unexpected - # Return None to indicate this isn't valid test output - # (Valid Jest output should have testResults array, even if empty) - if "testResults" not in data: - return None - - return results - - -def parse_mocha_json(json_output: str) -> Optional[Dict[str, str]]: - """Parse Mocha JSON output. - - IMPORTANT: We prioritize finding valid test results over detecting errors. - Even if output contains errors (module errors, syntax errors, etc.), if we find valid - test results JSON, we parse and return it. We only return None if we're certain - the framework didn't run (no test results + error indicators). - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If Mocha itself failed to run (not the same as tests failing) - """ - results = {} - - # PRIORITY 1: Try to parse as pure JSON (test results take precedence) - try: - data = json.loads(json_output.strip()) - # Validate this is Mocha JSON by checking for 'stats' key - if "stats" not in data: - data = None - except json.JSONDecodeError: - data = None - - # PRIORITY 2: If direct parse failed, search for JSON in mixed output - if data is None: - # Look for stats key in JSON - stats_pos = json_output.find('"stats"') - if stats_pos == -1: - # PRIORITY 3: No JSON found - NOW check if there are error indicators - error_indicators = [ - "Error: Cannot find module", # Module loading errors - "SyntaxError:", # JavaScript syntax errors - "TypeError:", # Type errors - "ReferenceError:", # Reference errors - "No test files found", # Mocha-specific: no tests found - ] - has_errors = any(indicator in json_output for indicator in error_indicators) - # Return None ONLY if: no JSON found AND errors present - # Return empty dict if: no JSON found AND no errors (rare but valid) - return None if has_errors else results - - # Find the opening brace before "stats" - json_start = json_output.rfind("{", 0, stats_pos) - if json_start == -1: - return None - - # Try parsing from this position - json_portion = json_output[json_start:] - - # Use json.JSONDecoder to find where the object ends - decoder = json.JSONDecoder() - try: - data, _ = decoder.raw_decode(json_portion) - except json.JSONDecodeError: - return None - - # Validate extracted JSON has 'stats' - if "stats" not in data: - return None - - # At this point, we have valid Mocha JSON with 'stats' - # Parse test results even if some tests failed - those are legitimate results - - # Process passed tests - for test in data.get("passes", []): - file_path = test.get("file", "") - full_title = test.get("fullTitle", "") - test_id = f"{file_path}::{full_title}" if full_title else file_path - results[test_id] = "PASSED" - - # Process failed tests - for test in data.get("failures", []): - file_path = test.get("file", "") - full_title = test.get("fullTitle", "") - test_id = f"{file_path}::{full_title}" if full_title else file_path - results[test_id] = "FAILED" - - # Process pending/skipped tests - for test in data.get("pending", []): - file_path = test.get("file", "") - full_title = test.get("fullTitle", "") - test_id = f"{file_path}::{full_title}" if full_title else file_path - results[test_id] = "SKIPPED" - - return results - - -def parse_gtest_json(json_output: str) -> Dict[str, str]: - """Parse Google Test JSON output. - - IMPORTANT: We prioritize finding valid test results over detecting errors. - Even if output contains errors (compilation errors, linking errors, etc.), if we find valid - test results JSON, we parse and return it. We only return None if we're certain - the tests didn't run (no test results + error indicators). - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If GTest itself failed to run (not the same as tests failing) - """ - results = {} - - # PRIORITY 1: Try to parse as pure JSON (test results take precedence) - try: - data = json.loads(json_output.strip()) - # Validate this is GTest JSON by checking for 'testsuites' key - if "testsuites" not in data: - data = None - except json.JSONDecodeError: - data = None - - # PRIORITY 2: If direct parse failed, search for JSON in mixed output - if data is None: - # Try to find JSON in mixed output - json_start = json_output.find('{"testsuites"') - if json_start == -1: - json_start = json_output.find('{\n "testsuites"') - if json_start == -1: - # PRIORITY 3: No JSON found - NOW check if there are error indicators - error_indicators = [ - "error:", # C++ compilation errors - "undefined reference to", # Linking errors - "fatal error:", # Fatal compilation errors - "cannot find -l", # Linking library errors (e.g., "cannot find -lgtest") - ": No such file or directory", # File not found errors - ] - has_errors = any(indicator in json_output for indicator in error_indicators) - # Return None ONLY if: no JSON found AND errors present - # Return empty dict if: no JSON found AND no errors (rare but valid) - return None if has_errors else results - - # Extract JSON object - json_portion = json_output[json_start:] - decoder = json.JSONDecoder() - try: - data, _ = decoder.raw_decode(json_portion) - except json.JSONDecodeError: - return None - - # Validate extracted JSON has 'testsuites' - if "testsuites" not in data: - return None - - # At this point, we have valid GTest JSON with 'testsuites' - # Parse test results even if some tests failed - those are legitimate results - - # Parse test results from testsuites - testsuites = data.get("testsuites", []) - if not isinstance(testsuites, list): - testsuites = [testsuites] if isinstance(testsuites, dict) else [] - - for testsuite in testsuites: - suite_name = testsuite.get("name", "") - - # Handle both 'testsuite' (array) and direct test cases - test_cases = testsuite.get("testsuite", []) - if not test_cases: - test_cases = testsuite.get("tests", []) - - for test_case in test_cases: - test_name = test_case.get("name", "") - classname = test_case.get("classname", suite_name) - - # Build test ID in format: SuiteName::TestName - test_id = f"{classname}::{test_name}" if classname else test_name - - # Determine test status - status = test_case.get("status", "RUN") - result = test_case.get("result", "COMPLETED") - - # Check for failures - failures = test_case.get("failures", []) - if failures and len(failures) > 0: - results[test_id] = "FAILED" - elif status == "NOTRUN" or result == "SKIPPED": - results[test_id] = "SKIPPED" - elif result == "COMPLETED" or status == "RUN": - results[test_id] = "PASSED" - else: - results[test_id] = "FAILED" - - return results - - -def parse_maven_text_output(text_output: str) -> Dict[str, str]: - """Parse Maven text output for test results.""" - results = {} - - # Look for test summary lines like: - # Tests run: 5, Failures: 1, Errors: 0, Skipped: 0 - summary_pattern = r"Tests run: (\d+),\s*Failures: (\d+),\s*Errors: (\d+),\s*Skipped: (\d+)" - - # Check for compilation errors - if tests can't compile, mark them as failed - compilation_error_pattern = r"\[ERROR\].*?testCompile.*?Compilation failure" - if re.search(compilation_error_pattern, text_output, re.DOTALL | re.IGNORECASE): - # Find test files mentioned in compilation errors - test_file_pattern = r"/workspace/repo/[^/]+/src/test/java/([\w/]+)\.java" - for match in re.finditer(test_file_pattern, text_output): - test_class = match.group(1).replace("/", ".") - # Mark as failed due to compilation - results[f"{test_class}::compile"] = "FAILED" - # If we found compilation errors, return early - if results: - return results - - # Check for BUILD FAILURE - if "BUILD FAILURE" in text_output: - # If build failed and we haven't found specific test failures, mark as generic failure - if not results: - results["maven::build"] = "FAILED" - return results - - # Parse test run summaries per module - lines = text_output.split("\n") - current_module = None - - for line in lines: - # Track which module we're in - if "Building" in line and "[" in line and "]" in line: - # Extract module name from lines like "[INFO] Building Docs Web 1.12-SNAPSHOT [4/4]" - parts = line.split("Building") - if len(parts) > 1: - module_parts = parts[1].strip().split() - if len(module_parts) > 0: - current_module = module_parts[0] - - # Look for test summary - summary_match = re.search(summary_pattern, line) - if summary_match: - total = int(summary_match.group(1)) - failures = int(summary_match.group(2)) - errors = int(summary_match.group(3)) - skipped = int(summary_match.group(4)) - - if total > 0: - # We have test counts but might not have individual test names - # Generate generic test IDs based on the current module - module_name = current_module or "unknown" - passed = total - failures - errors - skipped - - for j in range(passed): - results[f"{module_name}::test_{j + 1}"] = "PASSED" - for j in range(failures + errors): - results[f"{module_name}::test_failed_{j + 1}"] = "FAILED" - for j in range(skipped): - results[f"{module_name}::test_skipped_{j + 1}"] = "SKIPPED" - return results - - -def parse_cargo_nextest(output: str) -> Dict[str, str]: - """Parse cargo-nextest text output. - - IMPORTANT: We prioritize finding valid test results over detecting errors. - Even if output contains errors (warnings, etc.), if we find valid test results, - we parse and return them. We only return None if we're certain the tests didn't - run (no test results + error indicators). - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If cargo-nextest failed to run (not the same as tests failing) - """ - results = {} - - # PRIORITY 1: Parse individual test result lines - # Format: PASS [ 1.588s] rusty::tests integration::linking::test_name - # FAIL [ 5.845s] rusty codegen::tests::parameters_tests::test_name - test_line_pattern = re.compile(r"^\s*(PASS|FAIL|SIGKILL|SKIP)\s+\[.*?\]\s+(.+)$", re.MULTILINE) - - for match in test_line_pattern.finditer(output): - status = match.group(1) - test_name = match.group(2).strip() - - if status == "PASS": - results[test_name] = "PASSED" - elif status in ("FAIL", "SIGKILL"): - results[test_name] = "FAILED" - elif status == "SKIP": - results[test_name] = "SKIPPED" - - # PRIORITY 2: If we found NO test results, check for error indicators - # Only return None if we're certain tests didn't run (compilation/linking errors) - if not results: - error_indicators = [ - "error[E", # Rust compiler errors (e.g., error[E0425]) - "error: could not compile", # Cargo compilation errors - "error: linking with", # Linking errors - "error: aborting due to", # Compilation aborted - ] - has_errors = any(indicator in output for indicator in error_indicators) - # Return None ONLY if: no results found AND errors present - # Return empty dict if: no results found AND no errors (rare but valid - no tests in project) - return None if has_errors else results - - return results - - -def parse_bun_text(text_output: str) -> Dict[str, str]: - """ - Parse Bun test framework output. - - IMPORTANT: We prioritize finding valid test results over detecting errors. - Even if output contains errors (TypeScript, compilation, etc.), if we find valid - test results, we parse and return them. We only return None if we're - certain Bun didn't run (no test results + error indicators). - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If Bun itself failed to run (not the same as tests failing) - """ - results = {} - current_file = None - current_describe = None - - # PRIORITY 1: Try to parse test results (✓ and ✗ symbols) - for line in text_output.split("\n"): - # Track current file (lines ending with .ts: or .js:) - if re.match(r"^[^\s].*\.(ts|js|tsx|jsx):?\s*$", line.strip()): - current_file = line.strip().rstrip(":") - current_describe = None - continue - - # Track describe blocks (indented text followed by colon, but not test results) - describe_match = re.match(r"^\s+([^✓✗\n]+):\s*$", line) - if describe_match: - current_describe = describe_match.group(1).strip() - continue - - # Remove ANSI color codes - clean_line = re.sub(r"\x1b\[[0-9;]*m", "", line) - - # Match passed tests: ✓ test_name [time] - pass_match = re.match(r"^\s*✓\s+(.+?)(?:\s+\[[\d.]+m?s\])?\s*$", clean_line) - if pass_match: - test_name = pass_match.group(1).strip() - # Build test ID with file, describe block, and test name - test_id = test_name - if current_file: - test_id = f"{current_file}::{test_name}" - if current_describe: - test_id = f"{current_file}::{current_describe} > {test_name}" - results[test_id] = "PASSED" - continue - - # Match failed tests: ✗ test_name [time] - fail_match = re.match(r"^\s*✗\s+(.+?)(?:\s+\[[\d.]+m?s\])?\s*$", clean_line) - if fail_match: - test_name = fail_match.group(1).strip() - # Build test ID with file, describe block, and test name - test_id = test_name - if current_file: - test_id = f"{current_file}::{test_name}" - if current_describe: - test_id = f"{current_file}::{current_describe} > {test_name}" - results[test_id] = "FAILED" - continue - - # Alternative format: FAIL filepath > describe > test_name - alt_fail_match = re.match(r"^\s*FAIL\s+(.+?)\s+>\s+(.+?)\s*$", clean_line) - if alt_fail_match: - file_path = alt_fail_match.group(1).strip() - test_path = alt_fail_match.group(2).strip() - test_id = f"{file_path}::{test_path}" - results[test_id] = "FAILED" - continue - - # PRIORITY 2: If no individual test results found, try parsing summary - if not results: - # Look for summary like "5 pass, 2 fail" or "X passing (Yms)" - summary_match = re.search(r"(\d+)\s+pass(?:ing|ed)?.*?(\d+)\s+fail(?:ing|ed)?", text_output.lower()) - if summary_match: - passed = int(summary_match.group(1)) - failed = int(summary_match.group(2)) - - # Generate generic test IDs - for i in range(passed): - results[f"test_{i + 1}"] = "PASSED" - for i in range(failed): - results[f"test_failed_{i + 1}"] = "FAILED" - - # PRIORITY 3: No test results found - NOW check if there are error indicators - # Only return None if we're sure tests didn't run (no results + errors present) - # NOTE: error_indicators are a LAST RESORT - we prefer finding test results - if not results: - error_indicators = [ - "error TS", # TypeScript compilation errors (e.g., error TS2307:) - "Error: Cannot find module", # Module loading errors - "SyntaxError:", # JavaScript/TypeScript syntax errors - "error: ", # Generic Bun errors (lowercase 'error:') - "Error:", # Generic errors - "ModuleNotFoundError", # Module not found - "bun: command not found", # Bun not installed - "panicked at", # Bun runtime panics - "Segmentation fault", # Critical runtime errors - ] - has_errors = any(indicator in text_output for indicator in error_indicators) - # Return None ONLY if: no test results found AND errors present - # Return empty dict if: no test results found AND no errors (rare but valid) - return None if has_errors else results - - return results - - -def parse_cppunit_text(text_output: str) -> Dict[str, str]: - """Parse CppUnit text output for test results. - - IMPORTANT: We prioritize finding valid test results over detecting errors. - Even if output contains errors (warnings, etc.), if we find valid test results, - we parse and return them. We only return None if we're certain the tests didn't - run (no test results + error indicators). - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If CppUnit failed to run (not the same as tests failing) - """ - results = {} - - # PRIORITY 1: Parse individual test result lines - # Format: TestClassName::testMethodName : OK - # TestClassName::testMethodName : FAIL - test_line_pattern = re.compile( - r"^([A-Za-z_][A-Za-z0-9_]*::[A-Za-z_][A-Za-z0-9_]*)\s*:\s*(OK|FAIL|ERROR)$", re.MULTILINE - ) - - for match in test_line_pattern.finditer(text_output): - test_name = match.group(1).strip() - status = match.group(2).strip() - - if status == "OK": - results[test_name] = "PASSED" - elif status in ["FAIL", "ERROR"]: - results[test_name] = "FAILED" - - # PRIORITY 2: If we found NO test results, check for error indicators - # Only return None if we're certain tests didn't run (compilation/linking errors) - if not results: - error_indicators = [ - "error:", # C++ compilation errors - "undefined reference to", # Linking errors - "fatal error:", # Fatal compilation errors - "ld returned", # Linker errors - "cannot find -l", # Library linking errors - ] - has_errors = any(indicator in text_output for indicator in error_indicators) - # Return None ONLY if: no results found AND errors present - # Return empty dict if: no results found AND no errors (rare but valid - no tests) - return None if has_errors else results - - return results - - -def parse_minitest_text(text_output: str, test_metadata_path: str = None) -> Dict[str, str]: - """ - Parse mini.nvim (MiniTest) test framework output. - - MiniTest is used by Neovim plugins for testing. - Example output: - Total number of cases: 5 - tests/test_treesitter.lua: ooooo - - Fails (0) and Notes (0) - - Or with failures: - FAIL in tests/test_treesitter.lua | wrap_cursor | normal: error message - FAIL in tests/test_treesitter.lua | enumerate: error message - - Fails (2) and Notes (0) - - IMPORTANT: MiniTest only outputs individual test names when they FAIL. - When all tests pass, only summary is shown - no individual test names. - - Solution: When all tests pass, read test_metadata.json to get expected test names - and return them as PASSED. This ensures real test names are used consistently. - """ - results = {} - - # Parse individual test results from FAIL/NOTE lines - # Format: FAIL in file.lua | group | test_name: error message - # Use [^|:]+ to stop at pipe OR colon (prevents capturing error message) - fail_pattern = re.compile( - r"^(?:\x1b\[\d+(?:;\d+)?m)?FAIL(?:\x1b\[0m)?\s+in\s+([^|]+)\s*\|\s*([^|:]+)(?:\s*\|\s*([^:]+))?:", re.MULTILINE - ) - - for match in fail_pattern.finditer(text_output): - file_path = match.group(1).strip() - group = match.group(2).strip() - test_name = match.group(3).strip() if match.group(3) else "" - - # Create test ID: file | group | test_name or file | group - if test_name: - test_id = f"{file_path} | {group} | {test_name}" - else: - test_id = f"{file_path} | {group}" - - results[test_id] = "FAILED" - - return results - - -def parse_telescope_text(text_output: str) -> Dict[str, str]: - """ - Parse telescope test framework output. - - Telescope outputs lines like: - ✓ test_name - ✗ test_name - - test_name (skipped) - - Also handles PlenaryBusted output for Neovim plugins. - - IMPORTANT: We prioritize finding valid test results over detecting errors. - We only return None if we're certain the framework didn't run. - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If telescope failed to run (not the same as tests failing) - """ - results = {} - - for line in text_output.split("\n"): - line = line.strip() - - # Match passed tests: ✓ test_name or "Success: test_name" - if "✓" in line: - test_name = line.split("✓", 1)[1].strip() - if test_name: # Avoid empty test names - results[test_name] = "PASSED" - elif line.lower().startswith("success:"): - test_name = line.split(":", 1)[1].strip() - if test_name: - results[test_name] = "PASSED" - - # Match failed tests: ✗ test_name or "Failed: test_name" - elif "✗" in line: - test_name = line.split("✗", 1)[1].strip() - if test_name: - results[test_name] = "FAILED" - elif line.lower().startswith("failed:"): - test_name = line.split(":", 1)[1].strip() - if test_name: - results[test_name] = "FAILED" - - # Match skipped tests: - test_name or "Skipped: test_name" - elif line.startswith("- ") and "skip" in line.lower(): - test_name = line[2:].strip() - # Remove "(skipped)" suffix if present - test_name = re.sub(r"\s*\(skipped\)\s*$", "", test_name, flags=re.IGNORECASE) - if test_name: - results[test_name] = "SKIPPED" - elif line.lower().startswith("skipped:"): - test_name = line.split(":", 1)[1].strip() - if test_name: - results[test_name] = "SKIPPED" - - # If no results found, try parsing summary line - if not results: - # Look for summary like "5 passed, 2 failed, 1 skipped" - summary_pattern = r"(\d+)\s+passed.*?(\d+)\s+failed" - match = re.search(summary_pattern, text_output.lower()) - if match: - passed = int(match.group(1)) - failed = int(match.group(2)) - - # Generate generic test IDs - for i in range(passed): - results[f"test_{i + 1}"] = "PASSED" - for i in range(failed): - results[f"test_failed_{i + 1}"] = "FAILED" - - # PRIORITY 2: If we found NO test results, check for error indicators - # Only return None if we're certain tests didn't run (Lua/Neovim errors) - if not results: - error_indicators = [ - "Error:", # Generic Lua errors - "error loading module", # Lua module loading errors - "attempt to call", # Lua runtime errors - "bad argument", # Lua runtime errors - "stack traceback:", # Lua errors with traceback - ] - has_errors = any(indicator in text_output for indicator in error_indicators) - # Return None ONLY if: no results found AND errors present - # Return empty dict if: no results found AND no errors (rare but valid - no tests) - return None if has_errors else results - - return results - - -def parse_lust_text(text_output: str) -> Dict[str, str]: - """ - Parse lust test framework output. - - Lust outputs test results with dots (.) for pass, F for fail. - Example output: - ..F. - 4 tests, 1 failure - test/my_test.lua:15: Expected true but got false - - We parse individual test results when available, or fall back to summary. - """ - results = {} - - # Try to parse individual test results from verbose output - # Pattern: " test_name ... ok" or " test_name ... FAILED" - test_pattern = re.compile(r"^\s*(.+?)\s+\.\.\.\s+(ok|FAILED|ERROR)", re.MULTILINE) - matches = test_pattern.findall(text_output) - - if matches: - # Found individual test results - for test_name, status in matches: - test_name = test_name.strip() - if status == "ok": - results[test_name] = "PASSED" - else: - results[test_name] = "FAILED" - return results - - # Try to extract test descriptions from failure messages - # Pattern: "test_file.lua:line_number: test description" - failure_pattern = re.compile(r"^([^\s:]+\.lua):(\d+):\s*(.+)$", re.MULTILINE) - failures = failure_pattern.findall(text_output) - - if failures: - for filepath, _, description in failures: - test_id = f"{filepath}::{description.strip()}" - results[test_id] = "FAILED" - - # Parse summary line to get total count: "X tests, Y failures" - summary_match = re.search(r"(\d+)\s+tests?,\s+(\d+)\s+failures?", text_output.lower()) - if summary_match: - total_tests = int(summary_match.group(1)) - failures = int(summary_match.group(2)) - - # If we haven't parsed individual tests yet, generate generic ones - if not results: - passed = total_tests - failures - for i in range(passed): - results[f"test_{i + 1}"] = "PASSED" - for i in range(failures): - results[f"test_failed_{i + 1}"] = "FAILED" - return results - - # Fallback: if no detailed info, check for overall success/failure - if not results: - if "0 failures" in text_output.lower() or "0 errors" in text_output.lower(): - results["test_suite"] = "PASSED" - else: - results["test_suite"] = "FAILED" - - return results - - -def parse_bespoke_libgeos(text_output: str) -> Dict[str, str]: - """Parse libgeos/GEOS test output format. - - Format: - capi::GEOSBoundary: . - capi::GEOSBuffer: ..................... - geos::operation::OverlayNGEmptyCoordDim: [1=F][2=F].[4=F][5=F][6=F] - geos::operation::buffer::BufferOp: ..........................[27=X] - - Where: - - dots (.) = passing tests - - [N=F] = explicit failure markers - - [N=X] = exception markers (also failures) - - standalone F or X = failure/exception - - IMPORTANT: We prioritize finding valid test results over detecting errors. - We only return None if we're certain the framework didn't run. - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If libgeos tests failed to run (not the same as tests failing) - """ - results = {} - - # PRIORITY 1: Parse individual test result lines - # Pattern: TestSuite::TestName: followed by dots, Fs, Xs, or [N=F]/[N=X] markers - # Example: capi::GEOSBoundary: . - # Example: geos::OverlayNGEmptyCoordDim: [1=F][2=F].[4=F] - # Example: geos::operation::buffer::BufferOp: ..........................[27=X] - test_line_pattern = re.compile( - r"^([a-zA-Z_][a-zA-Z0-9_:]*::[a-zA-Z_][a-zA-Z0-9_]*)\s*:\s*(.+?)(?:\n|$)", re.MULTILINE - ) - - for match in test_line_pattern.finditer(text_output): - test_id = match.group(1) # Full name like "capi::GEOSBoundary" - test_output_line = match.group(2) # Everything after the colon - - # Check for failure markers: - # 1. [N=F] pattern (explicit failure notation) - # 2. [N=X] pattern (exception notation) - # 3. Standalone F or X characters - has_failure = bool(re.search(r"\[.*=[FX]\]|(? str: - """Normalize XCTest case identifiers from swift test console output.""" - name = raw_name.strip() - - # Typical format: -[Module.Class testMethod] - if name.startswith("-[") and name.endswith("]"): - inner = name[2:-1].strip() - if " " in inner: - class_name, method = inner.split(" ", 1) - return f"{class_name}::{method}" - return inner - - # Alternate format: Module.Class.testMethod - if "." in name: - parts = name.split(".", 1) - return f"{parts[0]}::{parts[1]}" - - return name - - -def parse_swift_test_text(text_output: str) -> Dict[str, str]: - """Parse vanilla `swift test` console output (without --xunit-output).""" - results = {} - test_case_pattern = re.compile(r"Test Case '([^']+)' (passed|failed|skipped)", re.IGNORECASE) - - for match in test_case_pattern.finditer(text_output): - raw_name, status = match.groups() - test_id = _normalize_swift_test_name(raw_name) - results[test_id] = status.upper() - - if results: - return results - - # Fallback: parse summary line to infer aggregate results if per-test lines missing - summary_match = re.search( - r"Executed\s+(\d+)\s+tests?,\s+with\s+(\d+)\s+failures?", - text_output, - re.IGNORECASE, - ) - if summary_match: - total_tests = int(summary_match.group(1)) - failures = int(summary_match.group(2)) - passes = max(total_tests - failures, 0) - - for i in range(passes): - results[f"swift_test_pass_{i + 1}"] = "PASSED" - for i in range(failures): - results[f"swift_test_fail_{i + 1}"] = "FAILED" - - return results - - -def parse_xctest_output(output: str) -> Dict[str, str]: - """Parse XCTest results, preferring XML when available.""" - xml_results = parse_junit_xml(output) - if xml_results: - return xml_results - return parse_swift_test_text(output) - - -def normalize_test_id(test_id: str, framework: str = "") -> str: - """Normalize test IDs for stable matching across different formats. - - This function performs several normalizations: - - 1. Removes unstable runtime prefixes that change between runs: - - (N/M) - Test execution order (e.g., "(2/5) test_name") - - [N/M] - Alternative bracket format - - #N - Test number prefix (e.g., "#42 test_name") - - N. - Numbered list format (e.g., "1. test_name") - - 2. Removes common file extensions (.py, .js, .ts, .go, etc.) from test paths - to allow matching between "test_file.py::test" and "test_file::test" - - 3. Normalizes delimiters (`.`, `::`, `/`) to a canonical form (`::`) - when they appear between alphanumeric characters, allowing matching - between "testa.testb::testc" and "testa/testb.testc" - - Examples: - "(2/5) test_name" -> "test_name" - "test_file.py::test_name" -> "test_file::test_name" - "testa.testb::testc" -> "testa::testb::testc" - "testa/testb.testc" -> "testa::testb::testc" - "tests/module.js::describe::it" -> "tests::module::describe::it" - - Args: - test_id: Original test ID from parser - framework: Test framework name (for future framework-specific rules if needed) - - Returns: - Normalized test ID - """ - # Step 1: Remove unstable runtime prefixes - - # Universal pattern: Remove (N/M) or [N/M] prefixes (test execution order) - # Matches: "(2/5) test", "[2/5] test", "(123/456) test", "( 1/75) test" (with internal space) - normalized = re.sub(r"^[\(\[]?\s*\d+/\d+[\)\]]?\s+", "", test_id) - - # Universal pattern: Remove #N prefix (test numbering) - # Matches: "#42 test", "# 42 test" - normalized = re.sub(r"^#\s*\d+\s+", "", normalized) - - # Universal pattern: Remove "N. " prefix (numbered list) - # Matches: "1. test", "42. test" - normalized = re.sub(r"^\d+\.\s+", "", normalized) - - # Step 2: Remove common file extensions before delimiters - # This prevents .py from becoming ::py after delimiter normalization - # Match extensions like .py, .js, .ts, etc. that appear before :: / . or end of string - extensions_pattern = ( - r"\.(py|pyw|js|mjs|cjs|ts|mts|cts|jsx|tsx|" - r"go|java|rb|rs|c|cpp|cc|cxx|h|hpp|hxx|" - r"swift|kt|kts|scala|php|cs|fs|" - r"ex|exs|erl|hrl|clj|cljs|cljc|" - r"lua|pl|pm|t|r|R|m|mm|" - r"f|f90|f95|for|vb|pas|pp|" - r"d|nim|zig|v|sv|vhd|vhdl|" - r"tcl|sh|bash|zsh|fish|ps1|psm1|psd1)" - r"(?=::|/|\.|$)" - ) - normalized = re.sub(extensions_pattern, "", normalized, flags=re.IGNORECASE) - - # Step 3: Normalize delimiters (., ::, /) to :: when between word characters - # This allows matching "testa.testb::testc" with "testa/testb.testc" - delimiter_pattern = r"(?<=\w)(::|\.|/)(?=\w)" - normalized = re.sub(delimiter_pattern, "::", normalized) - - return normalized - - -def parse_tap_text(text_output: str) -> Dict[str, str]: - """ - Parse TAP (Test Anything Protocol) output. - - TAP is used by tape, node-tap, and other JavaScript test frameworks. - - Format: - TAP version 13 - # Subtest: Test name - 1..N - ok 1 - assertion name - not ok 2 - assertion name - ok 1 - Test name # time=123ms - not ok 2 - Test name - 1..N - - IMPORTANT: We prioritize finding valid test results over detecting errors. - We only return None if we're certain the tests didn't run. - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If TAP tests failed to run (not the same as tests failing) - """ - results = {} - - # PRIORITY 1: Parse top-level test results (not indented subtests) - # Format: "ok N - Test name" or "not ok N - Test name" - # Skip lines starting with whitespace (subtests) - tap_test_pattern = re.compile( - r"^(not )?ok\s+(\d+)\s*(?:-\s*)?(.+?)(?:\s*#\s*(skip|todo|time=.*))?$", re.MULTILINE | re.IGNORECASE - ) - - for match in tap_test_pattern.finditer(text_output): - is_failure = match.group(1) is not None # "not ok" prefix - test_num = match.group(2) - test_name = match.group(3).strip() if match.group(3) else f"test_{test_num}" - directive = match.group(4) - - # Clean up test name (remove timing info like "# time=123ms") - test_name = re.sub(r"\s*#\s*time=[\d.]+m?s\s*$", "", test_name, flags=re.IGNORECASE) - - test_id = test_name if test_name else f"test_{test_num}" - - # Check for skip directive - if directive and directive.lower().startswith("skip"): - results[test_id] = "SKIPPED" - elif is_failure: - results[test_id] = "FAILED" - else: - results[test_id] = "PASSED" - - # PRIORITY 2: If no results found, try parsing summary line - # Format: "# tests N", "# pass N", "# fail N" - if not results: - pass_match = re.search(r"#\s*pass\s+(\d+)", text_output, re.IGNORECASE) - fail_match = re.search(r"#\s*fail\s+(\d+)", text_output, re.IGNORECASE) - - if pass_match or fail_match: - passed = int(pass_match.group(1)) if pass_match else 0 - failed = int(fail_match.group(1)) if fail_match else 0 - - for i in range(passed): - results[f"tap_test_passed_{i + 1}"] = "PASSED" - for i in range(failed): - results[f"tap_test_failed_{i + 1}"] = "FAILED" - - # PRIORITY 3: If no results, check for error indicators - if not results: - error_indicators = [ - "npm ERR!", # npm errors - "Error: Cannot find module", # Module loading errors - "SyntaxError:", # JavaScript syntax errors - "TypeError:", # Type errors - ] - has_errors = any(indicator in text_output for indicator in error_indicators) - # Return None ONLY if: no results found AND errors present - return None if has_errors else results - - return results - - -def parse_hardhat_mocha_text(text_output: str) -> Dict[str, str]: - """ - Parse Hardhat/Mocha console text output (non-JSON reporter). - - Hardhat uses Mocha under the hood and outputs text like: - Contract: FeeSharingProxy: - withdrawFees - ✓ Shouldn't be able to use zero token address - ✓ Shouldn't be able to withdraw second time in period - 1) Should fail with specific error - - 5 passing (1s) - 1 failing - - IMPORTANT: We prioritize finding valid test results over detecting errors. - - Returns: - Dict[str, str]: Test results mapping test IDs to status (PASSED/FAILED/SKIPPED) - None: If tests failed to run (not the same as tests failing) - """ - results = {} - - # Track current context (Contract/describe blocks) - current_context = [] - - # PRIORITY 1: Parse individual test results - for line in text_output.split("\n"): - stripped = line.strip() - - # Track Contract: or describe blocks - contract_match = re.match(r"^Contract:\s*(.+?):\s*$", stripped) - if contract_match: - current_context = [contract_match.group(1)] - continue - - # Track describe blocks (indented without checkmark/number) - if stripped and not stripped.startswith(("✓", "✗", "-")) and not re.match(r"^\d+\)", stripped): - # Check if this looks like a describe block (usually followed by test cases) - if ":" not in stripped and len(stripped) < 100: - # This might be a describe block, but we'll handle it dynamically - pass - - # Match passed tests: ✓ test_name or ✔ test_name - pass_match = re.match(r"^[✓✔]\s+(.+?)(?:\s+\(\d+m?s\))?$", stripped) - if pass_match: - test_name = pass_match.group(1).strip() - test_id = f"{' > '.join(current_context)} > {test_name}" if current_context else test_name - results[test_id] = "PASSED" - continue - - # Match failed tests: N) test_name or ✗ test_name - fail_match = re.match(r"^(?:\d+\)|[✗✘])\s*(.+?)$", stripped) - if fail_match: - test_name = fail_match.group(1).strip() - test_id = f"{' > '.join(current_context)} > {test_name}" if current_context else test_name - results[test_id] = "FAILED" - continue - - # Match skipped tests: - test_name - skip_match = re.match(r"^-\s+(.+?)$", stripped) - if skip_match: - test_name = skip_match.group(1).strip() - test_id = f"{' > '.join(current_context)} > {test_name}" if current_context else test_name - results[test_id] = "SKIPPED" - continue - - # PRIORITY 2: Parse summary if no individual results found - if not results: - # Look for "N passing" and "N failing" - pass_match = re.search(r"(\d+)\s+passing", text_output, re.IGNORECASE) - fail_match = re.search(r"(\d+)\s+failing", text_output, re.IGNORECASE) - - if pass_match or fail_match: - passed = int(pass_match.group(1)) if pass_match else 0 - failed = int(fail_match.group(1)) if fail_match else 0 - - for i in range(passed): - results[f"mocha_test_passed_{i + 1}"] = "PASSED" - for i in range(failed): - results[f"mocha_test_failed_{i + 1}"] = "FAILED" - - # PRIORITY 3: If no results, check for error indicators - if not results: - error_indicators = [ - "Error: Cannot find module", - "SyntaxError:", - "CompilerError:", # Solidity compilation errors - "Error: HH", # Hardhat errors - ] - has_errors = any(indicator in text_output for indicator in error_indicators) - return None if has_errors else results - - return results - - -def parse_pytest_text(text_output: str) -> Dict[str, str]: - """ - Parse pytest plain text output (-v flag). - - Pytest outputs lines like: - tests/test_foo.py::test_one PASSED - tests/test_foo.py::test_two FAILED - tests/test_foo.py::test_three SKIPPED - - Or in short form: - tests/test_foo.py .F.s - - Also handles summary lines like: - ===== 3 passed, 1 failed, 1 skipped in 0.5s ===== - """ - results = {} - - # Pattern 1: Verbose output with test names - # e.g., "tests/test_foo.py::test_one PASSED" - verbose_pattern = re.compile( - r"^([\w./]+::\w+(?:::\w+)*)\s+(PASSED|FAILED|SKIPPED|ERROR|XFAIL|XPASS)", re.MULTILINE - ) - - for match in verbose_pattern.finditer(text_output): - test_id = match.group(1).strip() - status = match.group(2).upper() - - if status in ("PASSED", "XPASS"): - results[test_id] = "PASSED" - elif status in ("FAILED", "ERROR", "XFAIL"): - results[test_id] = "FAILED" - elif status == "SKIPPED": - results[test_id] = "SKIPPED" - - if results: - return results - - # Pattern 2: Short form with dots (. = pass, F = fail, s = skip) - # e.g., "tests/test_foo.py .F.s" - short_pattern = re.compile(r"^([\w./]+\.py)\s+([.FsExX]+)", re.MULTILINE) - - for match in short_pattern.finditer(text_output): - file_path = match.group(1) - outcomes = match.group(2) - - for i, char in enumerate(outcomes): - test_id = f"{file_path}::test_{i + 1}" - if char == ".": - results[test_id] = "PASSED" - elif char.upper() == "F": - results[test_id] = "FAILED" - elif char.lower() == "s": - results[test_id] = "SKIPPED" - - if results: - return results - - # Pattern 3: Summary line fallback - # e.g., "===== 3 passed, 1 failed, 1 skipped in 0.5s =====" - summary_pattern = re.compile(r"(\d+)\s+passed(?:,\s*(\d+)\s+failed)?(?:,\s*(\d+)\s+(?:skipped|deselected))?") - match = summary_pattern.search(text_output) - if match: - passed = int(match.group(1) or 0) - failed = int(match.group(2) or 0) - skipped = int(match.group(3) or 0) - - for i in range(passed): - results[f"pytest_test_passed_{i + 1}"] = "PASSED" - for i in range(failed): - results[f"pytest_test_failed_{i + 1}"] = "FAILED" - for i in range(skipped): - results[f"pytest_test_skipped_{i + 1}"] = "SKIPPED" - - return results - - -def parse_test_output(output: str, framework: str) -> Dict[str, str]: - """ - Parse test output to extract individual test results. - - Returns: {'test_id': 'PASSED'|'FAILED'|'SKIPPED'} - """ - # Direct framework → parser mapping - parsers = { - "pytest": parse_junit_xml, - "unittest": parse_junit_xml, - "junit": parse_junit_xml, - "maven": parse_maven_text_output, - "gtest": parse_gtest_json, - "cargo-nextest": parse_cargo_nextest, - "go": parse_go_json, - "jest": parse_jest_vitest_json, - "vitest": parse_jest_vitest_json, - "mocha": parse_mocha_json, - "bun": parse_bun_text, - "ctest": parse_junit_xml, - "cppunit": parse_cppunit_text, - "bespoke_libgeos": parse_bespoke_libgeos, - # XCTest using hybrid approach - "xctest": parse_xctest_output, - "testing": parse_xctest_output, # New Swift Testing framework (Swift 6+) - # Lua frameworks - "busted": parse_junit_xml, # Uses JUnit XML output - "luaunit": parse_junit_xml, # Uses JUnit XML output - "telescope": parse_telescope_text, - "lust": parse_lust_text, - "minitest": parse_minitest_text, # Neovim mini.nvim test framework - # TAP (Test Anything Protocol) - used by tape, node-tap - "tap": parse_tap_text, - "tape": parse_tap_text, - # Hardhat (Solidity) - uses Mocha console output - "hardhat": parse_hardhat_mocha_text, - } - - parser = parsers.get(framework) - if parser: - result = parser(output) - # Fallback for common frameworks if their primary parser returns None/empty - if not result: - if framework in ["junit", "maven"]: - result = parse_maven_text_output(output) - elif framework == "pytest": - # Pytest often outputs plain text, not JUnit XML - result = parse_pytest_text(output) - elif framework == "mocha": - # Mocha might output text instead of JSON (console reporter) - result = parse_hardhat_mocha_text(output) - return result or {} - - # Try auto-detection for unknown frameworks - # Check for TAP output - if "TAP version" in output or re.search(r"^(?:not )?ok\s+\d+", output, re.MULTILINE): - return parse_tap_text(output) or {} - - # Check for Mocha/Hardhat console output - if "Contract:" in output or re.search(r"^\s*[✓✔]\s+", output, re.MULTILINE): - return parse_hardhat_mocha_text(output) or {} - - return {} diff --git a/responses_api_agents/swe_agents/swe_bench_ext/utils.py b/responses_api_agents/swe_agents/swe_bench_ext/utils.py deleted file mode 100644 index 7733ff34a1..0000000000 --- a/responses_api_agents/swe_agents/swe_bench_ext/utils.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""SWE-Bench-Ext test output parsing utilities for swe_agents. - -Thin wrapper around lighthouse's parsing library (the same one used by -swe_bench_ext_agent). All parsing, normalization, and fuzzy matching -logic is delegated to lighthouse — no custom parsers here. - -Usage from SweBenchExtDatasetProcessor.postprocess_after_run(): - - from responses_api_agents.swe_agents.swe_bench_ext.utils import parse_and_check_tests - - result = parse_and_check_tests( - test_output=log_text, - test_framework="pytest", - fail_to_pass=["test_a", "test_b"], - pass_to_pass=["test_c"], - instance_id="my-task-123", - ) - # result["resolved"] -> bool -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -from responses_api_agents.swe_agents.swe_bench_ext.parsing import ( - normalize_test_id, - parse_test_output, -) - - -# Marker strings used by generate_test_run_script to delimit structured output -_TEST_OUTPUT_START = "<<>>" -_TEST_OUTPUT_END = "<<>>" -_RESULT_FILE_START = "<<>>" -_RESULT_FILE_END = "<<>>" - - -def _extract_between_markers(text: str, start: str, end: str) -> Optional[str]: - """Extract text between two markers, or None if not found.""" - s = text.find(start) - e = text.find(end) - if s != -1 and e != -1 and s < e: - return text[s + len(start) : e].strip() - return None - - -def _match_test_with_fuzzy( - test_id: str, - parsed_results: Dict[str, str], - build_failed_packages: set, -) -> str: - """Fuzzy-match a test ID against parsed results. - - Mirrors SweBenchExtTask._match_test_with_fuzzy from - benchmark-swe-bench-ext/swe_bench_ext/task.py. - """ - # Direct match - if test_id in parsed_results: - return parsed_results[test_id] - - # Check if this test belongs to a package that failed to build - for pkg in build_failed_packages: - if test_id.startswith(pkg): - return "FAILED" - - # Substring match (normalized IDs may differ in prefix) - for parsed_id, status in parsed_results.items(): - if test_id in parsed_id or parsed_id in test_id: - return status - - # Try matching by last component (after last ::) - if "::" in test_id: - suffix = test_id.rsplit("::", 1)[-1] - for parsed_id, status in parsed_results.items(): - if "::" in parsed_id and parsed_id.rsplit("::", 1)[-1] == suffix: - return status - - return "NOT_FOUND" - - -def parse_and_check_tests( - test_output: str, - test_framework: str, - fail_to_pass: List[str], - pass_to_pass: List[str], - instance_id: str = "", -) -> Dict[str, Any]: - """Parse test output and check FAIL_TO_PASS / PASS_TO_PASS resolution. - - Uses the same lighthouse parsing pipeline as swe_bench_ext_agent: - 1. Extract structured output from markers (if present) - 2. parse_test_output() with framework dispatcher - 3. normalize_test_id() on both parsed and expected IDs - 4. Fuzzy match each expected test - 5. Compute resolved = all F2P passed AND all P2P passed - - Returns a report dict suitable for writing to report.json. - """ - # Try to extract result file content from markers (same as task.py) - result_file_content = _extract_between_markers(test_output, _RESULT_FILE_START, _RESULT_FILE_END) - - if result_file_content: - parsed = parse_test_output(result_file_content, test_framework) - if not parsed: - parsed = parse_test_output(test_output, test_framework) - else: - parsed = parse_test_output(test_output, test_framework) - - if parsed is None: - parsed = {} - - # Normalize parsed test IDs - parsed = {normalize_test_id(tid, test_framework): status for tid, status in parsed.items()} - - # Normalize expected test IDs - norm_f2p = [normalize_test_id(tid, test_framework) for tid in fail_to_pass] - norm_p2p = [normalize_test_id(tid, test_framework) for tid in pass_to_pass] - - # Handle synthetic build/compile tests - for tid in norm_f2p + norm_p2p: - if (tid.endswith("::build") or tid.endswith("::compile")) and tid not in parsed: - parsed[tid] = "PASSED" - - # Identify packages that failed to build - build_failed_packages = {pkg for pkg, status in parsed.items() if status == "FAILED" and "::" not in pkg} - - # Match FAIL_TO_PASS - f2p_results = {} - for tid in norm_f2p: - f2p_results[tid] = _match_test_with_fuzzy(tid, parsed, build_failed_packages) - - # Match PASS_TO_PASS - p2p_results = {} - for tid in norm_p2p: - p2p_results[tid] = _match_test_with_fuzzy(tid, parsed, build_failed_packages) - - all_f2p_passed = all(v == "PASSED" for v in f2p_results.values()) if f2p_results else False - all_p2p_passed = all(v == "PASSED" for v in p2p_results.values()) - resolved = all_f2p_passed and all_p2p_passed - - return { - "resolved": resolved, - "patch_exists": True, - "patch_successfully_applied": True, - "fail_to_pass_results": f2p_results, - "pass_to_pass_results": p2p_results, - "f2p_passed": sum(1 for v in f2p_results.values() if v == "PASSED"), - "f2p_total": len(f2p_results), - "p2p_passed": sum(1 for v in p2p_results.values() if v == "PASSED"), - "p2p_total": len(p2p_results), - "parsed_count": len(parsed), - "framework": test_framework, - } diff --git a/responses_api_agents/swe_agents/swe_env_adapter.py b/responses_api_agents/swe_agents/swe_env_adapter.py new file mode 100644 index 0000000000..e24da6da2d --- /dev/null +++ b/responses_api_agents/swe_agents/swe_env_adapter.py @@ -0,0 +1,415 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenHands ``swe_agents`` adapter onto the ``swe_env`` infrastructure. + +Provisions the agent's working container via ``swe_env.lifecycle``, injects a +sandbox-reachable model endpoint for egress, lets the agent self-drive inside that +container, extracts the unified-diff patch, then scores it through the verifier in +its own fresh sandbox. Environment provisioning and verification are decoupled from +the agent loop. +""" + +from __future__ import annotations + +import dataclasses +import json +import shlex +from collections.abc import Mapping +from typing import Any + +from nemo_gym.sandbox import SandboxProvider +from resources_servers.swe_env.verify_task import verify_task +from responses_api_agents.swe_env import get_harness, model_endpoint, reward_from_report +from responses_api_agents.swe_env.harness import SweTask +from responses_api_agents.swe_env.lifecycle import acquire_sandbox + + +def _provider_name(provider: Mapping[str, Any] | SandboxProvider) -> str: + """Return the name of a sandbox provider. + + Args: + provider: Either a mapping keyed by provider name, or a ``SandboxProvider`` + instance with a ``name`` attribute. + + Returns: + The provider name, or ``"?"`` if it cannot be determined. + """ + if isinstance(provider, Mapping): + return next(iter(provider), "?") + return getattr(provider, "name", "?") + + +async def _read_output_jsonl_row(env, output_glob: str) -> dict[str, Any]: + """Return the last row of the newest OpenHands ``output.jsonl`` (or ``{}`` if absent). + + OpenHands (``RUNTIME=local``) writes its result row to an ``output.jsonl`` file under + the eval output directory, with the patch at ``row["test_result"]["git_patch"]`` and any + agent failure at ``row["error"]`` — NOT to the working tree, so a plain ``git diff`` would + miss the patch. + + When several ``output.jsonl`` files match (e.g. a re-run left a stale one), the newest by + mtime is picked. ``find -printf "%T@ %p"`` emits `` `` per match; + ``sort -n | tail -1`` selects the most-recently-modified, and the leading float timestamp + plus single space is stripped back off. + + Args: + env: The sandbox handle exposing ``execute`` for running shell commands. + output_glob: Path or glob under which to search for ``output.jsonl`` files. + + Returns: + The parsed last JSON row of the newest matching ``output.jsonl`` as a dict, or an + empty dict if no file or content is found. + """ + found = await env.execute( + f'find {shlex.quote(output_glob)} -name output.jsonl -printf "%T@ %p\\n" 2>/dev/null | sort -n | tail -1' + ) + newest = (found.get("stdout", "") or "").strip() + # newest is " "; the path may contain spaces, so split only on the first one. + path = newest.split(" ", 1)[1].strip() if " " in newest else "" + if not path: + return {} + catted = await env.execute(f"cat {shlex.quote(path)}") + raw = (catted.get("stdout", "") or "").strip() + if not raw: + return {} + return json.loads(raw.splitlines()[-1]) + + +async def _extract_patch_from_output_jsonl(env, output_glob: str) -> str: + """Read the unified-diff patch from the newest OpenHands ``output.jsonl``. + + Args: + env: The sandbox handle exposing ``execute`` for running shell commands. + output_glob: Path or glob under which to search for ``output.jsonl`` files. + + Returns: + The patch string from ``row["test_result"]["git_patch"]``, or an empty string if + absent. + """ + row = await _read_output_jsonl_row(env, output_glob) + return (row.get("test_result") or {}).get("git_patch", "") or "" + + +# --- OpenHands self-driving launch builders ------------------------------------------------------- +# These target a single swe_env sandbox: the Gym repo is bind-mounted at its host path (so +# OpenHands' venv abs-symlinks + the nemo_gym editable install resolve), OpenHands self-drives +# RUNTIME=local on the family's workdir, and the patch is read from output.jsonl. + +_OH_OUTPUT_DIR = "/root/eval_results" +_OH_CONFIG_FILE = "/root/config.toml" +_OH_DATA_JSONL = "/root/dataset/data.jsonl" +_OH_METRICS_FPATH = "/root/nemo_gym_metrics.json" + + +def openhands_config_toml( + model: str, + *, + temperature: float = 0.0, + top_p: float = 1.0, + max_output_tokens: int | None = None, +) -> str: + """Build the OpenHands ``[llm.model]`` config TOML. + + ``native_tool_calling=false`` is more robust for small open models that don't emit a + strict tool-call format. By default no output-token cap is emitted (so litellm/the model + picks its own default); the cap line is emitted only when a caller explicitly passes one + (e.g. to bound an unknown model whose default max_tokens would be the full context window, + causing a vLLM 400 as the conversation grows). + + Args: + model: The model identifier to write into the config. + temperature: Sampling temperature. Defaults to ``0.0``. + top_p: Nucleus sampling probability mass. Defaults to ``1.0``. + max_output_tokens: Optional cap on output tokens. When ``None`` (the default), no + cap line is emitted. + + Returns: + The rendered ``config.toml`` contents as a string. + """ + lines = [ + "[llm.model]\n", + f'model = "{model}"\n', + 'api_key = "EMPTY"\n', # pragma: allowlist secret + 'custom_llm_provider = "openai"\n', + "native_tool_calling = false\n", + f"temperature = {float(temperature)}\n", + f"top_p = {float(top_p)}\n", + ] + if max_output_tokens is not None: + lines.append(f"max_output_tokens = {int(max_output_tokens)}\n") + lines.append("log_completions = true\n") + lines.append('log_completions_folder = "/root/completions"\n') + return "".join(lines) + + +def build_openhands_launch_command( + *, + setup_dir: str, + instance_id: str, + dataset_name: str, + split: str, + ng_config_dict_quoted: str, + model_server_name: str, + agent_cls: str = "CodeActAgent", + max_iter: int = 100, + command_exec_timeout: int = 300, + tmux_memory_limit_mb: int = 8192, +) -> str: + """Build the in-sandbox bash that runs OpenHands ``run_infer.sh`` (RUNTIME=local). + + Args: + setup_dir: Directory containing the OpenHands checkout and its miniforge3 install. + instance_id: The benchmark instance identifier to run. + dataset_name: OpenHands dataset/workspace selector, which sets its DATASET_TYPE + (e.g. ``SWE-Gym`` maps to ``/testbed``). + split: Dataset split to use (e.g. ``test``). + ng_config_dict_quoted: The already-shlex-quoted NeMo Gym global config dict, which + routes OpenHands' ``NemoGymClient`` back to the model server. + model_server_name: The NeMo Gym model server name to target. + agent_cls: OpenHands agent class to run. Defaults to ``"CodeActAgent"``. + max_iter: Maximum agent iterations. Defaults to ``100``. + command_exec_timeout: Per-command execution timeout in seconds. Defaults to ``300``. + tmux_memory_limit_mb: tmux memory limit in megabytes. Defaults to ``8192``. + + Returns: + The shell command string to execute inside the sandbox. + """ + oh = f"{setup_dir}/OpenHands" + return ( + "set -e && " + f"export PATH={setup_dir}/miniforge3/bin:$PATH && " + "git config --global --add safe.directory '*' && " + f"mkdir -p /root/completions /root/dataset {_OH_OUTPUT_DIR} && " + "uid=$(id -ru 2>/dev/null || id -u) && export TMUX_TMPDIR=/tmp && " + "export TMUX=/tmp/tmux-$uid/default && mkdir -p /tmp/tmux-$uid && chmod 700 /tmp/tmux-$uid && " + "tmux -S /tmp/tmux-$uid/default start-server || true && " + f"cd {oh} && export RUNTIME=local && " + "export LOG_LEVEL=CRITICAL && export LOG_TO_FILE=False && export DEBUG=False && " + f"export NEMO_GYM_METRICS_FPATH={_OH_METRICS_FPATH} && echo '{{}}' > $NEMO_GYM_METRICS_FPATH && " + f"export NEMO_GYM_CONFIG_DICT={ng_config_dict_quoted} && " + f"export NEMO_GYM_MODEL_SERVER_NAME={model_server_name} && " + f"export VIRTUAL_ENV={oh}/.venv && export PATH=$PATH:{oh}/.venv/bin && " + "export POETRY_VIRTUALENVS_IN_PROJECT=true && export POETRY_VIRTUALENVS_CREATE=false && " + f"export POETRY_VIRTUALENVS_PATH={oh} && " + f"export TMUX_MEMORY_LIMIT={tmux_memory_limit_mb} && export COMMAND_EXEC_TIMEOUT={command_exec_timeout} && " + "export PYTHONDONTWRITEBYTECODE=1 && " + "./evaluation/benchmarks/swe_bench/scripts/run_infer.sh " + f"llm.model '' {agent_cls} 0 {max_iter} 1 {dataset_name} {split} {_OH_OUTPUT_DIR} " + f"{instance_id} {_OH_DATA_JSONL} {_OH_CONFIG_FILE}" + ) + + +async def provision_and_extract_patch( + task: SweTask, + *, + provider: Mapping[str, Any] | SandboxProvider, + agent_launch_command: str, + model_server: Mapping[str, Any] | None = None, + opensandbox_service_url: str | None = None, + extra_env: Mapping[str, str] | None = None, + stage_files: Mapping[str, str] | None = None, + patch_output_glob: str | None = None, + agent_timeout_s: int | float = 1800, +) -> str: + """Provision a working sandbox, self-drive the agent, and return the unified-diff patch. + + No verification happens here — grading is the verifier's job (over HTTP). The patch is + returned to the caller, which POSTs it to the verifier. + + Two egress styles are supported: + + * ``model_server`` -> a sandbox-reachable OpenAI ``base_url`` (via ``model_endpoint.resolve``), + for agents that call the model via a standard OpenAI/litellm client (e.g. mini-swe-agent). + * ``extra_env`` -> injected verbatim, for agents hard-wired to NeMo Gym's ``ServerClient``. + The in-tree OpenHands fork's ``CodeActAgent`` unconditionally routes through + ``NemoGymClient`` (no litellm fallback), so it needs ``NEMO_GYM_CONFIG_DICT`` + + ``NEMO_GYM_MODEL_SERVER_NAME`` + ``NEMO_GYM_METRICS_FPATH`` — NOT ``OPENAI_BASE_URL``. + + Args: + task: The SWE task describing the instance, image, and working directory. + provider: The sandbox provider (mapping keyed by name, or a ``SandboxProvider``). + agent_launch_command: The shell command that runs the agent inside the sandbox. + model_server: Optional model-server config; when given, a sandbox-reachable endpoint + is resolved and injected into the agent's environment. + opensandbox_service_url: Optional OpenSandbox service URL used when resolving the + model endpoint. + extra_env: Optional environment variables injected verbatim into the sandbox. + stage_files: Optional ``{remote_path: content}`` files written into the live sandbox + before launch (e.g. OpenHands ``config.toml`` and the instance ``data.jsonl``). + patch_output_glob: When given, the patch is read from the OpenHands ``output.jsonl`` + under this path; otherwise it comes from ``git diff --cached`` on ``repo_workdir``. + agent_timeout_s: Timeout in seconds for the agent run. Defaults to ``1800``. + + Returns: + The extracted unified-diff patch as a string (empty if none was produced). + """ + result = await provision_and_collect( + task, + provider=provider, + agent_launch_command=agent_launch_command, + model_server=model_server, + opensandbox_service_url=opensandbox_service_url, + extra_env=extra_env, + stage_files=stage_files, + patch_output_glob=patch_output_glob, + agent_timeout_s=agent_timeout_s, + ) + return result["patch"] + + +def _build_agent_spec(task, provider, model_server, opensandbox_service_url, extra_env): + """Build the agent sandbox spec, injecting egress env (model endpoint and/or extra env). + + Args: + task: The SWE task whose benchmark selects the harness and seeds the spec. + provider: The sandbox provider, used to resolve the model endpoint for egress. + model_server: Optional model-server config; when given, a sandbox-reachable endpoint + is resolved and merged into the spec's environment. + opensandbox_service_url: Optional OpenSandbox service URL used when resolving the + model endpoint. + extra_env: Optional environment variables merged verbatim into the spec. + + Returns: + The sandbox spec with egress environment variables applied. + """ + harness = get_harness(task.benchmark) + spec = harness.build_spec(task) + # Model-server egress: inject only a sandbox-reachable endpoint (never the global dict). + if model_server is not None: + endpoint = model_endpoint.resolve( + _provider_name(provider), model_server, opensandbox_service_url=opensandbox_service_url + ) + spec = dataclasses.replace(spec, env={**spec.env, **endpoint.to_sandbox_env()}) + # NeMo-Gym-client egress / any extra in-sandbox env (e.g. OpenHands NEMO_GYM_* vars). + if extra_env: + spec = dataclasses.replace(spec, env={**spec.env, **dict(extra_env)}) + return spec + + +async def provision_and_collect( + task: SweTask, + *, + provider: Mapping[str, Any] | SandboxProvider, + agent_launch_command: str, + model_server: Mapping[str, Any] | None = None, + opensandbox_service_url: str | None = None, + extra_env: Mapping[str, str] | None = None, + stage_files: Mapping[str, str] | None = None, + patch_output_glob: str | None = None, + agent_timeout_s: int | float = 1800, +) -> dict[str, Any]: + """Provision and self-drive the agent, returning the patch and error signals. + + A superset of ``provision_and_extract_patch`` — it also surfaces the OpenHands + ``output.jsonl`` ``error`` field so the worker can classify ``agent_error_kind`` for + masking, and the agent ``env.execute`` ``error_type`` (``"timeout"``, ``"sandbox"``, or + ``None``). ``env.execute`` does not raise on timeout; it returns ``error_type`` instead, so + the worker must read this field to set ``agent_timed_out`` (otherwise a timed-out agent + would wrongly not be masked). + + Args: + task: The SWE task describing the instance, image, and working directory. + provider: The sandbox provider (mapping keyed by name, or a ``SandboxProvider``). + agent_launch_command: The shell command that runs the agent inside the sandbox. + model_server: Optional model-server config; when given, a sandbox-reachable endpoint + is resolved and injected into the agent's environment. + opensandbox_service_url: Optional OpenSandbox service URL used when resolving the + model endpoint. + extra_env: Optional environment variables injected verbatim into the sandbox. + stage_files: Optional ``{remote_path: content}`` files written into the live sandbox + before launch. + patch_output_glob: When given, the patch is read from the OpenHands ``output.jsonl`` + under this path; otherwise it comes from ``git diff --cached`` on ``repo_workdir``. + agent_timeout_s: Timeout in seconds for the agent run. Defaults to ``1800``. + + Returns: + A dict with keys ``"patch"`` (the unified-diff string), ``"agent_error"`` (the + OpenHands error field or ``None``), and ``"error_type"`` (``"timeout"``, + ``"sandbox"``, or ``None``). + """ + spec = _build_agent_spec(task, provider, model_server, opensandbox_service_url, extra_env) + async with acquire_sandbox(provider, spec, instance_id=task.instance_id) as env: + for remote_path, content in (stage_files or {}).items(): + await env.write_text(remote_path, content) + run = await env.execute(agent_launch_command, cwd=task.repo_workdir, timeout_s=agent_timeout_s) + error_type = run.get("error_type") + if patch_output_glob: + row = await _read_output_jsonl_row(env, patch_output_glob) + patch = (row.get("test_result") or {}).get("git_patch", "") or "" + return {"patch": patch, "agent_error": row.get("error"), "error_type": error_type} + diff = await env.execute(f"cd {task.repo_workdir} && git add -A && git diff --cached", cwd=task.repo_workdir) + return {"patch": diff.get("stdout", "") or "", "agent_error": None, "error_type": error_type} + + +async def run_self_driving( + task: SweTask, + *, + provider: Mapping[str, Any] | SandboxProvider, + agent_launch_command: str, + model_server: Mapping[str, Any] | None = None, + opensandbox_service_url: str | None = None, + extra_env: Mapping[str, str] | None = None, + stage_files: Mapping[str, str] | None = None, + patch_output_glob: str | None = None, + agent_timeout_s: int | float = 1800, +) -> dict[str, Any]: + """Provision, self-drive, extract the patch, then grade it in-process via ``verify_task``. + + This bundles provisioning and verification for standalone reproduction and tests. In + production, verification runs over HTTP (the agent worker extracts the patch and POSTs it + to the verifier separately). + + Args: + task: The SWE task describing the instance, image, and working directory. + provider: The sandbox provider (mapping keyed by name, or a ``SandboxProvider``). + agent_launch_command: The shell command that runs the agent inside the sandbox. + model_server: Optional model-server config; when given, a sandbox-reachable endpoint + is resolved and injected into the agent's environment. + opensandbox_service_url: Optional OpenSandbox service URL used when resolving the + model endpoint. + extra_env: Optional environment variables injected verbatim into the sandbox. + stage_files: Optional ``{remote_path: content}`` files written into the live sandbox + before launch. + patch_output_glob: When given, the patch is read from the OpenHands ``output.jsonl`` + under this path; otherwise it comes from ``git diff --cached`` on ``repo_workdir``. + agent_timeout_s: Timeout in seconds for the agent run. Defaults to ``1800``. + + Returns: + A dict with the instance id, model patch, resolution status, reward, whether a patch + exists, whether the sample is masked, and the verifier's error kind. + """ + patch = await provision_and_extract_patch( + task, + provider=provider, + agent_launch_command=agent_launch_command, + model_server=model_server, + opensandbox_service_url=opensandbox_service_url, + extra_env=extra_env, + stage_files=stage_files, + patch_output_glob=patch_output_glob, + agent_timeout_s=agent_timeout_s, + ) + # Score the patch in the verifier's OWN fresh sandbox (decoupled verification). + report = await verify_task(provider, dataclasses.replace(task, model_patch=patch)) + masked = report.error_kind is not None + return { + "instance_id": task.instance_id, + "model_patch": patch, + "resolved": report.resolved, + "reward": reward_from_report(report), + "patch_exists": bool(patch.strip()), + "mask_sample": masked, + "error_kind": report.error_kind, + } diff --git a/responses_api_agents/swe_agents/tests/test_app.py b/responses_api_agents/swe_agents/tests/test_app.py index 77144bc63d..f04b6b2a1f 100644 --- a/responses_api_agents/swe_agents/tests/test_app.py +++ b/responses_api_agents/swe_agents/tests/test_app.py @@ -12,7 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import asyncio +"""Unit tests for the swe_agents app: config models, dataset processors, the OpenHands agent +runner, and the SWEBenchWrapper, including sample masking and the verifier POST contract.""" + import json import shutil import tempfile @@ -30,17 +32,14 @@ ) from nemo_gym.server_utils import ServerClient from responses_api_agents.swe_agents.app import ( - ActiveContainerCommand, AgentPromptOverride, BaseDatasetHarnessProcessor, ExecuteContainerCommandArgs, NVInternalDatasetProcessor, - OpenHandsHarnessProcessor, R2EGymDatasetProcessor, RunOpenHandsAgent, SweBenchDatasetProcessor, SWEBenchMetrics, - SweBenchMultilingualDatasetProcessor, SWEBenchVerifyResponse, SWEBenchWrapper, SWEBenchWrapperConfig, @@ -403,25 +402,11 @@ def test_setup_returns_none(self) -> None: processor = BaseDatasetHarnessProcessor(config=config) assert processor.setup() is None - def test_get_run_command_returns_none(self) -> None: - config = _minimal_server_config() - processor = BaseDatasetHarnessProcessor(config=config) - assert processor.get_run_command() is None - def test_postprocess_after_run_returns_none(self) -> None: config = _minimal_server_config() processor = BaseDatasetHarnessProcessor(config=config) assert processor.postprocess_after_run(Path("/tmp/report.json")) is None - def test_get_command_sleep_until_predictions_file(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config(tmpdir) - processor = BaseDatasetHarnessProcessor(config=config) - cmd = processor._get_command_sleep_until_predictions_file() - assert "until" in cmd - assert "sleep 5" in cmd - assert str(config.output_for_eval_mounted_path) in cmd - def test_run_setup_command_success(self) -> None: config = _minimal_server_config() processor = BaseDatasetHarnessProcessor(config=config) @@ -495,52 +480,6 @@ def _make_processor(self, tmpdir, instance_dict_override=None) -> NVInternalData ) return NVInternalDatasetProcessor(config=config) - def test_get_run_command(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - processor = self._make_processor(tmpdir) - result = processor.get_run_command() - assert isinstance(result, ExecuteContainerCommandArgs) - assert result.mode == "eval" - assert "git reset --hard abc123" in result.command - assert "git apply" in result.command - assert "run_script.sh" in result.command - assert "parsing_script.py" in result.command - - def test_get_run_command_env_parsing(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - processor = self._make_processor( - tmpdir, - { - "base_dockerfile": "ENV KEY=VALUE\nENV SPACE_KEY some_value", - "instance_dockerfile": "", - }, - ) - result = processor.get_run_command() - assert "export KEY=VALUE" in result.command - assert 'export SPACE_KEY="some_value"' in result.command - - def test_get_run_command_list_test_files(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - processor = self._make_processor( - tmpdir, - { - "selected_test_files_to_run": ["test_x.py", "test_y.py"], - }, - ) - result = processor.get_run_command() - assert "test_x.py,test_y.py" in result.command - - def test_get_run_command_no_repo_cmd(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - processor = self._make_processor( - tmpdir, - { - "before_repo_set_cmd": "", - }, - ) - result = processor.get_run_command() - assert isinstance(result, ExecuteContainerCommandArgs) - def test_check_tests_passed_all_pass(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: processor = self._make_processor(tmpdir) @@ -660,70 +599,6 @@ def test_normalize_test_name_multiple_patterns(self) -> None: assert SWERebenchDatasetProcessor._normalize_test_name("test_foo [2s]") == "test_foo" assert SWERebenchDatasetProcessor._normalize_test_name("test_foo [200ms]") == "test_foo" - def test_get_run_command(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - instance_dict = { - "install_config": { - "test_cmd": ["pytest tests/"], - "install": ["pip install -e ."], - "log_parser": "pytest_parser", - }, - "repo": "owner/repo_name", - "test_patch": "diff --git a/test.py b/test.py\n", - "FAIL_TO_PASS": '["test_a"]', - "PASS_TO_PASS": '["test_b"]', - } - config = _make_instance_config( - tmpdir, - problem_info={ - "problem_statement": "Fix", - "instance_id": "owner__repo-123", - "base_commit": "abc", - "dataset_name": "SWE-rebench", - "split": "test", - "instance_dict": json.dumps(instance_dict), - "container_formatter": ["/containers/{instance_id}.sif"], - }, - ) - processor = SWERebenchDatasetProcessor(config=config) - result = processor.get_run_command() - assert isinstance(result, ExecuteContainerCommandArgs) - assert "pytest tests/" in result.command - assert "pip install -e ." in result.command - assert "git apply" in result.command - assert result.mode == "eval" - - # Check that eval metadata files were written - eval_meta_dir = config.persistent_dir / "eval_meta" - assert (eval_meta_dir / "expected_passed.json").exists() - assert (eval_meta_dir / "fail_to_pass.json").exists() - assert (eval_meta_dir / "pass_to_pass.json").exists() - - def test_get_run_command_string_test_cmd(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - instance_dict = { - "install_config": {"test_cmd": "pytest tests/", "install": "pip install ."}, - "repo": "owner/repo", - "test_patch": "", - "FAIL_TO_PASS": [], - "PASS_TO_PASS": [], - } - config = _make_instance_config( - tmpdir, - problem_info={ - "problem_statement": "Fix", - "instance_id": "owner__repo-1", - "base_commit": "abc", - "dataset_name": "SWE-rebench", - "split": "test", - "instance_dict": json.dumps(instance_dict), - "container_formatter": ["/containers/{instance_id}.sif"], - }, - ) - processor = SWERebenchDatasetProcessor(config=config) - result = processor.get_run_command() - assert "pytest tests/" in result.command - def test_postprocess_no_test_output(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: instance_dict = {"install_config": {"log_parser": "pytest_parser"}} @@ -775,161 +650,6 @@ def test_setup_already_exists(self) -> None: result = processor.setup() assert result == setup_dir - def test_get_run_command(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config(tmpdir) - processor = SweBenchDatasetProcessor(config=config) - result = processor.get_run_command() - assert isinstance(result, ExecuteContainerCommandArgs) - assert "run_local_evaluation" in result.command - assert "django__django-12345" in result.command - assert result.mode == "eval" - assert result.timeout == config.swebench_tests_timeout + 120 - - -######################################## -# SweBenchMultilingualDatasetProcessor tests -######################################## - - -class TestSweBenchMultilingualDatasetProcessor: - def test_get_run_command(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config( - tmpdir, - swebench_multilingual_setup_dir=Path(tmpdir) / "swebench_ml", - ) - processor = SweBenchMultilingualDatasetProcessor(config=config) - result = processor.get_run_command() - assert isinstance(result, ExecuteContainerCommandArgs) - assert "SWE-bench_Multilingual" in result.command - assert result.mode == "eval" - - -######################################## -# R2EGymDatasetProcessor tests -######################################## - - -class TestR2EGymDatasetProcessor: - def test_get_run_command(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config(tmpdir) - processor = R2EGymDatasetProcessor(config=config) - result = processor.get_run_command() - assert isinstance(result, ExecuteContainerCommandArgs) - assert "run_local_evaluation.py" in result.command - assert result.mode == "eval" - - -######################################## -# OpenHandsHarnessProcessor tests -######################################## - - -class TestOpenHandsHarnessProcessor: - def test_get_run_command(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config(tmpdir) - config.persistent_dir.mkdir(parents=True, exist_ok=True) - processor = OpenHandsHarnessProcessor(config=config) - result = processor.get_run_command() - assert isinstance(result, ExecuteContainerCommandArgs) - assert result.mode == "agent" - assert "timeout" in result.command - assert "run_infer.sh" in self._read_agent_script(config) - - def _read_agent_script(self, config) -> str: - # The script is written at persistent_dir / agent_script_{agent_run_id}.sh - script_path = config.persistent_dir / f"agent_script_{config.agent_run_id}.sh" - return script_path.read_text() - - def test_get_run_command_with_debug(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config(tmpdir, debug=True) - config.persistent_dir.mkdir(parents=True, exist_ok=True) - processor = OpenHandsHarnessProcessor(config=config) - processor.get_run_command() - assert "NG_PROFILING_DIR" in self._read_agent_script(config) - - def test_get_run_command_with_logging(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config(tmpdir, openhands_should_log=True) - config.persistent_dir.mkdir(parents=True, exist_ok=True) - processor = OpenHandsHarnessProcessor(config=config) - processor.get_run_command() - assert "LOG_LEVEL=DEBUG" in self._read_agent_script(config) - - def test_get_run_command_nv_internal(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config( - tmpdir, - problem_info={ - "problem_statement": "Fix", - "instance_id": "nv__test-1", - "base_commit": "abc", - "dataset_name": "nv-internal-1", - "split": "test", - "instance_dict": "{}", - "container_formatter": ["/containers/{instance_id}.sif"], - }, - ) - config.persistent_dir.mkdir(parents=True, exist_ok=True) - processor = OpenHandsHarnessProcessor(config=config) - processor.get_run_command() - assert "cryptography" in self._read_agent_script(config) - - def test_get_run_command_swe_rebench(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config( - tmpdir, - problem_info={ - "problem_statement": "Fix", - "instance_id": "owner__repo-1", - "base_commit": "abc", - "dataset_name": "SWE-rebench", - "split": "test", - "instance_dict": "{}", - "container_formatter": ["/containers/{instance_id}.sif"], - }, - ) - config.persistent_dir.mkdir(parents=True, exist_ok=True) - processor = OpenHandsHarnessProcessor(config=config) - processor.get_run_command() - script = self._read_agent_script(config) - # Should skip workspace check for SWE-rebench - assert "Exiting because /workspace" not in script - - def test_get_run_command_with_prompt_overrides(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config( - tmpdir, - resolved_user_prompt_template="/path/to/user_prompt.j2", - resolved_system_prompt_template="/path/to/system_prompt.j2", - ) - config.persistent_dir.mkdir(parents=True, exist_ok=True) - processor = OpenHandsHarnessProcessor(config=config) - processor.get_run_command() - script = self._read_agent_script(config) - assert "user_prompt.j2" in script - assert "system_prompt.j2" in script - - def test_get_run_command_diversify_tool_names(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config(tmpdir, resolved_diversify_tool_names=True) - config.persistent_dir.mkdir(parents=True, exist_ok=True) - processor = OpenHandsHarnessProcessor(config=config) - processor.get_run_command() - assert "DIVERSIFY_TOOL_NAMES=true" in self._read_agent_script(config) - - def test_get_run_command_camel_case_tool_names(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - config = _make_instance_config(tmpdir, resolved_camel_case_tool_names=True) - config.persistent_dir.mkdir(parents=True, exist_ok=True) - processor = OpenHandsHarnessProcessor(config=config) - processor.get_run_command() - assert "CAMEL_CASE_TOOL_NAMES=true" in self._read_agent_script(config) - ######################################## # runner_ray_remote tests @@ -941,30 +661,6 @@ def test_is_ray_remote(self) -> None: assert hasattr(runner_ray_remote, "remote") -######################################## -# ActiveContainerCommand tests -######################################## - - -class TestActiveContainerCommand: - @pytest.mark.asyncio - async def test_creation(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - log_path = Path(tmpdir) / "test.log" - log_file = open(log_path, "w") - try: - process = await asyncio.create_subprocess_shell("true", stdout=log_file, stderr=log_file) - cmd = ActiveContainerCommand( - process=process, - log_file=log_file, - log_file_path=log_path, - ) - assert cmd.log_file_path == log_path - await process.wait() - finally: - log_file.close() - - ######################################## # RunOpenHandsAgent tests ######################################## @@ -1034,144 +730,71 @@ def test_openhands_dir_copy_no_output_file_found(self) -> None: agent._openhands_dir_copy_from_host(output_file_path="nonexistent.jsonl") @pytest.mark.asyncio - async def test_start_container_command(self) -> None: + async def test_run_decoupled_agent_timeout_sets_agent_timed_out(self, monkeypatch) -> None: + """env.execute does not raise on agent timeout (it returns error_type="timeout"), so + provision_and_collect succeeds and the except never fires. The worker must still set + agent_timed_out from the surfaced error_type so the sample is masked.""" with tempfile.TemporaryDirectory() as tmpdir: - agent = self._make_agent(tmpdir) - agent.config.persistent_dir.mkdir(parents=True, exist_ok=True) - - cmd = ExecuteContainerCommandArgs( - command="echo hello", - expected_file_pattern="/tmp/*.json", - mode="agent", - timeout=10, - ) - - active = await agent._start_container_command(cmd, "echo done") - await active.process.wait() - active.log_file.close() - assert active.log_file_path.exists() - - @pytest.mark.asyncio - async def test_finish_container_command_success(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - agent = self._make_agent(tmpdir) - agent.config.persistent_dir.mkdir(parents=True, exist_ok=True) - - expected_file = Path(tmpdir) / "output.json" - expected_file.write_text("{}") - - cmd = ExecuteContainerCommandArgs( - command="echo hello", - expected_file_pattern=str(expected_file), - mode="eval", - timeout=10, + agent = self._make_agent( + tmpdir, + problem_info={ + "problem_statement": "Fix bug", + "instance_id": "psf__requests-2317", + "base_commit": "abc123", + "dataset_name": "swe-bench-ext", + "split": "test", + "instance_dict": json.dumps({"FAIL_TO_PASS": [], "PASS_TO_PASS": []}), + }, ) - active = await agent._start_container_command(cmd, "echo done") - result = await agent._finish_container_command(active, cmd) - assert result == str(expected_file) + agent.config.metrics_fpath.parent.mkdir(parents=True, exist_ok=True) + agent.config.metrics_fpath.write_text("{}") - @pytest.mark.asyncio - async def test_finish_container_command_no_file(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - agent = self._make_agent(tmpdir) - agent.config.persistent_dir.mkdir(parents=True, exist_ok=True) + # provision returns a non-raising timeout: patch present but the agent self-driver + # was killed by the timeout. + async def _fake_provision(*_args, **_kwargs): + return {"patch": "", "agent_error": None, "error_type": "timeout"} - cmd = ExecuteContainerCommandArgs( - command="echo hello", - expected_file_pattern=str(Path(tmpdir) / "nonexistent*.json"), - mode="eval", - timeout=10, - ) - active = await agent._start_container_command(cmd, "echo done") - with pytest.raises(ValueError, match="Expected exactly one file"): - await agent._finish_container_command(active, cmd) + import responses_api_agents.swe_agents.swe_env_adapter as adapter - @pytest.mark.asyncio - async def test_finish_container_command_multiple_files(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - agent = self._make_agent(tmpdir) - agent.config.persistent_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(adapter, "provision_and_collect", _fake_provision) - (Path(tmpdir) / "output1.json").write_text("{}") - import time as _time + result = await agent._run_decoupled_agent() - _time.sleep(0.05) - (Path(tmpdir) / "output2.json").write_text("{}") - - cmd = ExecuteContainerCommandArgs( - command="echo hello", - expected_file_pattern=str(Path(tmpdir) / "output*.json"), - mode="eval", - timeout=10, - ) - active = await agent._start_container_command(cmd, "echo done") - result = await agent._finish_container_command(active, cmd) - assert "output2.json" in result # should pick latest + assert result is None # decoupled contract: no report file + persisted = swe_app.SWEBenchMetrics.model_validate_json(agent.config.metrics_fpath.read_text()) + assert persisted.agent_timed_out is True @pytest.mark.asyncio - async def test_finish_container_command_timeout(self) -> None: + async def test_run_decoupled_agent_clean_run_not_timed_out(self, monkeypatch) -> None: + """A clean agent run (no error_type) must not set agent_timed_out, so a valid sample is + graded normally.""" with tempfile.TemporaryDirectory() as tmpdir: - agent = self._make_agent(tmpdir) - agent.config.persistent_dir.mkdir(parents=True, exist_ok=True) - - cmd = ExecuteContainerCommandArgs( - command="sleep 100", - expected_file_pattern=str(Path(tmpdir) / "*.json"), - mode="agent", - timeout=1, + agent = self._make_agent( + tmpdir, + problem_info={ + "problem_statement": "Fix bug", + "instance_id": "psf__requests-2317", + "base_commit": "abc123", + "dataset_name": "swe-bench-ext", + "split": "test", + "instance_dict": json.dumps({"FAIL_TO_PASS": [], "PASS_TO_PASS": []}), + }, ) - active = await agent._start_container_command(cmd, "sleep 100") - with pytest.raises(ValueError, match="timed out"): - await agent._finish_container_command(active, cmd) + agent.config.metrics_fpath.parent.mkdir(parents=True, exist_ok=True) + agent.config.metrics_fpath.write_text("{}") - @pytest.mark.asyncio - async def test_finish_container_command_nonzero_exit(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - agent = self._make_agent(tmpdir) - agent.config.persistent_dir.mkdir(parents=True, exist_ok=True) + async def _fake_provision(*_args, **_kwargs): + return {"patch": "DIFF\n", "agent_error": None, "error_type": None} - cmd = ExecuteContainerCommandArgs( - command="exit 1", - expected_file_pattern=str(Path(tmpdir) / "*.json"), - mode="eval", - timeout=10, - ) - active = await agent._start_container_command(cmd, "bash -c 'exit 1'") - with pytest.raises(RuntimeError, match="Command failed with return code"): - await agent._finish_container_command(active, cmd) + import responses_api_agents.swe_agents.swe_env_adapter as adapter - @pytest.mark.asyncio - async def test_kill_active_command(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - agent = self._make_agent(tmpdir) - agent.config.persistent_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr(adapter, "provision_and_collect", _fake_provision) - cmd = ExecuteContainerCommandArgs( - command="sleep 100", - expected_file_pattern="/tmp/*.json", - mode="agent", - timeout=60, - ) - active = await agent._start_container_command(cmd, "sleep 100") - await agent._kill_active_command(active) - assert active.process.returncode is not None + await agent._run_decoupled_agent() - @pytest.mark.asyncio - async def test_kill_active_command_already_finished(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - agent = self._make_agent(tmpdir) - agent.config.persistent_dir.mkdir(parents=True, exist_ok=True) - - cmd = ExecuteContainerCommandArgs( - command="true", - expected_file_pattern="/tmp/*.json", - mode="agent", - timeout=10, - ) - active = await agent._start_container_command(cmd, "true") - await active.process.wait() - # Should not raise even if already finished - await agent._kill_active_command(active) + persisted = swe_app.SWEBenchMetrics.model_validate_json(agent.config.metrics_fpath.read_text()) + assert persisted.agent_timed_out in (False, None) + assert persisted.patch_exists is True ######################################## @@ -1204,386 +827,6 @@ def test_resolve_absolute_path_relative(self, monkeypatch) -> None: assert Path(result).is_absolute() -class TestSWEBenchWrapperFindContainer: - def _create_wrapper_for_find(self, monkeypatch) -> SWEBenchWrapper: - return _create_wrapper(monkeypatch) - - def test_exact_match(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - container_file = Path(tmpdir) / "django__django-12345.sif" - container_file.touch() - - data_point = { - "instance_id": "django__django-12345", - "dataset_name": "SWE-bench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - def test_string_container_formatter(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - container_file = Path(tmpdir) / "django__django-12345.sif" - container_file.touch() - - data_point = { - "instance_id": "django__django-12345", - "dataset_name": "SWE-bench", - "container_formatter": str(Path(tmpdir) / "{instance_id}.sif"), - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - def test_1776_replacement(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - container_file = Path(tmpdir) / "django_1776_django-12345.sif" - container_file.touch() - - data_point = { - "instance_id": "django__django-12345", - "dataset_name": "SWE-bench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - def test_s_replacement(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - container_file = Path(tmpdir) / "django_s_django-12345.sif" - container_file.touch() - - data_point = { - "instance_id": "django__django-12345", - "dataset_name": "SWE-bench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - def test_lowercase_match(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - container_file = Path(tmpdir) / "django_1776_django-12345.sif" - container_file.touch() - - data_point = { - "instance_id": "Django__Django-12345", - "dataset_name": "SWE-bench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - result = wrapper._find_container(data_point) - assert "django" in result.lower() - - def test_fuzzy_search(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - container_file = Path(tmpdir) / "prefix_django__django-12345_suffix.sif" - container_file.touch() - - data_point = { - "instance_id": "django__django-12345", - "dataset_name": "SWE-bench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - def test_not_found(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - data_point = { - "instance_id": "nonexistent__repo-123", - "dataset_name": "SWE-bench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - with pytest.raises(FileNotFoundError, match="No container file found"): - wrapper._find_container(data_point) - - def test_r2e_gym_dataset(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - # R2E-Gym modifies instance_id: org__RepoName- -> reponame_final_ - container_file = Path(tmpdir) / "reponame_final_123.sif" - container_file.touch() - - data_point = { - "instance_id": "org__RepoName-123", - "dataset_name": "R2E-Gym/R2E-Gym-Subset", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - def test_swe_rebench_dataset(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - # SWE-rebench fuzzy match: glob {instance_id}*.sif against the directory - container_file = Path(tmpdir) / "owner__repo-123-abc.sif" - container_file.touch() - - data_point = { - "instance_id": "owner__repo-123", - "dataset_name": "SWE-rebench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - def test_swe_rebench_exact_match(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - container_file = Path(tmpdir) / "owner__repo-123.sif" - container_file.touch() - - data_point = { - "instance_id": "owner__repo-123", - "dataset_name": "SWE-rebench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - def test_swe_rebench_not_found(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - data_point = { - "instance_id": "owner__repo-123", - "dataset_name": "SWE-rebench", - "container_formatter": [str(Path(tmpdir) / "{instance_id}.sif")], - } - with pytest.raises(FileNotFoundError, match="No SIF found"): - wrapper._find_container(data_point) - - def test_multiple_container_formatters(self, monkeypatch) -> None: - wrapper = self._create_wrapper_for_find(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - dir2 = Path(tmpdir) / "dir2" - dir2.mkdir() - container_file = dir2 / "django__django-12345.sif" - container_file.touch() - - data_point = { - "instance_id": "django__django-12345", - "dataset_name": "SWE-bench", - "container_formatter": [ - str(Path(tmpdir) / "dir1" / "{instance_id}.sif"), - str(dir2 / "{instance_id}.sif"), - ], - } - result = wrapper._find_container(data_point) - assert result == str(container_file) - - -class TestSWEBenchWrapperBuildApptainerCommand: - def test_basic_command(self, monkeypatch) -> None: - wrapper = _create_wrapper(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - params = _make_instance_config(tmpdir) - params.persistent_dir.mkdir(parents=True, exist_ok=True) - (params.persistent_dir / "container_scripts").mkdir(parents=True, exist_ok=True) - - # Create openhands dirs needed for mount - oh_dir = Path(params.openhands_setup_dir) / "OpenHands" - for subdir in [".eval_sessions", "logs", "evaluation/oh"]: - (oh_dir / subdir).mkdir(parents=True, exist_ok=True) - miniforge = Path(params.openhands_setup_dir) / "miniforge3" - miniforge.mkdir(parents=True, exist_ok=True) - - cmd_args = ExecuteContainerCommandArgs( - command="echo hello", - expected_file_pattern="/tmp/*.json", - mode="agent", - timeout=300, - ) - result = wrapper._build_apptainer_command(params, cmd_args) - assert "apptainer exec" in result - assert "--writable-tmpfs" in result - assert params.container in result - - def test_eval_mode_swebench_mounts(self, monkeypatch) -> None: - wrapper = _create_wrapper(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - params = _make_instance_config(tmpdir) - params.persistent_dir.mkdir(parents=True, exist_ok=True) - - oh_dir = Path(params.openhands_setup_dir) / "OpenHands" - for subdir in [".eval_sessions", "logs", "evaluation/oh"]: - (oh_dir / subdir).mkdir(parents=True, exist_ok=True) - (Path(params.openhands_setup_dir) / "miniforge3").mkdir(parents=True, exist_ok=True) - - cmd_args = ExecuteContainerCommandArgs( - command="run_eval", - expected_file_pattern="/tmp/*.json", - mode="eval", - timeout=300, - ) - result = wrapper._build_apptainer_command(params, cmd_args) - assert "/swebench_setup" in result - - def test_memory_limit(self, monkeypatch) -> None: - wrapper = _create_wrapper(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - params = _make_instance_config(tmpdir, apptainer_memory_limit_mb=16384) - params.persistent_dir.mkdir(parents=True, exist_ok=True) - - oh_dir = Path(params.openhands_setup_dir) / "OpenHands" - for subdir in [".eval_sessions", "logs", "evaluation/oh"]: - (oh_dir / subdir).mkdir(parents=True, exist_ok=True) - (Path(params.openhands_setup_dir) / "miniforge3").mkdir(parents=True, exist_ok=True) - - cmd_args = ExecuteContainerCommandArgs( - command="echo hello", - expected_file_pattern="/tmp/*.json", - mode="agent", - timeout=300, - ) - result = wrapper._build_apptainer_command(params, cmd_args) - assert "ulimit -v" in result - - def test_nv_internal_eval_mounts(self, monkeypatch) -> None: - wrapper = _create_wrapper(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - params = _make_instance_config( - tmpdir, - problem_info={ - "problem_statement": "Fix", - "instance_id": "nv__test-1", - "base_commit": "abc", - "dataset_name": "nv-internal-1", - "split": "test", - "instance_dict": "{}", - "container_formatter": ["/containers/{instance_id}.sif"], - }, - ) - params.persistent_dir.mkdir(parents=True, exist_ok=True) - (params.persistent_dir / "run_script.sh").write_text("#!/bin/bash") - (params.persistent_dir / "parsing_script.py").write_text("print('ok')") - - oh_dir = Path(params.openhands_setup_dir) / "OpenHands" - for subdir in [".eval_sessions", "logs", "evaluation/oh"]: - (oh_dir / subdir).mkdir(parents=True, exist_ok=True) - (Path(params.openhands_setup_dir) / "miniforge3").mkdir(parents=True, exist_ok=True) - - cmd_args = ExecuteContainerCommandArgs( - command="run_eval", - expected_file_pattern="/tmp/*.json", - mode="eval", - timeout=300, - ) - result = wrapper._build_apptainer_command(params, cmd_args) - assert "/root/run_script.sh" in result - assert "/root/parsing_script.py" in result - assert "/root/patch.diff" in result - - def test_r2e_gym_agent_removes_tests(self, monkeypatch) -> None: - wrapper = _create_wrapper(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - params = _make_instance_config( - tmpdir, - problem_info={ - "problem_statement": "Fix", - "instance_id": "org__Repo-1", - "base_commit": "abc", - "dataset_name": "R2E-Gym/R2E-Gym-Subset", - "split": "test", - "instance_dict": "{}", - "container_formatter": ["/containers/{instance_id}.sif"], - }, - ) - params.persistent_dir.mkdir(parents=True, exist_ok=True) - - oh_dir = Path(params.openhands_setup_dir) / "OpenHands" - for subdir in [".eval_sessions", "logs", "evaluation/oh"]: - (oh_dir / subdir).mkdir(parents=True, exist_ok=True) - (Path(params.openhands_setup_dir) / "miniforge3").mkdir(parents=True, exist_ok=True) - - cmd_args = ExecuteContainerCommandArgs( - command="run_agent", - expected_file_pattern="/tmp/*.json", - mode="agent", - timeout=300, - ) - wrapper._build_apptainer_command(params, cmd_args) - # The rm -rf commands are in the container script, not the apptainer command - script_path = params.persistent_dir / "container_scripts" / "agent_script.sh" - script_content = script_path.read_text() - assert "rm -rf" in script_content - assert "r2e_tests" in script_content - - def test_swe_rebench_eval_env_args(self, monkeypatch) -> None: - wrapper = _create_wrapper(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - params = _make_instance_config( - tmpdir, - problem_info={ - "problem_statement": "Fix", - "instance_id": "owner__repo-1", - "base_commit": "abc", - "dataset_name": "SWE-rebench", - "split": "test", - "instance_dict": "{}", - "container_formatter": ["/containers/{instance_id}.sif"], - }, - ) - params.persistent_dir.mkdir(parents=True, exist_ok=True) - - # Create eval meta files - eval_meta_dir = params.persistent_dir / "eval_meta" - eval_meta_dir.mkdir(parents=True, exist_ok=True) - (eval_meta_dir / "expected_passed.json").write_text("[]") - (eval_meta_dir / "fail_to_pass.json").write_text("[]") - (eval_meta_dir / "pass_to_pass.json").write_text("[]") - - oh_dir = Path(params.openhands_setup_dir) / "OpenHands" - for subdir in [".eval_sessions", "logs", "evaluation/oh"]: - (oh_dir / subdir).mkdir(parents=True, exist_ok=True) - (Path(params.openhands_setup_dir) / "miniforge3").mkdir(parents=True, exist_ok=True) - - cmd_args = ExecuteContainerCommandArgs( - command="run_eval", - expected_file_pattern="/tmp/*.json", - mode="eval", - timeout=300, - ) - result = wrapper._build_apptainer_command(params, cmd_args) - assert "_JAVA_OPTIONS" in result - assert "/swe_rebench_setup" in result - - def test_prompt_template_mounts(self, monkeypatch) -> None: - wrapper = _create_wrapper(monkeypatch) - with tempfile.TemporaryDirectory() as tmpdir: - user_prompt = Path(tmpdir) / "user_prompt.j2" - system_prompt = Path(tmpdir) / "system_prompt.j2" - user_prompt.write_text("user prompt") - system_prompt.write_text("system prompt") - - params = _make_instance_config( - tmpdir, - resolved_user_prompt_template=str(user_prompt), - resolved_system_prompt_template=str(system_prompt), - ) - params.persistent_dir.mkdir(parents=True, exist_ok=True) - - oh_dir = Path(params.openhands_setup_dir) / "OpenHands" - for subdir in [".eval_sessions", "logs", "evaluation/oh"]: - (oh_dir / subdir).mkdir(parents=True, exist_ok=True) - (Path(params.openhands_setup_dir) / "miniforge3").mkdir(parents=True, exist_ok=True) - - cmd_args = ExecuteContainerCommandArgs( - command="echo hello", - expected_file_pattern="/tmp/*.json", - mode="agent", - timeout=300, - ) - result = wrapper._build_apptainer_command(params, cmd_args) - assert "user_prompt.j2" in result - assert "system_prompt.j2" in result - - class TestSWEBenchWrapperGetOpenhandsTrajectory: def test_with_completions(self, monkeypatch) -> None: wrapper = _create_wrapper(monkeypatch) @@ -1702,8 +945,11 @@ def test_basic_setup_params(self, monkeypatch) -> None: assert isinstance(params, SWEBenchWrapperInstanceConfig) assert isinstance(processor, SweBenchDatasetProcessor) assert params.instance_id == "django__django-12345" - assert params.eval_command is not None - assert params.agent_command is not None + # _setup_params does not build launch/eval commands; the verifier path owns launch + + # eval, so these fields stay None. + assert params.eval_command is None + assert params.agent_command is None + assert params.eval_via_verifier is True assert params.metrics_fpath.exists() def test_setup_params_nv_internal(self, monkeypatch) -> None: @@ -2008,3 +1254,145 @@ def test_loads_from_lib_agent_dir(self) -> None: mod = _load_rebench_log_parsers(rebench_dir) assert "lib_test" in mod.NAME_TO_PARSER + + +class TestDecoupledCutover: + """run() with eval_via_verifier: sample masking behavior and the verifier POST contract.""" + + def test_should_mask_sample_all_combinations(self) -> None: + # resolved + clean agent finish -> NOT masked + assert swe_app._should_mask_sample(True, None, False, False) is False + # resolved but the agent hit max-turns / context window -> accidental reward, masked + assert swe_app._should_mask_sample(True, "max_iteration", False, False) is True + assert swe_app._should_mask_sample(True, "context_window", False, False) is True + # resolved + a different agent error (stuck_in_loop) -> NOT masked on that arm + assert swe_app._should_mask_sample(True, "stuck_in_loop", False, False) is False + # eval timed out -> masked regardless of resolved + assert swe_app._should_mask_sample(False, None, True, False) is True + # agent timed out (wall-clock) -> masked regardless + assert swe_app._should_mask_sample(False, None, False, True) is True + # unresolved, clean -> NOT masked + assert swe_app._should_mask_sample(False, None, False, False) is False + + @pytest.mark.asyncio + async def test_verify_patch_via_server_builds_request_and_parses_subset(self, monkeypatch) -> None: + wrapper = _create_wrapper(monkeypatch) + monkeypatch.setattr(swe_app, "raise_for_status", AsyncMock(return_value=None)) + monkeypatch.setattr( + swe_app, + "get_response_json", + AsyncMock(return_value={"resolved": True, "error_kind": None, "patch_exists": True, "reward": 1.0}), + ) + wrapper.server_client.post = AsyncMock(return_value=MagicMock()) + + with tempfile.TemporaryDirectory() as tmpdir: + instance_dict = { + "base_commit": "abc123", + "test_patch": "TP", + "FAIL_TO_PASS": ["test_x.py::test_a"], + "PASS_TO_PASS": ["test_x.py::test_b"], + } + params = _make_instance_config( + tmpdir, + eval_via_verifier=True, + verifier_server_name="swe_verifier", + container_formatter="docker://swebench/sweb.eval.x86_64.{instance_id}", + problem_info={ + "instance_id": "psf__requests-2317", + "base_commit": "abc123", + "dataset_name": "swe-bench-ext", + "split": "test", + "instance_dict": json.dumps(instance_dict), + }, + ) + # the worker persists the patch into metrics before run() POSTs to the verifier + params.metrics_fpath.write_text(json.dumps({"model_patch": "<>"})) + + subset = await wrapper._verify_patch_via_server(params) + + assert subset["resolved"] is True + call = wrapper.server_client.post.call_args + assert call.kwargs["server_name"] == "swe_verifier" + assert call.kwargs["url_path"] == "/verify" + req = call.kwargs["json"] + assert req["response"]["metadata"]["model_patch"] == "<>" + md = req["responses_create_params"]["metadata"] + assert md["instance_id"] == "psf__requests-2317" + # image resolved from the docker formatter (id munged). The instance ships no test_command, + # so we fall back to the conda+pytest default carrying the F2P+P2P node ids. + assert md["image"] == "swebench/sweb.eval.x86_64.psf_1776_requests-2317" + assert "conda activate testbed" in md["test_command"] + assert "test_x.py::test_a" in md["test_command"] and "test_x.py::test_b" in md["test_command"] + # no per-row framework on this instance -> forwarded as empty (verifier auto-detects) + assert md["test_framework"] == "" + assert md["benchmark"] == "swe-bench-ext" + + @pytest.mark.asyncio + async def test_verify_patch_via_server_forwards_instance_test_command_and_framework(self, monkeypatch) -> None: + """A swe-bench-ext row carrying its own per-framework test_command/test_framework is + forwarded verbatim (not clobbered by a hardcoded conda+pytest default), so multi-framework + (cargo/go/npm/...) rows grade correctly.""" + wrapper = _create_wrapper(monkeypatch) + monkeypatch.setattr(swe_app, "raise_for_status", AsyncMock(return_value=None)) + monkeypatch.setattr( + swe_app, + "get_response_json", + AsyncMock(return_value={"resolved": True, "error_kind": None, "patch_exists": True}), + ) + wrapper.server_client.post = AsyncMock(return_value=MagicMock()) + + with tempfile.TemporaryDirectory() as tmpdir: + instance_dict = { + "base_commit": "abc123", + "test_patch": "TP", + "test_command": "cargo test --offline", + "test_framework": "cargo", + "FAIL_TO_PASS": ["tests::a"], + "PASS_TO_PASS": ["tests::b"], + } + params = _make_instance_config( + tmpdir, + eval_via_verifier=True, + verifier_server_name="swe_verifier", + container_formatter="docker://swebench/sweb.eval.x86_64.{instance_id}", + problem_info={ + "instance_id": "rust__crate-1", + "base_commit": "abc123", + "dataset_name": "swe-bench-ext", + "split": "test", + "instance_dict": json.dumps(instance_dict), + }, + ) + params.metrics_fpath.write_text(json.dumps({"model_patch": "<>"})) + + await wrapper._verify_patch_via_server(params) + + md = wrapper.server_client.post.call_args.kwargs["json"]["responses_create_params"]["metadata"] + # forwarded verbatim; the conda+pytest default must not clobber the row's own command + assert md["test_command"] == "cargo test --offline" + assert md["test_framework"] == "cargo" + assert "conda activate testbed" not in md["test_command"] + + @pytest.mark.asyncio + async def test_verify_patch_via_server_infra_error_is_masked_not_raised(self, monkeypatch) -> None: + wrapper = _create_wrapper(monkeypatch) + wrapper.server_client.post = AsyncMock(side_effect=RuntimeError("connreset")) + with tempfile.TemporaryDirectory() as tmpdir: + params = _make_instance_config( + tmpdir, + eval_via_verifier=True, + verifier_server_name="swe_verifier", + problem_info={ + "instance_id": "psf__requests-2317", + "base_commit": "abc", + "dataset_name": "swe-bench-ext", + "split": "test", + "instance_dict": json.dumps({"FAIL_TO_PASS": [], "PASS_TO_PASS": []}), + }, + ) + params.metrics_fpath.write_text(json.dumps({"model_patch": "<>"})) + subset = await wrapper._verify_patch_via_server(params) + # never raises; returns a masked subset so the agent still emits a present row + assert subset["resolved"] is False + assert subset["error_kind"] == "sandbox" + assert subset["patch_exists"] is True diff --git a/responses_api_agents/swe_agents/tests/test_swe_env_adapter.py b/responses_api_agents/swe_agents/tests/test_swe_env_adapter.py new file mode 100644 index 0000000000..26e2e67b11 --- /dev/null +++ b/responses_api_agents/swe_agents/tests/test_swe_env_adapter.py @@ -0,0 +1,359 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SELF_DRIVING swe_env adapter for swe_agents: provision -> self-drive -> extract +patch -> score via the verifier, all through the decoupled swe_env infra.""" + +from __future__ import annotations + +import asyncio + +import responses_api_agents.swe_env.harnesses # noqa: F401 (register harnesses) +from nemo_gym.sandbox import SandboxExecResult, SandboxHandle, SandboxStatus, register_provider +from responses_api_agents.swe_agents.swe_env_adapter import ( + build_openhands_launch_command, + openhands_config_toml, + provision_and_collect, + provision_and_extract_patch, + run_self_driving, +) +from responses_api_agents.swe_env.harness import SweTask + + +_GOLD = "--- a/calc.py\n+++ b/calc.py\n@@ -1,2 +1,2 @@\n def add(a, b):\n- return a - b\n+ return a + b\n" + +# Records the env of every spec a fake sandbox was created with (egress-injection assertions). +_CREATED_ENVS: list[dict] = [] +# Records files staged into a fake sandbox (target paths) for stage_files assertions. +_UPLOADED_PATHS: list[str] = [] +# Records the output.jsonl `find` command(s) the adapter issued (newest-by-mtime assertions). +_FIND_COMMANDS: list[str] = [] + + +class _FakeProvider: + name = "fake-adapter" + + def __init__( + self, + *, + diff_output=_GOLD, + # Trailing-status pytest text is the format the test parser (parse_and_check_tests) + # recognizes. + test_output="test_calc.py::test_add PASSED\n", + output_jsonl_patch=None, + # error_type returned for the agent launch command (env.execute does not raise on + # timeout, it returns error_type instead). Only applied to the agent run, not git/cat/find. + agent_error_type=None, + **_, + ): + self._diff = diff_output + self._test_output = test_output + # When set, the agent emits its patch via an OpenHands-style output.jsonl (not git diff). + self._output_jsonl_patch = output_jsonl_patch + self._agent_error_type = agent_error_type + + async def create(self, spec): + _CREATED_ENVS.append(dict(spec.env or {})) + return SandboxHandle(sandbox_id="h", provider_name=self.name, raw={"workdir": spec.workdir}) + + async def exec(self, handle, command, *, cwd=None, env=None, timeout_s=None, user=None): + if self._output_jsonl_patch is not None: + if "find" in command and "output.jsonl" in command: + # The adapter selects the newest match via `find -printf "%T@ %p" + # | sort -n | tail -1`, which runs for real in the sandbox shell. We record the + # command (so tests can assert the newest-by-mtime shape) and emulate the + # pipeline's single-line " " result that the adapter must un-prefix. + _FIND_COMMANDS.append(command) + return SandboxExecResult(stdout="200.5 /root/eval/x/output.jsonl\n", stderr="", return_code=0) + if command.startswith("cat "): + import json + + row = {"instance_id": "adapter-1", "test_result": {"git_patch": self._output_jsonl_patch}} + return SandboxExecResult(stdout=json.dumps(row) + "\n", stderr="", return_code=0) + if "git diff" in command: + return SandboxExecResult(stdout=self._diff, stderr="", return_code=0) + if "pytest" in command: + return SandboxExecResult(stdout=self._test_output, stderr="", return_code=0) + # The agent launch command (anything else): surface the configured error_type so the + # worker can detect a non-raising agent timeout. + return SandboxExecResult( + stdout="", stderr="", return_code=124 if self._agent_error_type else 0, error_type=self._agent_error_type + ) + + async def upload_file(self, handle, source_path, target_path): + _UPLOADED_PATHS.append(target_path) + return None + + async def download_file(self, *a, **k): + return None + + async def status(self, handle): + return SandboxStatus.RUNNING + + async def close(self, handle): + return None + + async def aclose(self): + return None + + +register_provider("fake-adapter", _FakeProvider, override=True) + + +def _task() -> SweTask: + return SweTask( + instance_id="adapter-1", + image="img:tag", + base_commit="HEAD", + repo_workdir="/testbed", + test_command="python -m pytest -rA -q", + test_framework="pytest", + fail_to_pass=["test_calc.py::test_add"], + benchmark="swe-bench-ext", + ) + + +def test_self_driving_agent_patch_is_verified_resolved(): + out = asyncio.run( + run_self_driving( + _task(), + provider={"fake-adapter": {}}, + agent_launch_command="bash /openhands_setup/run_infer.sh", + model_server={"model": "qwen"}, + ) + ) + assert out["model_patch"].startswith("--- a/calc.py") + assert out["resolved"] is True + assert out["reward"] == 1.0 + assert out["patch_exists"] is True + assert out["mask_sample"] is False + + +def test_self_driving_no_patch_is_unresolved(): + out = asyncio.run( + run_self_driving( + _task(), + provider={"fake-adapter": {"diff_output": ""}}, + agent_launch_command="bash /openhands_setup/run_infer.sh", + ) + ) + assert out["patch_exists"] is False + assert out["resolved"] is False + assert out["reward"] == 0.0 + + +def test_self_driving_extra_env_is_injected_into_sandbox(): + """OpenHands-style egress: NEMO_GYM_* vars must reach the agent sandbox verbatim.""" + _CREATED_ENVS.clear() + oh_env = { + "NEMO_GYM_CONFIG_DICT": '{"head_server": {"host": "127.0.0.1", "port": 9099}}', + "NEMO_GYM_MODEL_SERVER_NAME": "vllm_model", + "NEMO_GYM_METRICS_FPATH": "/root/metrics.json", + } + asyncio.run( + run_self_driving( + _task(), + provider={"fake-adapter": {}}, + agent_launch_command="bash run_infer.sh", + extra_env=oh_env, + ) + ) + # The agent sandbox (first created) carries the injected egress env. + assert _CREATED_ENVS, "no sandbox created" + agent_env = _CREATED_ENVS[0] + for key, value in oh_env.items(): + assert agent_env.get(key) == value + + +def test_self_driving_patch_from_output_jsonl_is_verified(): + """OpenHands emits its patch via output.jsonl[test_result][git_patch], not git diff.""" + out = asyncio.run( + run_self_driving( + _task(), + provider={"fake-adapter": {"output_jsonl_patch": _GOLD}}, + agent_launch_command="bash run_infer.sh", + patch_output_glob="/root/eval", + ) + ) + assert out["model_patch"].startswith("--- a/calc.py") + assert out["patch_exists"] is True + assert out["resolved"] is True + assert out["reward"] == 1.0 + + +def test_self_driving_output_jsonl_missing_yields_empty_patch(): + out = asyncio.run( + run_self_driving( + _task(), + # output_jsonl_patch set but find returns a path; cat returns empty row patch + provider={"fake-adapter": {"output_jsonl_patch": ""}}, + agent_launch_command="bash run_infer.sh", + patch_output_glob="/root/eval", + ) + ) + assert out["patch_exists"] is False + assert out["resolved"] is False + assert out["reward"] == 0.0 + + +def test_output_jsonl_selected_by_newest_mtime_not_first_traversal(): + """The adapter selects the prediction file by newest mtime, not first in traversal order + (which could pick a stale re-run artifact and diverge resolved/reward). Asserts the adapter + issues a newest-by-mtime selection and correctly strips the `%T@ ` mtime prefix back off.""" + _FIND_COMMANDS.clear() + out = asyncio.run( + run_self_driving( + _task(), + provider={"fake-adapter": {"output_jsonl_patch": _GOLD}}, + agent_launch_command="bash run_infer.sh", + patch_output_glob="/root/eval", + ) + ) + assert _FIND_COMMANDS, "adapter never issued the output.jsonl find" + cmd = _FIND_COMMANDS[-1] + # newest-by-mtime: emit " ", sort ascending, take the last (largest mtime). + assert "-printf" in cmd and "%T@" in cmd + assert "sort -n" in cmd and "tail -1" in cmd + # a first-in-traversal selection must not be used + assert "head -1" not in cmd + # and the mtime prefix was stripped so the right path was catted -> patch recovered + resolved + assert out["model_patch"].startswith("--- a/calc.py") + assert out["resolved"] is True + + +def test_output_jsonl_path_with_spaces_survives_mtime_prefix_strip(): + """A path with spaces must survive un-prefixing: split only on the FIRST space (after the + float mtime), never on path-internal spaces.""" + from responses_api_agents.swe_agents import swe_env_adapter as A + + class _Env: + async def execute(self, command, **_): + if "find" in command: + return {"stdout": "200.5 /root/eval dir/x/output.jsonl\n"} + assert command.startswith("cat "), command + # the catted path must be the space-containing path, fully intact + assert "/root/eval dir/x/output.jsonl" in command + import json + + return {"stdout": json.dumps({"test_result": {"git_patch": _GOLD}}) + "\n"} + + row = asyncio.run(A._read_output_jsonl_row(_Env(), "/root/eval dir")) + assert (row.get("test_result") or {}).get("git_patch") == _GOLD + + +def test_provision_and_extract_patch_stages_files_and_returns_patch_without_verifying(): + """Agent-side primitive the worker uses: stage files, self-drive, return patch (NO grading).""" + _UPLOADED_PATHS.clear() + patch = asyncio.run( + provision_and_extract_patch( + _task(), + provider={"fake-adapter": {"output_jsonl_patch": _GOLD}}, + agent_launch_command="bash run_infer.sh", + extra_env={"NEMO_GYM_MODEL_SERVER_NAME": "vllm_model"}, + stage_files={"/root/config.toml": "[llm.model]\n", "/root/dataset/data.jsonl": "{}\n"}, + patch_output_glob="/root/eval", + ) + ) + # Returns the patch (a plain str), runs no verification. + assert isinstance(patch, str) and patch.startswith("--- a/calc.py") + # Both staged files were written into the sandbox before launch. + assert "/root/config.toml" in _UPLOADED_PATHS + assert "/root/dataset/data.jsonl" in _UPLOADED_PATHS + + +def test_provision_and_collect_surfaces_agent_timeout_error_type(): + """env.execute does not raise on agent timeout (it returns error_type), so the agent run + 'succeeds' from the adapter's view. provision_and_collect surfaces that error_type so the + worker can set agent_timed_out; without it the timed-out sample would be wrongly unmasked.""" + out = asyncio.run( + provision_and_collect( + _task(), + provider={"fake-adapter": {"output_jsonl_patch": _GOLD, "agent_error_type": "timeout"}}, + agent_launch_command="bash run_infer.sh", + patch_output_glob="/root/eval", + ) + ) + assert out["error_type"] == "timeout" + # the patch is still collected (the agent had produced one before the timeout) + assert out["patch"].startswith("--- a/calc.py") + + +def test_provision_and_collect_clean_run_has_no_error_type(): + """A clean agent run surfaces error_type=None (worker leaves the sample unmasked) via both + egress styles (output.jsonl and git-diff).""" + out_jsonl = asyncio.run( + provision_and_collect( + _task(), + provider={"fake-adapter": {"output_jsonl_patch": _GOLD}}, + agent_launch_command="bash run_infer.sh", + patch_output_glob="/root/eval", + ) + ) + assert out_jsonl["error_type"] is None + out_diff = asyncio.run( + provision_and_collect( + _task(), + provider={"fake-adapter": {}}, + agent_launch_command="bash run_infer.sh", + ) + ) + assert out_diff["error_type"] is None + assert out_diff["patch"].startswith("--- a/calc.py") + + +def test_openhands_config_toml_uses_nonnative_fc(): + toml = openhands_config_toml("Qwen/Qwen2.5-Coder-3B-Instruct", temperature=0.0, top_p=1.0) + assert "[llm.model]" in toml + assert 'model = "Qwen/Qwen2.5-Coder-3B-Instruct"' in toml + # non-native FC is the robust choice for small open models (validated) + assert "native_tool_calling = false" in toml + assert "log_completions_folder" in toml + + +def test_openhands_config_toml_has_no_output_cap_by_default(): + """The default config sets no output-token cap, so the model/litellm default applies. The + default config must not emit a max_output_tokens line, which could otherwise truncate output.""" + toml = openhands_config_toml("Qwen/Qwen2.5-Coder-3B-Instruct") + assert "max_output_tokens" not in toml + + +def test_openhands_config_toml_emits_cap_only_when_requested(): + """Opt-in: a caller that needs to bound an unknown model (whose default max_tokens would be the + full context window -> vLLM 400) can still pass max_output_tokens explicitly.""" + toml = openhands_config_toml("some/unknown-model", max_output_tokens=8192) + assert "max_output_tokens = 8192" in toml + + +def test_build_openhands_launch_command_has_runtime_local_egress_and_dataset(): + cmd = build_openhands_launch_command( + setup_dir="/gym/responses_api_agents/swe_agents/swe_openhands_setup", + instance_id="psf__requests-2317", + dataset_name="SWE-Gym", + split="test", + ng_config_dict_quoted="'<>'", + model_server_name="vllm_model", + agent_cls="CodeActAgent", + max_iter=30, + ) + # RUNTIME=local self-drive + the OpenHands runner + assert "export RUNTIME=local" in cmd + assert "run_infer.sh" in cmd + # egress routes OpenHands' NemoGymClient back to the real model server + assert "export NEMO_GYM_CONFIG_DICT='<>'" in cmd + assert "export NEMO_GYM_MODEL_SERVER_NAME=vllm_model" in cmd + assert "NEMO_GYM_METRICS_FPATH" in cmd + # dataset name selects OpenHands' workspace; instance + output dir wired + assert "SWE-Gym test /root/eval_results psf__requests-2317" in cmd + # git dubious-ownership guard for the host-owned bind mount under a root container + assert "safe.directory '*'" in cmd