diff --git a/responses_api_agents/mini_swe_agent_2/__init__.py b/responses_api_agents/mini_swe_agent_2/__init__.py index e69de29bb2..ab7fcbfde7 100644 --- a/responses_api_agents/mini_swe_agent_2/__init__.py +++ b/responses_api_agents/mini_swe_agent_2/__init__.py @@ -0,0 +1 @@ +"""Agent package that runs mini-swe-agent on SWE-bench tasks inside a sandbox.""" diff --git a/responses_api_agents/mini_swe_agent_2/app.py b/responses_api_agents/mini_swe_agent_2/app.py index febe8fe696..6c7a68b1f3 100644 --- a/responses_api_agents/mini_swe_agent_2/app.py +++ b/responses_api_agents/mini_swe_agent_2/app.py @@ -12,6 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Agent server that runs mini-swe-agent against SWE-bench tasks inside a sandbox. + +Each task is executed in an isolated sandbox via a Ray remote worker: the agent produces a code +patch, which is then graded either in-process with the swebench harness or by POSTing it to a +shared swe_env verifier. The server converts mini-swe-agent trajectories into Gym Responses API +rows and aggregates pass/resolution metrics across rollouts. +""" + import asyncio import hashlib import json @@ -29,7 +37,7 @@ import yaml from fastapi import Body, FastAPI from minisweagent.config import builtin_config_dir, get_config_path -from pydantic import ConfigDict +from pydantic import ConfigDict, Field from nemo_gym.base_resources_server import ( BaseRunRequest, @@ -50,6 +58,8 @@ from nemo_gym.server_utils import ( ServerClient, get_first_server_config_dict, + get_response_json, + raise_for_status, ) @@ -72,6 +82,23 @@ class MiniSWEAgentConfig(BaseResponsesAPIAgentConfig): tool_choice: Optional[str | dict[str, Any]] = None sandbox_resource_profiles: Optional[list[dict[str, str]]] = None + # Opt-in: score the patch via the shared swe_env verifier instead of grading in-process. + eval_via_verifier: bool = Field( + default=False, + description=( + "If True, score the agent's patch by POSTing it to the shared swe_env verifier " + "(verifier_server_name) instead of grading it in-process. Default False grades " + "in-process; setting it True reuses the same swe_env verifier other agents use." + ), + ) + verifier_server_name: Optional[str] = Field( + default=None, + description=( + "Name of the resources_servers/swe_env verifier to POST /verify to when " + "eval_via_verifier=True. Required when eval_via_verifier is True." + ), + ) + class MiniSWEAgentRunRequest(BaseRunRequest): model_config = ConfigDict(extra="allow") @@ -92,10 +119,31 @@ class MiniSWEAgentVerifyResponse(BaseVerifyResponse): }, ) def runner_ray_remote(runner: Callable, params: dict[str, Any]) -> Any: + """Invoke a runner callable inside a Ray remote task. + + Args: + runner: The callable to execute on the worker. + params: Keyword arguments to pass to ``runner``. + + Returns: + The value returned by ``runner``. + """ return runner(**params) def _json_dict_from_metadata(value: Any, *, field_name: str) -> dict[str, Any]: + """Coerce a metadata value into a dict, parsing JSON strings as needed. + + Args: + value: The metadata value to coerce; may be None, a dict, or a JSON-encoded string. + field_name: The metadata field name, used only for the error message. + + Returns: + The value as a dict; an empty dict when ``value`` is None. + + Raises: + ValueError: If the value is not None, a dict, or a JSON string decoding to an object. + """ if value is None: return {} if isinstance(value, dict): @@ -112,7 +160,17 @@ def _responses_create_params_to_model_kwargs( *, default_tool_choice: Any = None, ) -> dict[str, Any]: - """Convert Gym Responses API rollout params into mini-swe-agent chat-completions kwargs.""" + """Convert Gym Responses API rollout params into mini-swe-agent chat-completions kwargs. + + Args: + params: The Responses API create params (temperature, top_p, max_output_tokens, metadata, + tool_choice, etc.). + default_tool_choice: A tool choice that overrides ``params["tool_choice"]`` when provided. + + Returns: + A dict of chat-completions kwargs (e.g. temperature, max_tokens, extra_body, tool_choice) + containing only the fields that were set. + """ model_kwargs: dict[str, Any] = {} for key in ("temperature", "top_p", "top_logprobs", "parallel_tool_calls"): value = params.get(key) @@ -144,6 +202,14 @@ def _responses_create_params_to_model_kwargs( def _opensandbox_connection(provider: dict[str, Any] | None) -> dict[str, Any] | None: + """Extract the opensandbox connection dict from a sandbox provider config. + + Args: + provider: The sandbox provider config mapping, or None. + + Returns: + The nested opensandbox connection dict, or None if it is absent or malformed. + """ if provider is None: return None provider_config = provider.get(OPENSANDBOX_PROVIDER_NAME) @@ -156,6 +222,14 @@ def _opensandbox_connection(provider: dict[str, Any] | None) -> dict[str, Any] | def _sandbox_provider_for_config_dump(provider: dict[str, Any]) -> dict[str, Any]: + """Return a copy of the provider config safe to write to disk, with the API key removed. + + Args: + provider: The sandbox provider config mapping. + + Returns: + A deep copy of ``provider`` with any opensandbox connection ``api_key`` stripped. + """ provider_for_disk = deepcopy(provider) connection = _opensandbox_connection(provider_for_disk) if connection is not None: @@ -164,6 +238,15 @@ def _sandbox_provider_for_config_dump(provider: dict[str, Any]) -> dict[str, Any def _sandbox_runtime_env(provider: dict[str, Any] | None) -> dict[str, Any]: + """Build the Ray runtime env for the sandbox worker, injecting the API key when available. + + Args: + provider: The sandbox provider config mapping, or None. + + Returns: + A runtime env dict with ``py_executable`` set, and ``env_vars`` carrying the opensandbox + API key when the provider supplies one. + """ runtime_env: dict[str, Any] = {"py_executable": sys.executable} connection = _opensandbox_connection(provider) if connection is None: @@ -175,6 +258,14 @@ def _sandbox_runtime_env(provider: dict[str, Any] | None) -> dict[str, Any]: def _restore_sandbox_provider_secrets(config: dict[str, Any]) -> None: + """Repopulate the opensandbox API key in a loaded config from the environment. + + Mutates ``config`` in place so the provider connection has an ``api_key`` again after the key + was stripped for on-disk storage, reading it from the environment variable. + + Args: + config: The loaded agent config dict to mutate. + """ provider = config.get("environment", {}).get("provider") connection = _opensandbox_connection(provider if isinstance(provider, dict) else None) if connection is None or connection.get("api_key"): @@ -185,6 +276,11 @@ def _restore_sandbox_provider_secrets(config: dict[str, Any]) -> None: def _bash_tool_choice() -> dict[str, Any]: + """Build the tool-choice payload that forces the model to call the ``bash`` function. + + Returns: + A tool-choice dict selecting the ``bash`` function. + """ return {"type": "function", "function": {"name": "bash"}} @@ -194,6 +290,17 @@ def _sandbox_spec_for_instance( resource_profiles: list[dict[str, Any]] | None, instance_id: str, ) -> dict[str, Any]: + """Build the per-instance sandbox spec, applying a deterministically chosen resource profile. + + Args: + spec: The base sandbox spec mapping, or None. + resource_profiles: Candidate resource profile mappings to choose from, or None to skip. + instance_id: The instance identifier, hashed to pick a profile deterministically. + + Returns: + A copy of the spec with the selected resource profile merged into its ``resources``, or the + plain spec copy when no profiles are provided. + """ instance_spec = dict(spec or {}) if not resource_profiles: return instance_spec @@ -207,6 +314,11 @@ def _sandbox_spec_for_instance( def _swebench_config_path() -> Path: + """Locate the bundled mini-swe-agent SWE-bench config file. + + Returns: + The path to the first existing candidate config, falling back to the canonical location. + """ for candidate in ( builtin_config_dir / "extra" / "swebench.yaml", builtin_config_dir / "benchmarks" / "swebench.yaml", @@ -217,6 +329,18 @@ def _swebench_config_path() -> Path: def _swebench_image_name(instance: dict[str, Any], subset: str) -> str: + """Resolve the Docker image name for a SWE-bench instance. + + Uses the instance's explicit ``image_name`` when present; otherwise derives the conventional + image reference from the instance id and subset. + + Args: + instance: The task instance dict; may carry ``image_name`` and must carry ``instance_id``. + subset: The dataset subset (e.g. "verified"), which selects the image naming scheme. + + Returns: + The fully qualified Docker image name, lowercased. + """ image_name = instance.get("image_name") if image_name: return str(image_name) @@ -231,6 +355,14 @@ def _swebench_image_name(instance: dict[str, Any], subset: str) -> str: def _message_content_to_text(content: Any) -> str: + """Flatten a message ``content`` field into a single text string. + + Args: + content: A message content value: a string, a list of content parts, None, or other. + + Returns: + The concatenated text; an empty string when ``content`` is None. + """ if isinstance(content, str): return content if isinstance(content, list): @@ -245,6 +377,14 @@ def _message_content_to_text(content: Any) -> str: def _strip_extra(item: Any) -> dict[str, Any]: + """Normalize a trajectory item to a dict and drop its ``extra`` key. + + Args: + item: A pydantic model, a dict, or any other value to normalize. + + Returns: + A dict without the ``extra`` key; non-dict, non-model values are wrapped as a user message. + """ if hasattr(item, "model_dump"): item = item.model_dump() if not isinstance(item, dict): @@ -255,6 +395,17 @@ def _strip_extra(item: Any) -> dict[str, Any]: def _split_trajectory_for_responses( messages: list[dict[str, Any]], ) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: + """Split a mini-swe-agent message trajectory into Responses API input, output, and responses. + + The leading system/user messages become input messages; subsequent assistant turns, tool calls, + tool outputs, and raw response objects become output items. + + Args: + messages: The ordered list of trajectory messages from the agent run. + + Returns: + A tuple ``(input_messages, output_items, raw_responses)`` in Responses API form. + """ input_messages: list[dict[str, Any]] = [] output_items: list[dict[str, Any]] = [] raw_responses: list[dict[str, Any]] = [] @@ -311,6 +462,11 @@ def _split_trajectory_for_responses( def _default_response_object() -> dict[str, Any]: + """Build a fully populated default Responses API response object. + + Returns: + A dict with a fresh id, current timestamp, and default values for every Responses API field. + """ return { "id": f"resp_{str(uuid4())}", "created_at": int(time.time()), @@ -352,6 +508,18 @@ def _default_response_object() -> dict[str, Any]: def _is_resolved(instance_id: str, eval_report: dict[str, Any]) -> bool: + """Determine whether an instance was resolved according to its eval report. + + An instance counts as resolved only when the harness marked it resolved and reported at least + one test outcome. + + Args: + instance_id: The instance identifier to look up in the report. + eval_report: The eval report dict produced by the swebench harness. + + Returns: + True if the instance is resolved with reported test results; False otherwise or on any error. + """ try: if not eval_report: return False @@ -376,16 +544,43 @@ def _is_resolved(instance_id: str, eval_report: dict[str, Any]) -> bool: def _metadata_dict(verify_response: dict[str, Any]) -> dict[str, Any]: + """Extract the ``metadata`` mapping from a verify/rollout response. + + Args: + verify_response: The verify or rollout response dict. + + Returns: + The metadata dict, or an empty dict when absent or not a dict. + """ metadata = verify_response.get("metadata") or {} return metadata if isinstance(metadata, dict) else {} def _eval_report_map(verify_response: dict[str, Any]) -> dict[str, Any]: + """Extract the per-instance eval report map from a verify/rollout response. + + Args: + verify_response: The verify or rollout response dict. + + Returns: + The ``eval_report`` mapping keyed by instance id, or an empty dict when absent. + """ report = _metadata_dict(verify_response).get("eval_report") or {} return report if isinstance(report, dict) else {} def _eval_instance_report(verify_response: dict[str, Any]) -> dict[str, Any]: + """Find the single instance's eval report within a verify/rollout response. + + Looks up the report by instance id, falling back to the first report entry that carries a + ``resolved`` flag. + + Args: + verify_response: The verify or rollout response dict. + + Returns: + The matching per-instance report dict, or an empty dict when none is found. + """ report_map = _eval_report_map(verify_response) instance_id = verify_response.get("instance_id") or _metadata_dict(verify_response).get("instance_id") if instance_id is not None: @@ -400,6 +595,15 @@ def _eval_instance_report(verify_response: dict[str, Any]) -> dict[str, Any]: def _test_status_counts(verify_response: dict[str, Any]) -> dict[str, int]: + """Count per-suite test successes and failures from a verify/rollout response. + + Args: + verify_response: The verify or rollout response dict. + + Returns: + A dict mapping ``_success`` and ``_failure`` keys to their counts; empty when + no test status is present. + """ report = _eval_instance_report(verify_response) tests_status = report.get("tests_status") if isinstance(report, dict) else None if not isinstance(tests_status, dict): @@ -424,6 +628,22 @@ def _run_eval_v2( run_id: str, is_golden: bool, ) -> dict[str, Any]: + """Grade a patch in-process with the swebench harness inside the task's sandbox. + + Writes the patch, runs the harness eval script in the sandbox, captures test output, and builds + the eval report. When grading a golden patch, applies the patch to the repo first. + + Args: + instance: The SWE-bench instance dict. + env: The sandbox environment used to execute the eval script. + model_patch: The patch diff to grade. + instance_dir: The directory where logs, patch, output, and report files are written. + run_id: A unique identifier for this eval run, used in output filenames. + is_golden: Whether ``model_patch`` is the golden patch and should be applied before eval. + + Returns: + A dict with ``instance_id``, ``model_patch``, and the harness ``eval_report``. + """ from swebench.harness.constants import SWEbenchInstance from swebench.harness.docker_build import setup_logger from swebench.harness.grading import get_eval_report @@ -476,6 +696,20 @@ def _run_eval_v2( def _run_mini_swe_v2(**params: Any) -> dict[str, Any]: + """Run a single mini-swe-agent task in a sandbox and return its trajectory and eval report. + + Builds the agent config, starts the sandbox environment, runs the agent (or applies the golden + patch), optionally grades the patch in-process, and converts the trajectory to Responses API form. + + Args: + **params: Run parameters including ``instance_dict``, ``instance_id``, ``output``, ``config``, + ``model``, ``api_key``, ``base_url``, ``subset``, ``step_timeout``, ``eval_timeout``, + ``step_limit``, ``run_golden``, and ``eval_via_verifier``. + + Returns: + A dict keyed by instance id whose value holds ``input_messages``, ``response_output``, + ``responses``, ``eval_report``, and ``exit_status``. + """ from minisweagent.agents.default import DefaultAgent from minisweagent.environments import get_environment from minisweagent.models import get_model @@ -549,16 +783,22 @@ def _run_mini_swe_v2(**params: Any) -> dict[str, Any]: {"instance_id": instance_id}, ) - print(f"[EVAL]{instance_id} Running eval", flush=True) - eval_report = _run_eval_v2( - instance=instance, - env=env, - model_patch=model_patch, - instance_dir=instance_dir, - run_id=run_id, - is_golden=params["run_golden"], - ) - print(f"[EVAL]{instance_id} Eval completed", flush=True) + if params.get("eval_via_verifier"): + # Skip the in-worker swebench harness; the patch is graded out-of-band by the shared + # swe_env verifier. Carry the patch forward so the caller can POST it. + print(f"[EVAL]{instance_id} Skipping in-worker eval (eval_via_verifier)", flush=True) + eval_report = {"instance_id": instance_id, "model_patch": model_patch} + else: + print(f"[EVAL]{instance_id} Running eval", flush=True) + eval_report = _run_eval_v2( + instance=instance, + env=env, + model_patch=model_patch, + instance_dir=instance_dir, + run_id=run_id, + is_golden=params["run_golden"], + ) + print(f"[EVAL]{instance_id} Eval completed", flush=True) input_messages, response_output, responses = _split_trajectory_for_responses(data.get("messages", [])) @@ -577,18 +817,42 @@ def _run_mini_swe_v2(**params: Any) -> dict[str, Any]: def run_mini_swe_with_sandbox(**params: Any) -> Any: + """Run a mini-swe-agent task in a sandbox. + + Args: + **params: The run parameters forwarded to the sandbox runner. + + Returns: + The per-instance result dict produced by the sandbox runner. + """ return _run_mini_swe_v2(**params) class MiniSWEAgent(SimpleResponsesAPIAgent): + """Responses API agent that runs mini-swe-agent on SWE-bench tasks and scores the patches. + + Bounds concurrency with a semaphore, dispatches each task to a Ray sandbox worker, grades the + resulting patch (in-process or via a shared verifier), and exposes rollout-level metrics. + """ + config: MiniSWEAgentConfig sem: Semaphore = None model_config = ConfigDict(arbitrary_types_allowed=True) def model_post_init(self, __context: Any) -> None: + """Initialize the concurrency semaphore from the configured concurrency limit. + + Args: + __context: The pydantic post-init context (unused). + """ self.sem = Semaphore(self.config.concurrency) def setup_webserver(self) -> FastAPI: + """Build the FastAPI app and register the agent's HTTP routes. + + Returns: + The configured FastAPI application. + """ app = FastAPI() self.setup_session_middleware(app) app.post("/v1/responses")(self.responses) @@ -597,6 +861,15 @@ def setup_webserver(self) -> FastAPI: return app def compute_metrics(self, tasks: list[list[dict[str, Any]]]) -> dict[str, Any]: + """Aggregate pass/resolution, eval-error, and test-status metrics across all rollouts. + + Args: + tasks: A list of tasks, each a list of rollout dicts for that task. + + Returns: + A dict of aggregate metrics, including pass@k, resolved counts and rates, eval error and + report rates, patch-applied rates, per-task metrics, and per-suite test status totals. + """ metrics, _, _, max_k = compute_pass_majority_metrics(tasks) metrics.pop("per_sample_aggregate", None) @@ -638,6 +911,15 @@ def compute_metrics(self, tasks: list[list[dict[str, Any]]]) -> dict[str, Any]: return metrics def _compute_per_task_eval_metrics(self, tasks: list[list[dict[str, Any]]]) -> list[dict[str, Any]]: + """Compute per-task eval metrics from each task's rollouts. + + Args: + tasks: A list of tasks, each a list of rollout dicts for that task. + + Returns: + A list of per-task metric dicts (task index, instance id, rollout/resolved/error counts, + and per-suite test status totals), skipping tasks with no rollouts. + """ per_task_metrics: list[dict[str, Any]] = [] for fallback_idx, rollouts in enumerate(tasks): if not rollouts: @@ -676,6 +958,15 @@ def _compute_per_task_eval_metrics(self, tasks: list[list[dict[str, Any]]]) -> l return per_task_metrics def get_key_metrics(self, agent_metrics: dict[str, Any]) -> dict[str, Any]: + """Select the headline metrics to surface from the full aggregate metrics. + + Args: + agent_metrics: The full metrics dict produced by ``compute_metrics``. + + Returns: + A dict containing the highest-k pass metrics plus selected reward, resolution, and error + rate metrics that are present in ``agent_metrics``. + """ key_metrics: dict[str, Any] = {} key_metrics.update(highest_k_metrics(agent_metrics, "pass@{k}", score_names=["accuracy"])) key_metrics.update(highest_k_metrics(agent_metrics, "pass@1[avg-of-{k}]", score_names=["accuracy"])) @@ -691,10 +982,133 @@ def get_key_metrics(self, agent_metrics: dict[str, Any]) -> dict[str, Any]: key_metrics[key] = agent_metrics[key] return key_metrics + async def _verify_patch_via_server( + self, + *, + instance: dict[str, Any], + patch: str, + instance_id: str, + subset: str, + split: str, + responses_create_params: NeMoGymResponseCreateParamsNonStreaming, + ) -> dict[str, Any]: + """POST the agent's patch to the shared swe_env verifier and return its eval subset. + + Builds a verify request carrying the per-task metadata the verifier reads (instance_id, image, + base_commit, repo_workdir, test_command, test_framework, test_patch, fail_to_pass, pass_to_pass, + benchmark, split) plus the patch in ``response.metadata.model_patch``, and POSTs it to the + configured verifier server via the server client. On any transport failure it returns a masked + subset (``resolved=False``, ``error_kind='sandbox'``) rather than raising, so the agent always + emits a present (masked) row instead of dropping the rollout. + + Args: + instance: The task instance dict holding fields such as FAIL_TO_PASS, PASS_TO_PASS, + base_commit, test_command, test_framework, and test_patch. + patch: The model-produced diff to grade. + instance_id: The SWE-bench instance identifier. + subset: The dataset subset (e.g. "verified"), used to resolve the Docker image name. + split: The dataset split passed through to the verifier. + responses_create_params: The Responses API params used to build the verify request. + + Returns: + The verifier's eval subset as a dict; on failure, a masked dict with ``resolved=False``, + ``error_kind='sandbox'``, and ``patch_exists`` reflecting whether a patch was present. + """ + + def _as_list(value: Any) -> list[str]: + if isinstance(value, str): + try: + return json.loads(value) + except json.JSONDecodeError: + return [value] + return value or [] + + f2p = _as_list(instance.get("FAIL_TO_PASS")) + p2p = _as_list(instance.get("PASS_TO_PASS")) + # Forward the instance's own per-framework eval command and framework when present. Fall back + # to the conda+pytest default only when the row ships no test_command. This supports both + # SWE-bench-Verified (no per-row command) and multi-framework rows (cargo/go/npm/...) that + # carry their own command and framework. + test_framework = instance.get("test_framework", "") or "" + test_command = instance.get("test_command", "") or "" + if not test_command: + nodeids = " ".join("'" + n + "'" for n in f2p + p2p) + test_command = ( + "source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && " + f"python -m pytest -rA {nodeids}" + ) + task_metadata = { + "instance_id": instance_id, + "image": _swebench_image_name(instance, subset), + "base_commit": instance.get("base_commit", "") or "", + "repo_workdir": "/testbed", + "test_command": test_command, + "test_framework": test_framework, + "test_patch": instance.get("test_patch", "") or "", + "fail_to_pass": f2p, + "pass_to_pass": p2p, + # "swe-bench-ext" is the flat host-graded harness the conda/pytest test_command above + # targets; it is the registered swe_env harness key, not the dataset subset. + "benchmark": "swe-bench-ext", + "split": split, + } + verify_request = { + "responses_create_params": responses_create_params.model_dump(exclude_none=True) + | {"metadata": task_metadata}, + "response": { + "id": f"mini-swe-{instance_id}", + "created_at": int(time.time()), + "model": responses_create_params.model, + "object": "response", + "output": [], + "metadata": {"model_patch": patch}, + }, + } + + async def _do_verify() -> dict[str, Any]: + verify_response = await self.server_client.post( + server_name=self.config.verifier_server_name, + url_path="/verify", + json=verify_request, + ) + await raise_for_status(verify_response) + return await get_response_json(verify_response) + + # Bound the whole call (including ServerClient's unbounded disconnect-retry loop) so a + # hung or retried verify cannot pin a rollout slot forever. + verify_timeout_s = float(getattr(self.config, "eval_timeout", None) or 900) + 900 + try: + return await asyncio.wait_for(_do_verify(), timeout=verify_timeout_s) + except Exception as e: # noqa: BLE001 (incl. asyncio.TimeoutError -> masked, never pins a slot) + print(f"Verifier POST failed for {instance_id}: {e}", flush=True) + return {"resolved": False, "error_kind": "sandbox", "patch_exists": bool(patch)} + async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()) -> NeMoGymResponse: + """Responses endpoint placeholder; this agent drives generation through its own run loop. + + Args: + body: The Responses API create params. + + Raises: + NotImplementedError: Always, since this agent does not serve standalone responses. + """ raise NotImplementedError async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: + """Run a single SWE-bench task end to end and return a scored verify response. + + Resolves the model server, builds the mini-swe-agent config, dispatches the task to a Ray + sandbox worker, scores the resulting patch (via the shared verifier or the in-worker harness), + assembles the Responses API response, persists the result, and returns it. + + Args: + body: The run request carrying the instance, dataset subset/split, and Responses API + create params. + + Returns: + A verify response with the reward, the assembled response object, the instance id, and the + eval report metadata. + """ async with self.sem: model_server_name = self.config.model_server.name global_config_dict = ServerClient.load_from_global_config().global_config_dict @@ -789,6 +1203,7 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: step_timeout=step_timeout, eval_timeout=eval_timeout, step_limit=step_limit, + eval_via_verifier=self.config.eval_via_verifier, ) runner = runner_ray_remote runtime_env = _sandbox_runtime_env(self.config.sandbox_provider) @@ -800,7 +1215,25 @@ async def run(self, body: MiniSWEAgentRunRequest) -> MiniSWEAgentVerifyResponse: input_messages = result["input_messages"] response_output = result["response_output"] responses = result["responses"] - reward = 1.0 if _is_resolved(instance_id, result["eval_report"]) else 0.0 + + if self.config.eval_via_verifier: + # Score the patch by POSTing to the shared swe_env verifier instead of grading via + # the in-worker swebench harness. The Ray worker still produced the trajectory and + # patch; the verifier is the authoritative grader here. + in_worker_report = result.get("eval_report") or {} + patch = in_worker_report.get("model_patch", "") or "" + eval_subset = await self._verify_patch_via_server( + instance=body.model_dump(), + patch=patch, + instance_id=instance_id, + subset=subset, + split=split, + responses_create_params=body.responses_create_params, + ) + reward = 1.0 if eval_subset.get("resolved") else 0.0 + result["eval_report"] = eval_subset + else: + reward = 1.0 if _is_resolved(instance_id, result["eval_report"]) else 0.0 except Exception as e: error_info = {"error": str(e), "traceback": traceback.format_exc()} diff --git a/responses_api_agents/mini_swe_agent_2/sandbox_environment.py b/responses_api_agents/mini_swe_agent_2/sandbox_environment.py index 704122b9ce..db512b5b5d 100644 --- a/responses_api_agents/mini_swe_agent_2/sandbox_environment.py +++ b/responses_api_agents/mini_swe_agent_2/sandbox_environment.py @@ -25,9 +25,17 @@ except ModuleNotFoundError: class Submitted(Exception): - """Compatibility shim for local mini-swe-agent versions before v2.""" + """Signals that the agent has submitted its final output, carrying the submission messages. + + Used when the installed mini-swe-agent does not provide its own ``Submitted`` exception. + """ def __init__(self, *messages: dict[str, Any]) -> None: + """Store the submission messages on the exception. + + Args: + *messages: The message dicts describing the submission. + """ self.messages = messages super().__init__() @@ -69,6 +77,18 @@ def __init__( config_class: type = MiniSWESandboxEnvironmentConfig, **kwargs: Any, ) -> None: + """Build the environment config and start the backing sandbox. + + Resolves the image (with rewrites), assembles the environment variables and resources from + the spec and config, and starts a sandbox for the task. + + Args: + config_class: The dataclass used to construct the environment config from ``kwargs``. + **kwargs: Fields forwarded to ``config_class`` (image, cwd, env, provider, spec, etc.). + + Raises: + ValueError: If no sandbox provider is configured. + """ self.config = config_class(**kwargs) if not self.config.provider: raise ValueError("MiniSWESandboxEnvironment requires provider") @@ -108,9 +128,22 @@ def __init__( ) def get_template_vars(self, **kwargs: Any) -> dict[str, Any]: + """Return the variables available for prompt/command templating. + + Args: + **kwargs: Extra variables to merge over the config fields. + + Returns: + A dict combining the config fields with the provided overrides. + """ return {**self.config.__dict__, **kwargs} def serialize(self) -> dict[str, Any]: + """Serialize the environment configuration for trajectory records. + + Returns: + A nested dict describing the environment config and its fully qualified type name. + """ return { "info": { "config": { @@ -121,6 +154,15 @@ def serialize(self) -> dict[str, Any]: } def _command(self, command: str) -> str: + """Wrap a command to activate the configured conda env when enabled. + + Args: + command: The shell command to run. + + Returns: + The command prefixed with conda activation, or the unchanged command when conda + activation is disabled or no env is configured. + """ if not self.config.activate_conda or not self.config.conda_env: return command quoted_env = shlex.quote(self.config.conda_env) @@ -133,6 +175,21 @@ def execute( is_eval: bool = False, timeout: int | None = None, ) -> dict[str, Any]: + """Execute a command in the sandbox and return its combined output and return code. + + Args: + action: The command to run, either a string or a dict with a ``command`` key. + cwd: The working directory; defaults to the configured cwd when empty. + is_eval: Whether this is an eval command, which selects the eval timeout. + timeout: An explicit timeout in seconds; overrides the step/eval default when set. + + Returns: + A dict with ``output`` (merged stdout and stderr), ``returncode``, and ``exception_info``. + + Raises: + RuntimeError: If the sandbox is not available. + Submitted: If the command output signals a final submission. + """ command = action.get("command", "") if isinstance(action, dict) else action timeout_s = timeout or (self.config.eval_timeout if is_eval else self.config.step_timeout) exec_cwd = cwd or self.config.cwd @@ -155,7 +212,14 @@ def execute( return response def _check_finished(self, output: dict[str, Any]) -> None: - """Match mini-swe-agent's submit sentinel handling for sandbox-backed runs.""" + """Raise ``Submitted`` when the command output begins with the submit sentinel. + + Args: + output: The execute-result dict whose ``output`` and ``returncode`` are inspected. + + Raises: + Submitted: If the first output line is the submit sentinel and the return code is zero. + """ lines = output.get("output", "").lstrip().splitlines(keepends=True) if lines and lines[0].strip() == "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT" and output["returncode"] == 0: submission = "".join(lines[1:]) @@ -168,6 +232,10 @@ def _check_finished(self, output: dict[str, Any]) -> None: ) def cleanup(self) -> None: + """Stop the backing sandbox and mark the environment closed. + + Idempotent: subsequent calls return immediately once the environment is closed. + """ if self._closed: return self._closed = True diff --git a/responses_api_agents/mini_swe_agent_2/tests/test_app.py b/responses_api_agents/mini_swe_agent_2/tests/test_app.py index 57ce79ad09..042acfa99c 100644 --- a/responses_api_agents/mini_swe_agent_2/tests/test_app.py +++ b/responses_api_agents/mini_swe_agent_2/tests/test_app.py @@ -12,12 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for the mini SWE agent app: parameter helpers, trajectory splitting, sandbox runs, and metric aggregation.""" + import json import sys from pathlib import Path from types import ModuleType, SimpleNamespace from typing import Any, Dict, Optional -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest import yaml @@ -134,6 +136,16 @@ def create_test_config( port: int = 8080, model_name: str = "test_model", ) -> MiniSWEAgentConfig: + """Build a MiniSWEAgentConfig populated with defaults suitable for tests. + + Args: + host (str): Host the agent server binds to. + port (int): Port the agent server binds to. + model_name (str): Name of the referenced model server. + + Returns: + MiniSWEAgentConfig: A configuration wired to use the sandbox environment. + """ return MiniSWEAgentConfig( name="mini_swe_agent_2", host=host, @@ -151,6 +163,12 @@ def create_test_config( def setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict): + """Configure server-client and server-config mocks with default return values. + + Args: + mock_load_from_global_config: Mock for ServerClient.load_from_global_config. + mock_get_first_server_config_dict: Mock for the first-server config lookup. + """ mock_server_client_instance = MagicMock() mock_server_client_instance.global_config_dict = {"policy_model_name": "test_model"} mock_load_from_global_config.return_value = mock_server_client_instance @@ -162,6 +180,12 @@ def setup_server_client_mocks(mock_load_from_global_config, mock_get_first_serve def setup_config_path_mock(mock_get_config_path, config_yaml: str = DEFAULT_CONFIG_YAML): + """Configure the config-path mock to return a path whose text is the given YAML. + + Args: + mock_get_config_path: Mock for the get_config_path lookup. + config_yaml (str): YAML content the mocked config path returns from read_text. + """ mock_config_path = MagicMock() mock_config_path.read_text.return_value = config_yaml mock_get_config_path.return_value = mock_config_path @@ -172,16 +196,22 @@ def setup_run_mini_swe_mock( mock_runner_ray_remote, run_mini_swe_result: Dict[str, Any] = None, ): - """Setup mock for Ray-based run_mini_swe execution""" + """Configure mocks so the Ray-based run_mini_swe execution returns a fixed result. + + Args: + mock_to_thread: Mock for asyncio.to_thread, which drives ray.get. + mock_runner_ray_remote: Mock for the Ray remote runner function. + run_mini_swe_result (Dict[str, Any]): Result to return; defaults to DEFAULT_RUN_MINI_SWE_RESULT when None. + """ if run_mini_swe_result is None: run_mini_swe_result = DEFAULT_RUN_MINI_SWE_RESULT - # Mock the Ray remote function to return a future-like object + # Make the Ray remote function return a future-like object. mock_future = MagicMock() mock_runner_ray_remote.remote.return_value = mock_future mock_runner_ray_remote.options.return_value.remote.return_value = mock_future - # Mock asyncio.to_thread (which calls ray.get) to return the result + # asyncio.to_thread drives ray.get; have it return the result directly. mock_to_thread.return_value = run_mini_swe_result @@ -195,7 +225,21 @@ def create_run_request( split: str = "train", input_data: list = None, ) -> MiniSWEAgentRunRequest: - """Create a test run request with default values.""" + """Create a run request with default values for use in tests. + + Args: + instance_id (str): Identifier of the task instance. + temperature (float): Sampling temperature for the model. + top_p (float): Nucleus sampling probability for the model. + max_output_tokens (int | None): Optional cap on generated tokens. + metadata (dict[str, Any] | None): Optional metadata passed in the responses params. + subset (str): Dataset subset name. + split (str): Dataset split name. + input_data (list): Input messages; defaults to an empty list when None. + + Returns: + MiniSWEAgentRunRequest: The assembled run request. + """ if input_data is None: input_data = [] @@ -219,6 +263,17 @@ def create_chat_completion_request( temperature: float = 0.7, max_tokens: Optional[int] = None, ) -> NeMoGymChatCompletionCreateParamsNonStreaming: + """Create chat-completion request params with default values for use in tests. + + Args: + model (str): Model name to request. + messages (list): Chat messages; defaults to a single user greeting when None. + temperature (float): Sampling temperature for the model. + max_tokens (Optional[int]): Optional cap on generated tokens, included only when set. + + Returns: + NeMoGymChatCompletionCreateParamsNonStreaming: The assembled request params. + """ if messages is None: messages = [{"role": "user", "content": "Hello!"}] @@ -236,6 +291,15 @@ def assert_run_response( expected_top_p: float = 0.8, expected_input_length: int = 2, ): + """Assert that a run response matches the expected reward, sampling params, and input shape. + + Args: + response (MiniSWEAgentVerifyResponse): The response returned by the agent run. + expected_reward (float): Reward value the response is expected to carry. + expected_temperature (float): Expected sampling temperature in the response params. + expected_top_p (float): Expected nucleus sampling probability in the response params. + expected_input_length (int): Expected number of input messages in the response params. + """ assert isinstance(response, MiniSWEAgentVerifyResponse) assert response.reward == expected_reward assert response.responses_create_params.temperature == expected_temperature @@ -253,6 +317,14 @@ def assert_run_mini_swe_called( split: str = "train", instance_id: str = "test_instance_123", ): + """Assert that the run_mini_swe execution was invoked exactly once with positional args. + + Args: + mock_to_thread: Mock for asyncio.to_thread used to dispatch the run. + subset (str): Dataset subset name expected for the run. + split (str): Dataset split name expected for the run. + instance_id (str): Task instance identifier expected for the run. + """ mock_to_thread.assert_called_once() call_args = mock_to_thread.call_args args = call_args[0] @@ -260,11 +332,15 @@ def assert_run_mini_swe_called( class TestApp: + """Tests for the MiniSWEAgent server: helpers, sandbox runs, endpoints, and metrics.""" + def test_sanity(self) -> None: + """Construct a MiniSWEAgent with an empty model name to confirm it instantiates.""" config = create_test_config(model_name="") MiniSWEAgent(config=config, server_client=MagicMock(spec=ServerClient)) def test_response_param_helpers_cover_metadata_and_tool_choice_modes(self) -> None: + """Verify metadata parsing and tool-choice normalization in the param helpers.""" assert _json_dict_from_metadata(None, field_name="extra_body") == {} assert _json_dict_from_metadata({"top_k": 20}, field_name="extra_body") == {"top_k": 20} @@ -303,6 +379,7 @@ def test_response_param_helpers_cover_metadata_and_tool_choice_modes(self) -> No _json_dict_from_metadata("[]", field_name="extra_body") def test_sandbox_resource_profiles_override_static_resources(self) -> None: + """Verify resource profiles take precedence over static resources in the sandbox spec.""" spec = _sandbox_spec_for_instance( {"resources": {"cpu": 1, "memory_mib": 8192, "disk_gib": 20}}, resource_profiles=[ @@ -319,6 +396,7 @@ def test_sandbox_resource_profiles_override_static_resources(self) -> None: assert _sandbox_spec_for_instance(None, resource_profiles=None, instance_id="task") == {} def test_sandbox_provider_config_dump_strips_api_key(self) -> None: + """Verify the config dump removes the API key while the runtime env still carries it.""" provider = { "opensandbox": { "connection": { @@ -336,6 +414,7 @@ def test_sandbox_provider_config_dump_strips_api_key(self) -> None: } def test_split_trajectory_and_resolution_helpers_cover_edge_cases(self) -> None: + """Verify trajectory splitting into input/output/raw items and resolution detection on edge cases.""" input_messages, output_items, raw_responses = _split_trajectory_for_responses( [ {"role": "system", "content": "sys"}, @@ -375,6 +454,12 @@ def test_split_trajectory_and_resolution_helpers_cover_edge_cases(self) -> None: ) def test_misc_mini_swe_helpers(self, monkeypatch, tmp_path) -> None: + """Verify image-name resolution, message-content flattening, and config-path discovery helpers. + + Args: + monkeypatch: Pytest fixture for patching module attributes. + tmp_path: Pytest fixture providing a temporary directory. + """ assert _swebench_image_name({"instance_id": "django__django-1"}, "verified") == ( "docker.io/swebench/sweb.eval.x86_64.django_1776_django-1:latest" ) @@ -396,6 +481,11 @@ def test_misc_mini_swe_helpers(self, monkeypatch, tmp_path) -> None: assert _swebench_config_path() == tmp_path / "missing" / "extra" / "swebench.yaml" def test_run_mini_swe_records_completion_and_errors(self, monkeypatch) -> None: + """Verify run_mini_swe_with_sandbox passes through results and propagates runner errors. + + Args: + monkeypatch: Pytest fixture for patching the underlying runner. + """ monkeypatch.setattr( mini_swe_app_module, "_run_mini_swe_v2", @@ -432,16 +522,41 @@ def fail_runner(**_params): assert run_mini_swe_with_sandbox(env="sandbox", instance_id="task-1") == {"task-1": "bad"} def test_run_mini_swe_v2_success_and_golden_paths(self, monkeypatch, tmp_path) -> None: + """Verify _run_mini_swe_v2 across the agent-run, golden-patch, string-instance, and error paths. + + Args: + monkeypatch: Pytest fixture for patching modules and environment. + tmp_path: Pytest fixture providing a temporary directory for config and output. + """ holder: dict[str, Any] = {} class FakeLogger: + """Minimal logger stub whose info method discards messages.""" + def info(self, _message: str) -> None: return None def setup_logger(_instance_id: str, _log_file: Path) -> FakeLogger: + """Return a FakeLogger, ignoring the instance id and log file arguments. + + Args: + _instance_id (str): Task instance identifier (unused). + _log_file (Path): Destination log path (unused). + + Returns: + FakeLogger: A throwaway logger. + """ return FakeLogger() def make_test_spec(instance: dict[str, Any]) -> SimpleNamespace: + """Build a stand-in test spec exposing an instance id and eval script. + + Args: + instance (dict[str, Any]): Instance dict supplying the instance id. + + Returns: + SimpleNamespace: Object with instance_id and eval_script attributes. + """ return SimpleNamespace( instance_id=instance["instance_id"], eval_script="#!/bin/bash\npytest -q", @@ -454,34 +569,88 @@ def get_eval_report( test_log_path: str, **_kwargs: Any, ): + """Return a stub eval report after confirming the test log path exists. + + Args: + test_spec (SimpleNamespace): Spec supplying the instance id. + prediction (dict[str, Any]): Prediction payload echoed into the report. + test_log_path (str): Path to the test log; asserted to exist. + **_kwargs (Any): Additional keyword arguments (unused). + + Returns: + dict: Report keyed by instance id marking the task resolved. + """ assert Path(test_log_path).exists() return {test_spec.instance_id: {"resolved": True, "prediction": prediction}} class FakeEnv: + """Stand-in sandbox environment recording executed commands and cleanup state.""" + def __init__(self, config: dict[str, Any]) -> None: + """Store the config and initialize command tracking and cleanup flag. + + Args: + config (dict[str, Any]): Environment configuration dict. + """ self.config = config self.commands: list[tuple[str, bool]] = [] self.cleaned = False def execute(self, command: str, is_eval: bool = False) -> dict[str, Any]: + """Record a command and return a successful fake result. + + Args: + command (str): Command to record. + is_eval (bool): Whether the command is an evaluation step. + + Returns: + dict[str, Any]: Fake output with a zero return code. + """ self.commands.append((command, is_eval)) return {"output": "tests passed", "returncode": 0} def cleanup(self) -> None: + """Mark the environment as cleaned up.""" self.cleaned = True class FakeAgent: + """Stand-in agent that records its config and returns a fixed trajectory.""" + def __init__(self, model: Any, env: FakeEnv, **agent_config: Any) -> None: + """Store the model, environment, and agent config. + + Args: + model (Any): Model object handed to the agent. + env (FakeEnv): Sandbox environment the agent runs against. + **agent_config (Any): Remaining agent configuration, also recorded in the holder. + """ self.model = model self.env = env self.agent_config = agent_config holder["agent_config"] = agent_config def run(self, problem_statement: str) -> dict[str, Any]: + """Return a fixed submitted result after asserting the problem statement. + + Args: + problem_statement (str): Task description; asserted to equal the fixture value. + + Returns: + dict[str, Any]: Result with a submitted status and a submission diff. + """ assert problem_statement == "Fix the bug" return {"exit_status": "submitted", "submission": "diff --git a/file b/file"} def save(self, path: Path | None, metadata: dict[str, Any]) -> dict[str, Any]: + """Record save inputs, optionally write the trajectory file, and return fixed messages. + + Args: + path (Path | None): Destination path; written with placeholder content when set. + metadata (dict[str, Any]): Save metadata recorded in the holder. + + Returns: + dict[str, Any]: A fixed messages trajectory for the run. + """ holder["save_path"] = path holder["save_metadata"] = metadata if path is not None: @@ -520,11 +689,27 @@ def save(self, path: Path | None, metadata: dict[str, Any]) -> dict[str, Any]: } def get_environment(config: dict[str, Any]) -> FakeEnv: + """Build a FakeEnv from the config and record it in the holder. + + Args: + config (dict[str, Any]): Environment configuration dict. + + Returns: + FakeEnv: The constructed fake environment. + """ env = FakeEnv(config) holder["env"] = env return env def get_model(config: dict[str, Any]) -> SimpleNamespace: + """Record the model config and return a namespace wrapping it. + + Args: + config (dict[str, Any]): Model configuration dict. + + Returns: + SimpleNamespace: Object exposing the config as an attribute. + """ holder["model_config"] = config return SimpleNamespace(config=config) @@ -716,6 +901,17 @@ async def test_run_writes_generation_params_to_config( tmp_path, monkeypatch, ) -> None: + """Verify run writes sampling params and tool choice into the generated config and runtime env. + + Args: + mock_to_thread: Mock for asyncio.to_thread. + mock_runner_ray_remote: Mock for the Ray remote runner. + mock_get_config_path: Mock for the config-path lookup. + mock_get_first_server_config_dict: Mock for the first-server config lookup. + mock_load_from_global_config: Mock for ServerClient.load_from_global_config. + tmp_path: Pytest fixture providing a temporary directory. + monkeypatch: Pytest fixture used to change the working directory. + """ monkeypatch.chdir(tmp_path) config = create_test_config() config.tool_choice = "bash" @@ -788,11 +984,11 @@ async def test_run_failed_execution( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - # Mock Ray remote function + # Provide a fake Ray remote future. mock_future = MagicMock() mock_runner_ray_remote.remote.return_value = mock_future - # Mock asyncio.to_thread (ray.get) to raise an exception + # Make the dispatched run raise a generic exception. mock_to_thread.side_effect = Exception("run_mini_swe failed") run_request = create_run_request(instance_id="test_instance_456", temperature=0.3, top_p=0.95) @@ -822,6 +1018,15 @@ async def test_run_mini_swe_not_found( mock_get_first_server_config_dict, mock_load_from_global_config, ) -> None: + """Verify run returns a zero-reward response when the runner raises FileNotFoundError. + + Args: + mock_to_thread: Mock for asyncio.to_thread. + mock_runner_ray_remote: Mock for the Ray remote runner. + mock_get_config_path: Mock for the config-path lookup. + mock_get_first_server_config_dict: Mock for the first-server config lookup. + mock_load_from_global_config: Mock for ServerClient.load_from_global_config. + """ config = create_test_config() mock_server_client = MagicMock(spec=ServerClient) server = MiniSWEAgent(config=config, server_client=mock_server_client) @@ -829,11 +1034,11 @@ async def test_run_mini_swe_not_found( setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) setup_config_path_mock(mock_get_config_path) - # Mock Ray remote function + # Provide a fake Ray remote future. mock_future = MagicMock() mock_runner_ray_remote.remote.return_value = mock_future - # Mock asyncio.to_thread (ray.get) to raise FileNotFoundError + # Make the dispatched run raise FileNotFoundError. mock_to_thread.side_effect = FileNotFoundError("run_mini_swe not found") run_request = create_run_request(instance_id="test_instance_789", temperature=0.2, top_p=1.0) @@ -851,6 +1056,7 @@ async def test_run_mini_swe_not_found( assert_run_mini_swe_called(mock_to_thread, instance_id="test_instance_789") async def test_responses_not_implemented(self) -> None: + """Verify the responses endpoint raises NotImplementedError.""" config = create_test_config() mock_server_client = MagicMock(spec=ServerClient) server = MiniSWEAgent(config=config, server_client=mock_server_client) @@ -861,6 +1067,7 @@ async def test_responses_not_implemented(self) -> None: await server.responses(request_body) async def test_aggregate_metrics_includes_eval_results(self) -> None: + """Verify aggregate_metrics reports pass@k, resolution, test status, and per-group eval results.""" config = create_test_config() mock_server_client = MagicMock(spec=ServerClient) server = MiniSWEAgent(config=config, server_client=mock_server_client) @@ -936,6 +1143,7 @@ async def test_aggregate_metrics_includes_eval_results(self) -> None: assert groups[1]["eval_error_rollout_count"] == 2 def test_endpoints_registration(self) -> None: + """Verify the responses, run, and aggregate_metrics endpoints are registered and reachable.""" config = create_test_config() mock_server_client = MagicMock(spec=ServerClient) server = MiniSWEAgent(config=config, server_client=mock_server_client) @@ -951,3 +1159,235 @@ def test_endpoints_registration(self) -> None: aggregate_response = client.post("/aggregate_metrics", json={"verify_responses": []}) assert aggregate_response.status_code == 200 + + +def _create_verifier_run_request( + instance_id: str = "psf__requests-2317", + subset: str = "verified", + split: str = "test", +) -> MiniSWEAgentRunRequest: + """Build a run request carrying the extra SWE-bench instance fields the verifier metadata reads. + + Args: + instance_id (str): Task instance identifier. + subset (str): Dataset subset name. + split (str): Dataset split name. + + Returns: + MiniSWEAgentRunRequest: A run request populated with base commit, test patch, and test node ids. + """ + return MiniSWEAgentRunRequest( + instance_id=instance_id, + subset=subset, + split=split, + base_commit="abc123", + test_patch="TP", + FAIL_TO_PASS=["test_x.py::test_a"], + PASS_TO_PASS=["test_x.py::test_b"], + responses_create_params=NeMoGymResponseCreateParamsNonStreaming( + input=[], + temperature=0.5, + top_p=0.8, + ), + ) + + +class TestCrossAgentVerifierReuse: + """Tests for opt-in scoring of the agent's patch via the shared SWE environment verifier.""" + + def _server(self, eval_via_verifier: bool = True) -> MiniSWEAgent: + """Construct a MiniSWEAgent configured to score patches through the shared verifier. + + Args: + eval_via_verifier (bool): Whether to route scoring through the verifier server. + + Returns: + MiniSWEAgent: An agent wired to the verifier server with a mocked server client. + """ + config = create_test_config() + config.eval_via_verifier = eval_via_verifier + config.verifier_server_name = "swe_verifier" + config.sandbox_provider = {"opensandbox": {}} + return MiniSWEAgent(config=config, server_client=MagicMock(spec=ServerClient)) + + def test_config_defaults_keep_legacy_path(self) -> None: + """Verify verifier-based scoring is disabled by default in the config.""" + config = create_test_config() + assert config.eval_via_verifier is False + assert config.verifier_server_name is None + + async def test_verify_patch_via_server_builds_request_and_parses_subset(self, monkeypatch) -> None: + """Verify _verify_patch_via_server builds the verifier request and parses the returned subset. + + Args: + monkeypatch: Pytest fixture for patching HTTP helpers in the app module. + """ + server = self._server() + monkeypatch.setattr(mini_swe_app_module, "raise_for_status", AsyncMock(return_value=None)) + monkeypatch.setattr( + mini_swe_app_module, + "get_response_json", + AsyncMock(return_value={"resolved": True, "error_kind": None, "patch_exists": True, "reward": 1.0}), + ) + server.server_client.post = AsyncMock(return_value=MagicMock()) + + body = _create_verifier_run_request() + subset = await server._verify_patch_via_server( + instance=body.model_dump(), + patch="<>", + instance_id="psf__requests-2317", + subset="verified", + split="test", + responses_create_params=body.responses_create_params, + ) + + assert subset["resolved"] is True + call = server.server_client.post.call_args + # The request POSTs to the shared verifier via the server client. + assert call.kwargs["server_name"] == "swe_verifier" + assert call.kwargs["url_path"] == "/verify" + req = call.kwargs["json"] + # The patch travels in response.metadata.model_patch, the field the verifier reads. + assert req["response"]["metadata"]["model_patch"] == "<>" + md = req["responses_create_params"]["metadata"] + assert md["instance_id"] == "psf__requests-2317" + # The image is resolved from the instance id and subset, munging __ to _1776_ for verified. + assert md["image"] == "docker.io/swebench/sweb.eval.x86_64.psf_1776_requests-2317:latest" + # The instance ships no test_command, so the conda+pytest default carrying the + # fail-to-pass and pass-to-pass node ids is used. + assert "conda activate testbed" in md["test_command"] + assert "test_x.py::test_a" in md["test_command"] + assert "test_x.py::test_b" in md["test_command"] + # With no per-row framework, an empty value is forwarded so the verifier auto-detects. + assert md["test_framework"] == "" + assert md["fail_to_pass"] == ["test_x.py::test_a"] + assert md["pass_to_pass"] == ["test_x.py::test_b"] + assert md["base_commit"] == "abc123" + assert md["test_patch"] == "TP" + assert md["repo_workdir"] == "/testbed" + assert md["split"] == "test" + # The benchmark is the registered harness key the test_command targets, not the subset. + assert md["benchmark"] == "swe-bench-ext" + + async def test_verify_patch_via_server_infra_error_is_masked_not_raised(self, monkeypatch) -> None: + """Verify an infrastructure error during verification is masked into a present, unresolved subset. + + Args: + monkeypatch: Pytest fixture (unused but kept for signature consistency). + """ + server = self._server() + server.server_client.post = AsyncMock(side_effect=RuntimeError("connreset")) + + body = _create_verifier_run_request() + subset = await server._verify_patch_via_server( + instance=body.model_dump(), + patch="<>", + instance_id="psf__requests-2317", + subset="verified", + split="test", + responses_create_params=body.responses_create_params, + ) + + # The call never raises; it returns a masked subset so the agent still emits a present row. + assert subset["resolved"] is False + assert subset["error_kind"] == "sandbox" + assert subset["patch_exists"] is True + + async def test_verify_patch_via_server_forwards_instance_test_command_and_framework(self, monkeypatch) -> None: + """Verify a row carrying its own test_command and test_framework is forwarded verbatim. + + A row supplying a per-framework test command (e.g. cargo, go, npm) is forwarded unchanged so + non-pytest rows are graded with their own command rather than the conda+pytest default. + + Args: + monkeypatch: Pytest fixture for patching HTTP helpers in the app module. + """ + server = self._server() + monkeypatch.setattr(mini_swe_app_module, "raise_for_status", AsyncMock(return_value=None)) + monkeypatch.setattr( + mini_swe_app_module, + "get_response_json", + AsyncMock(return_value={"resolved": True, "error_kind": None, "patch_exists": True}), + ) + server.server_client.post = AsyncMock(return_value=MagicMock()) + + body = _create_verifier_run_request() + instance = body.model_dump() + instance["test_command"] = "go test ./..." + instance["test_framework"] = "go" + await server._verify_patch_via_server( + instance=instance, + patch="<>", + instance_id="psf__requests-2317", + subset="verified", + split="test", + responses_create_params=body.responses_create_params, + ) + + md = server.server_client.post.call_args.kwargs["json"]["responses_create_params"]["metadata"] + # The row's command is forwarded verbatim; the conda+pytest default does not override it. + assert md["test_command"] == "go test ./..." + assert md["test_framework"] == "go" + assert "conda activate testbed" not in md["test_command"] + + @patch("responses_api_agents.mini_swe_agent_2.app.ServerClient.load_from_global_config") + @patch("responses_api_agents.mini_swe_agent_2.app.get_first_server_config_dict") + @patch("responses_api_agents.mini_swe_agent_2.app.get_config_path") + @patch("responses_api_agents.mini_swe_agent_2.app.runner_ray_remote") + @patch("asyncio.to_thread") + async def test_run_scores_via_verifier_instead_of_in_process_eval( + self, + mock_to_thread, + mock_runner_ray_remote, + mock_get_config_path, + mock_get_first_server_config_dict, + mock_load_from_global_config, + monkeypatch, + ) -> None: + """Verify run scores via the verifier and skips in-process evaluation on the verifier path. + + Args: + mock_to_thread: Mock for asyncio.to_thread. + mock_runner_ray_remote: Mock for the Ray remote runner. + mock_get_config_path: Mock for the config-path lookup. + mock_get_first_server_config_dict: Mock for the first-server config lookup. + mock_load_from_global_config: Mock for ServerClient.load_from_global_config. + monkeypatch: Pytest fixture for patching app-module attributes. + """ + server = self._server() + setup_server_client_mocks(mock_load_from_global_config, mock_get_first_server_config_dict) + setup_config_path_mock(mock_get_config_path) + # In-worker eval is skipped; the worker returns only the trajectory and patch. + worker_result = { + "test_instance_123": { + "input_messages": [ + {"type": "message", "role": "system", "content": "sys"}, + {"type": "message", "role": "user", "content": "Fix this bug."}, + ], + "response_output": [], + "responses": [], + "eval_report": {"instance_id": "test_instance_123", "model_patch": "<>"}, + } + } + setup_run_mini_swe_mock(mock_to_thread, mock_runner_ray_remote, run_mini_swe_result=worker_result) + + # _is_resolved must not be consulted on the verifier path; the verifier is authoritative. + monkeypatch.setattr( + mini_swe_app_module, + "_is_resolved", + lambda *_a, **_k: pytest.fail("the in-process _is_resolved must not run on the verifier path"), + ) + verify_mock = AsyncMock( + return_value={"resolved": True, "error_kind": None, "patch_exists": True, "reward": 1.0} + ) + monkeypatch.setattr(MiniSWEAgent, "_verify_patch_via_server", verify_mock) + + response = await server.run(create_run_request()) + + # The reward comes from the verifier subset, and the patch was forwarded to it. + assert response.reward == 1.0 + assert verify_mock.await_args.kwargs["patch"] == "<>" + assert verify_mock.await_args.kwargs["instance_id"] == "test_instance_123" + # The worker was told to skip in-process eval. + worker_params = mock_runner_ray_remote.remote.call_args.args[1] + assert worker_params["eval_via_verifier"] is True diff --git a/responses_api_agents/mini_swe_agent_2/tests/test_sandbox_environment.py b/responses_api_agents/mini_swe_agent_2/tests/test_sandbox_environment.py index 0419e15b85..435b906861 100644 --- a/responses_api_agents/mini_swe_agent_2/tests/test_sandbox_environment.py +++ b/responses_api_agents/mini_swe_agent_2/tests/test_sandbox_environment.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for the mini SWE sandbox environment: submit-sentinel detection and command execution.""" from typing import Any @@ -23,6 +24,7 @@ def test_check_finished_raises_submitted_for_submit_sentinel() -> None: + """Verify a zero-return-code submit sentinel raises Submitted carrying the patch as the exit message.""" env = MiniSWESandboxEnvironment.__new__(MiniSWESandboxEnvironment) try: @@ -46,6 +48,7 @@ def test_check_finished_raises_submitted_for_submit_sentinel() -> None: def test_check_finished_ignores_nonzero_submit_sentinel() -> None: + """Verify a submit sentinel with a nonzero return code does not raise Submitted.""" env = MiniSWESandboxEnvironment.__new__(MiniSWESandboxEnvironment) env._check_finished( @@ -58,11 +61,25 @@ def test_check_finished_ignores_nonzero_submit_sentinel() -> None: def test_execute_passes_configured_cwd_without_conda_cd() -> None: + """Verify execute forwards the requested cwd and prepends conda activation without a cd into cwd.""" + class FakeSandbox: + """Stand-in sandbox that records exec calls and returns a successful result.""" + def __init__(self) -> None: + """Initialize the recorded-call list.""" self.calls: list[dict[str, Any]] = [] def exec(self, command: str, **kwargs: Any): + """Record the command and keyword arguments and return a successful fake result. + + Args: + command (str): Command string to record. + **kwargs (Any): Additional exec options (e.g. cwd) to record. + + Returns: + Result: An object exposing stdout, stderr, and a zero return_code. + """ self.calls.append({"command": command, **kwargs}) return type("Result", (), {"stdout": "ok", "stderr": None, "return_code": 0})()