feat: declare failure_reason on BaseVerifyResponse

nemo_gym/base_resources_server.py

@@ -96,6 +96,10 @@ class BaseVerifyRequest(BaseRunRequest):
 class BaseVerifyResponse(BaseVerifyRequest):
     reward: float
 
+    # Human-readable diagnosis of why `reward` may not reflect policy quality.
+    # Machine-readable handling belongs to `mask_sample`/`failure_kind`.
+    failure_reason: Optional[str] = None
+
 
 class BaseMultiRewardVerifyResponse(BaseVerifyResponse):
     """Base verify response for environments with multiple reward objectives.

resources_servers/math_with_judge/tests/test_app.py

@@ -211,6 +211,7 @@ async def test_verify(self, config: LibraryJudgeMathResourcesServerConfig) -> No
         assert sorted(list(not_equal_verify_response.model_dump())) == [
             "expected_answer",
             "extracted_answer",
+            "failure_reason",
             "judge_evaluations",
             "library_reward",
             "response",
@@ -249,6 +250,7 @@ async def test_verify(self, config: LibraryJudgeMathResourcesServerConfig) -> No
         assert sorted(list(equal_verify_response.model_dump())) == [
             "expected_answer",
             "extracted_answer",
+            "failure_reason",
             "judge_evaluations",
             "library_reward",
             "response",

responses_api_agents/tool_simulation_agent/tests/test_app.py

@@ -444,6 +444,7 @@ async def test_run(self, agent_config: ToolSimulationAgentConfig) -> None:
             },
             "response": full_tool_call_response,
             "reward": 1,
+            "failure_reason": None,
         }
         assert valid_verify_response.json() == expected_valid_verify_response_json
         assert server_client_post_mock.call_args_list == expected_invalid_verify_response_calls

tests/unit_tests/test_base_resources_server.py

@@ -18,6 +18,7 @@
 from nemo_gym.base_resources_server import (
     BaseMultiRewardVerifyResponse,
     BaseResourcesServerConfig,
+    BaseVerifyResponse,
     ReverifyMode,
     SimpleResourcesServer,
 )
@@ -35,6 +36,21 @@ async def verify(self, body):
     return TestSimpleResourcesServer(config=config, server_client=MagicMock(spec=ServerClient))
 
 
+class TestBaseVerifyResponse:
+    def test_failure_reason_defaults_none_and_round_trips(self) -> None:
+        response = BaseVerifyResponse(
+            responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input="hi"),
+            response=NeMoGymResponse.model_construct(id="resp-1", output=[]),
+            reward=0.0,
+        )
+        assert response.failure_reason is None
+        assert response.model_dump()["failure_reason"] is None
+
+        rescued = response.model_copy(update={"failure_reason": "judge response unparseable after 3 attempts"})
+        assert rescued.model_dump()["failure_reason"] == "judge response unparseable after 3 attempts"
+        assert rescued.reward == 0.0
+
+
 class TestBaseMultiRewardVerifyResponse:
     def test_reward_components_round_trip(self) -> None:
         response = BaseMultiRewardVerifyResponse(