From 95d40fcf2607974de9844f37140241180a758526 Mon Sep 17 00:00:00 2001 From: Nikhil Deshmudre Date: Thu, 16 Jul 2026 00:16:58 -0700 Subject: [PATCH 1/4] feat: add BaseMultiRewardVerifyResponse for shared multi-reward contract Define reward_components once in nemo_gym as the trainer-agnostic producer-side contract, and migrate example_tool_call_multireward to inherit it. Closes #1664 Signed-off-by: Nikhil Deshmudre --- nemo_gym/base_resources_server.py | 6 ++++++ .../example_tool_call_multireward/app.py | 8 ++------ tests/unit_tests/test_base_resources_server.py | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/nemo_gym/base_resources_server.py b/nemo_gym/base_resources_server.py index 4d519aca7b..2bdcad4f81 100644 --- a/nemo_gym/base_resources_server.py +++ b/nemo_gym/base_resources_server.py @@ -92,6 +92,12 @@ class BaseVerifyResponse(BaseVerifyRequest): reward: float +class BaseMultiRewardVerifyResponse(BaseVerifyResponse): + """Use the same keys for every task in an environment.""" + + reward_components: dict[str, float] + + class BaseSeedSessionRequest(BaseModel): pass diff --git a/resources_servers/example_tool_call_multireward/app.py b/resources_servers/example_tool_call_multireward/app.py index 76ff41652b..99b4b4627f 100644 --- a/resources_servers/example_tool_call_multireward/app.py +++ b/resources_servers/example_tool_call_multireward/app.py @@ -43,7 +43,7 @@ from nemo_gym.base_resources_server import ( BaseResourcesServerConfig, BaseVerifyRequest, - BaseVerifyResponse, + BaseMultiRewardVerifyResponse, SimpleResourcesServer, ) @@ -58,13 +58,9 @@ class ToolCallMultiRewardVerifyRequest(BaseVerifyRequest): expected_call: Dict[str, Any] = Field(default_factory=dict) -class ToolCallMultiRewardVerifyResponse(BaseVerifyResponse): +class ToolCallMultiRewardVerifyResponse(BaseMultiRewardVerifyResponse): # Per-component scores are also surfaced as top-level fields so the aggregate # metrics endpoint profiles each one in addition to the combined reward. - # Decoupled per-component rewards (name -> score). How these reach a trainer - # depends on the training framework's NeMo Gym integration. Defined here (not on - # BaseVerifyResponse) so other environments' verify responses are unchanged. - reward_components: Dict[str, float] | None = None correctness: float = 0.0 schema_valid: float = 0.0 format: float = 0.0 diff --git a/tests/unit_tests/test_base_resources_server.py b/tests/unit_tests/test_base_resources_server.py index 076e1fe979..57ca65e239 100644 --- a/tests/unit_tests/test_base_resources_server.py +++ b/tests/unit_tests/test_base_resources_server.py @@ -18,6 +18,7 @@ from fastapi import Request from nemo_gym.base_resources_server import ( + BaseMultiRewardVerifyResponse, BaseResourcesServerConfig, BaseSeedSessionRequest, BaseSeedSessionResponse, @@ -27,9 +28,23 @@ SimpleResourcesServer, gym_tool, ) +from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.server_utils import SESSION_ID_KEY, ServerClient +class TestBaseMultiRewardVerifyResponse: + def test_reward_components_round_trip(self) -> None: + response = BaseMultiRewardVerifyResponse( + responses_create_params=NeMoGymResponseCreateParamsNonStreaming(input="hi"), + response=NeMoGymResponse.model_construct(id="resp-1", output=[]), + reward=2.0, + reward_components={"correctness": 1.0, "format": 1.0}, + ) + dumped = response.model_dump() + assert dumped["reward_components"] == {"correctness": 1.0, "format": 1.0} + assert dumped["reward"] == 2.0 + + class TestBaseResourcesServer: def test_sanity(self) -> None: config = BaseResourcesServerConfig(host="", port=0, entrypoint="", name="") From 6b9a4020b34e0456b51a470d6298761d31050ab0 Mon Sep 17 00:00:00 2001 From: Nikhil Deshmudre Date: Fri, 24 Jul 2026 14:04:39 -0700 Subject: [PATCH 2/4] docs: document shared multi-reward response contract Signed-off-by: Nikhil Deshmudre --- .../latest/pages/api-reference/index.mdx | 2 +- .../multi-reward-verification.mdx | 36 ++++++++++++------- nemo_gym/base_resources_server.py | 12 ++++++- .../example_tool_call_multireward/README.md | 3 ++ 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/fern/versions/latest/pages/api-reference/index.mdx b/fern/versions/latest/pages/api-reference/index.mdx index 9692a94c36..cefd97b15c 100644 --- a/fern/versions/latest/pages/api-reference/index.mdx +++ b/fern/versions/latest/pages/api-reference/index.mdx @@ -11,7 +11,7 @@ This reference is built from docstrings in the [source code](https://github.com/ | Module | Description | |--------|-------------| -| `nemo_gym.base_resources_server` | Base classes for building resources servers (`SimpleResourcesServer`, `BaseVerifyRequest`, `BaseVerifyResponse`) | +| `nemo_gym.base_resources_server` | Base classes for building resources servers (`SimpleResourcesServer`, `BaseVerifyRequest`, `BaseVerifyResponse`, `BaseMultiRewardVerifyResponse`) | | `nemo_gym.base_responses_api_agent` | Base classes for building agent servers (`SimpleResponsesAPIAgent`) | | `nemo_gym.base_responses_api_model` | Base classes for building model servers (`SimpleResponsesAPIModel`) | | `nemo_gym.config_types` | Pydantic configuration models for servers, datasets, and CLI | diff --git a/fern/versions/latest/pages/build-verifiers/multi-reward-verification.mdx b/fern/versions/latest/pages/build-verifiers/multi-reward-verification.mdx index 92852f5481..123b4c89dd 100644 --- a/fern/versions/latest/pages/build-verifiers/multi-reward-verification.mdx +++ b/fern/versions/latest/pages/build-verifiers/multi-reward-verification.mdx @@ -38,13 +38,16 @@ Skip if you already have one. Otherwise generate the app, config, and test layou gym env init --resources-server my_server ``` -### 2. Subclass `BaseVerifyResponse` to add `reward_components` +### 2. Subclass `BaseMultiRewardVerifyResponse` -This is the multi-reward contract: a `{objective_name: score}` dict. Defining it on a subclass (rather than on `BaseVerifyResponse`) keeps every other environment's verify response unchanged. +`BaseMultiRewardVerifyResponse` defines the shared multi-reward contract: a required `{objective_name: score}` dict. Use the same objective keys for every task in an environment. Environments that only return a scalar reward should continue to use `BaseVerifyResponse`. ```python -class MultiRewardResponse(BaseVerifyResponse): - reward_components: Dict[str, float] | None = None +from nemo_gym.base_resources_server import BaseMultiRewardVerifyResponse + + +class MultiRewardResponse(BaseMultiRewardVerifyResponse): + pass ``` ### 3. Set the scalar `reward` @@ -113,11 +116,10 @@ The `self._…` helpers (extracting calls, parsing arguments, checking required ### 3. Return all three forms -The example's response class adds the `reward_components` dict and the three top-level component fields. The scalar `reward` is already declared on `BaseVerifyResponse`, so it isn't redeclared here — `verify()` computes its value (the sum) in the return below. +The example's response class inherits the required `reward_components` dict from `BaseMultiRewardVerifyResponse` and adds the three top-level component fields. The scalar `reward` is inherited through `BaseVerifyResponse`, so it isn't redeclared here — `verify()` computes its value (the sum) in the return below. ```python -class ToolCallMultiRewardVerifyResponse(BaseVerifyResponse): - reward_components: Dict[str, float] | None = None +class ToolCallMultiRewardVerifyResponse(BaseMultiRewardVerifyResponse): correctness: float = 0.0 schema_valid: float = 0.0 format: float = 0.0 @@ -165,12 +167,20 @@ gym eval run --no-serve --agent example_tool_call_multireward_simple_agent \ Because `correctness`, `schema_valid`, and `format` are top-level numeric fields, the [aggregate-metrics](/evaluation/aggregate-metrics) step reports a separate mean (pass rate) for each one alongside the summed `reward`. The metrics file then has an entry per component (illustrative): ```json -{ - "reward": {"mean": 2.6, "min": 1.0, "max": 3.0}, - "correctness": {"mean": 0.8, "min": 0.0, "max": 1.0}, - "schema_valid": {"mean": 1.0, "min": 1.0, "max": 1.0}, - "format": {"mean": 0.8, "min": 0.0, "max": 1.0} -} +[ + { + "agent_ref": {"name": "example_tool_call_multireward_simple_agent"}, + "agent_metrics": { + "mean/reward": 2.6, + "mean/correctness": 0.8, + "mean/schema_valid": 1.0, + "mean/format": 0.8 + }, + "key_metrics": { + "mean/reward": 2.6 + } + } +] ``` That breakdown tells you *where* the agent struggles. Above, the model always emits schema-valid calls (`schema_valid` = 1.0) and usually picks the right city (`correctness` = 0.8), but sometimes wraps the call in extra prose (`format` = 0.8) — detail a single conflated score would hide. diff --git a/nemo_gym/base_resources_server.py b/nemo_gym/base_resources_server.py index e357e22546..44fe6e2195 100644 --- a/nemo_gym/base_resources_server.py +++ b/nemo_gym/base_resources_server.py @@ -87,7 +87,17 @@ class BaseVerifyResponse(BaseVerifyRequest): class BaseMultiRewardVerifyResponse(BaseVerifyResponse): - """Use the same keys for every task in an environment.""" + """Base verify response for environments with multiple reward objectives. + + Subclass this response instead of declaring ``reward_components`` on an + environment-specific ``BaseVerifyResponse`` subclass. The mapping is required, and + its objective keys should remain consistent across every task in the environment. + + Set the inherited ``reward`` to the scalar aggregate expected by single-reward + consumers. To include individual objectives in aggregate metrics, also expose them + as top-level numeric fields because metrics do not descend into this mapping. See + ``resources_servers/example_tool_call_multireward`` for a complete example. + """ reward_components: dict[str, float] diff --git a/resources_servers/example_tool_call_multireward/README.md b/resources_servers/example_tool_call_multireward/README.md index c0a86c01d1..6d13484cc1 100644 --- a/resources_servers/example_tool_call_multireward/README.md +++ b/resources_servers/example_tool_call_multireward/README.md @@ -15,6 +15,9 @@ rollout on three independent `{0, 1}` components: Each component is surfaced both as a top-level numeric field on the verify response and inside the `reward_components` field, alongside the summed scalar `reward`. +The response subclasses `BaseMultiRewardVerifyResponse`, which makes +`reward_components` required and establishes the shared multi-reward contract. Component +keys should remain consistent across every task in the environment. - **Evaluation**: because the components are top-level numeric fields, NeMo Gym's aggregate-metrics step reports an independent pass rate for each one. This shows *how* From adfd7b07c671befccb06602fcc723941cfc0d451 Mon Sep 17 00:00:00 2001 From: Nikhil Deshmudre Date: Fri, 24 Jul 2026 14:10:48 -0700 Subject: [PATCH 3/4] style: sort multi-reward server imports Signed-off-by: Nikhil Deshmudre --- resources_servers/example_tool_call_multireward/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources_servers/example_tool_call_multireward/app.py b/resources_servers/example_tool_call_multireward/app.py index 99b4b4627f..12cb75922b 100644 --- a/resources_servers/example_tool_call_multireward/app.py +++ b/resources_servers/example_tool_call_multireward/app.py @@ -41,9 +41,9 @@ from pydantic import Field from nemo_gym.base_resources_server import ( + BaseMultiRewardVerifyResponse, BaseResourcesServerConfig, BaseVerifyRequest, - BaseMultiRewardVerifyResponse, SimpleResourcesServer, ) From c7e83a9361a13a66697aa77a707ae5e7bd01a778 Mon Sep 17 00:00:00 2001 From: Nikhil Deshmudre Date: Fri, 24 Jul 2026 14:20:11 -0700 Subject: [PATCH 4/4] docs: clarify scalar multi-reward aggregation Signed-off-by: Nikhil Deshmudre --- .../latest/pages/build-verifiers/multi-reward-verification.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fern/versions/latest/pages/build-verifiers/multi-reward-verification.mdx b/fern/versions/latest/pages/build-verifiers/multi-reward-verification.mdx index 123b4c89dd..dca194805b 100644 --- a/fern/versions/latest/pages/build-verifiers/multi-reward-verification.mdx +++ b/fern/versions/latest/pages/build-verifiers/multi-reward-verification.mdx @@ -52,7 +52,7 @@ class MultiRewardResponse(BaseMultiRewardVerifyResponse): ### 3. Set the scalar `reward` -Every verify response carries a scalar `reward`, and most consumers read it rather than the components: aggregate metrics reports it as the overall score for evaluation, and single-reward trainers (e.g. GRPO) use it directly. Summing the components is a common convention. +Every verify response carries a scalar `reward`, and most consumers read it rather than the components: aggregate metrics reports it as the overall score for evaluation, and single-reward trainers (e.g. GRPO) use it directly. A weighted sum of the components is commonly used as the scalar reward. ### 4. Expose each reward component as a top-level field