Outputs builder

docs/reference.md

@@ -116,6 +116,29 @@ class RolloutInput(TypedDict):
     info: Info              # Optional
 ```
 
+### RolloutOutput
+
+```python
+class RolloutOutput(TypedDict):
+    # Required fields
+    example_id: int
+    task: str
+    prompt: Messages | None
+    completion: Messages | None
+    reward: float
+    timing: RolloutTiming
+    is_completed: bool
+    is_truncated: bool
+    metrics: dict[str, float]
+    # Optional fields
+    answer: str
+    info: Info
+    error: str | None
+    stop_condition: str | None
+```
+
+Serialized output from a rollout. This is the JSON-serializable representation of a `State` object, used in `GenerateOutputs` and for saving results to disk. Additional fields from `state_columns` may also be present.
+
 ### TrajectoryStep
 
 ```python
@@ -162,11 +185,11 @@ class RolloutTiming(TypedDict, total=False):
 
 ```python
 class GenerateOutputs(TypedDict):
-    states: list[State]
+    outputs: list[RolloutOutput]
     metadata: GenerateMetadata
 ```
 
-Output from `Environment.generate()`. Contains a list of `State` objects (one per rollout) and generation metadata. Each `State` contains the rollout's prompt, completion, answer, reward, metrics, and other per-rollout data.
+Output from `Environment.generate()`. Contains a list of `RolloutOutput` objects (one per rollout) and generation metadata. Each `RolloutOutput` is a serialized, JSON-compatible dict containing the rollout's prompt, completion, answer, reward, metrics, timing, and other per-rollout data.
 
 ### GenerateMetadata
 

tests/conftest.py

@@ -27,10 +27,12 @@
     GenerateMetadata,
     Info,
     RolloutInput,
+    RolloutOutput,
     RolloutTiming,
     SamplingArgs,
     TrajectoryStep,
 )
+from verifiers.utils.save_utils import state_to_output
 
 
 @pytest.fixture
@@ -492,6 +494,25 @@ def _make_state(
     return _make_state
 
 
+@pytest.fixture
+def make_output(make_state) -> Callable[..., RolloutOutput]:
+    """Fixture to make RolloutOutput objects for testing.
+
+    This creates a State first, then converts it to a RolloutOutput using
+    state_to_output(). This ensures the output matches the serialized format
+    used in GenerateOutputs.
+    """
+
+    def _make_output(
+        state_columns: list[str] = ["foo"],
+        **kwargs,
+    ) -> RolloutOutput:
+        state = make_state(**kwargs)
+        return state_to_output(state, state_columns)
+
+    return _make_output
+
+
 @pytest.fixture
 def make_metadata() -> Callable[..., GenerateMetadata]:
     """Fixture to make GenerateMetadata objects for testing."""

tests/test_env_group.py

@@ -406,7 +406,7 @@ async def mock_score_group(states, score_sem=None):
             inputs, client=mock_openai_client, model="test-model"
         )
 
-        states = outputs["states"]
+        states = outputs["outputs"]
         assert len(states) == 2
         for state in states:
             assert "completion" in state

tests/test_environment.py

@@ -249,7 +249,7 @@ async def mock_score_group(states, score_sem=None):
             model="test-model",
         )
 
-        states = outputs["states"]
+        states = outputs["outputs"]
         assert len(states) == 1
         assert "completion" in states[0]
         assert "reward" in states[0]
@@ -280,26 +280,25 @@ async def mock_score_group(states, score_sem=None):
             model="test-model",
         )
 
-        states = outputs["states"]
+        states = outputs["outputs"]
         assert len(states) == 1
         assert "completion" in states[0]
         assert "reward" in states[0]
         assert states[0]["reward"] == 1.0
 
-    def test_make_dataset(self, make_metadata, make_state):
+    def test_make_dataset(self, make_metadata, make_output):
         """Test creating a dataset from evaluation results."""
 
-        outputs = GenerateOutputs(states=[make_state()], metadata=make_metadata())
-        dataset = build_dataset(outputs)
+        results = GenerateOutputs(outputs=[make_output()], metadata=make_metadata())
+        dataset = build_dataset(results)
 
         assert len(dataset) == 1
         assert "prompt" in dataset.column_names
         assert "completion" in dataset.column_names
-        assert "answer" in dataset.column_names
         assert "reward" in dataset.column_names
         assert "task" in dataset.column_names
         assert "example_id" in dataset.column_names
-        assert "foo" in dataset.column_names  # custom field from make_state fixture
+        assert "foo" in dataset.column_names  # custom field from make_output fixture
 
     @pytest.mark.asyncio
     async def test_generate_updates_metadata(self, mock_openai_client):
@@ -484,7 +483,7 @@ async def test_retry_after_retryable_error(self, mock_openai_client, make_input)
         outputs = await env.generate(
             inputs, client=mock_openai_client, model="test-model", max_retries=3
         )
-        states = outputs["states"]
+        states = outputs["outputs"]
 
         assert states[0].get("error") is None
         assert env.call_counts[0] == 3
@@ -508,10 +507,12 @@ async def test_no_retry_after_non_retryable_error(
             inputs, client=mock_openai_client, model="test-model", max_retries=3
         )
 
-        states = outputs["states"]
+        rollout_outputs = outputs["outputs"]
         assert env.call_counts[0] == 1  # No retries for non-retryable error
-        assert states[0].get("error") is not None
-        assert isinstance(states[0]["error"], vf.ToolError)
+        assert rollout_outputs[0].get("error") is not None
+        assert (
+            "ToolError" in rollout_outputs[0]["error"]
+        )  # Error is serialized as repr string
 
     @pytest.mark.asyncio
     async def test_error_in_state_after_max_retries_exhausted(
@@ -528,10 +529,12 @@ async def test_error_in_state_after_max_retries_exhausted(
             inputs, client=mock_openai_client, model="test-model", max_retries=2
         )
 
-        states = outputs["states"]
+        rollout_outputs = outputs["outputs"]
         assert env.call_counts[0] == 3  # 1 initial + 2 retries
-        assert states[0].get("error") is not None
-        assert isinstance(states[0]["error"], vf.InfraError)
+        assert rollout_outputs[0].get("error") is not None
+        assert (
+            "InfraError" in rollout_outputs[0]["error"]
+        )  # Error is serialized as repr string
 
 
 class TestEmptyModelResponseErrors:

tests/test_environment_extra.py

@@ -159,7 +159,7 @@ def test_run_rollouts_with_max_concurrent(
             max_concurrent=2,
         )
     )
-    states = outputs["states"]
+    states = outputs["outputs"]
     assert len(states) == 3
 
 
@@ -178,7 +178,7 @@ def test_evaluate_fallback_and_repeat(mock_openai_client, make_dummy_env, make_i
         )
     )
     # Expect n * r rollouts in outputs
-    states = outputs["states"]
+    states = outputs["outputs"]
     assert len(states) == 2 * 2
 
 
@@ -190,7 +190,7 @@ async def test_generate_inside_running_loop(
     inputs = [make_input(example_id=0)]
     # Call the async API directly inside a running event loop to avoid nested sync wrapper issues
     outputs = await env.generate(inputs, client=mock_openai_client, model="test-model")
-    states = outputs["states"]
+    states = outputs["outputs"]
     assert len(states) == 1
     assert states[0].get("completion") is not None
 
@@ -218,7 +218,7 @@ def model_dump(self):
     assert isinstance(sanitized[0]["tool_calls"][0], str)
 
 
-def test_make_dataset_basic_without_tools(make_metadata, make_state):
-    outputs = GenerateOutputs(states=[make_state()], metadata=make_metadata())
-    ds = build_dataset(outputs)
+def test_make_dataset_basic_without_tools(make_metadata, make_output):
+    results = GenerateOutputs(outputs=[make_output()], metadata=make_metadata())
+    ds = build_dataset(results)
     assert len(ds) == 1 and "foo" in ds.column_names

tests/test_eval_cli.py

@@ -9,6 +9,7 @@
 import verifiers.utils.eval_utils
 from verifiers.types import GenerateOutputs
 from verifiers.utils.eval_utils import load_toml_config
+from verifiers.utils.save_utils import states_to_outputs
 
 
 @pytest.fixture
@@ -74,14 +75,15 @@ async def fake_run_evaluation(config, **kwargs):
             r = config.rollouts_per_example
             inputs = [_make_input(example_id=i // r) for i in range(n * r)]
             states = [_make_state(**inputs[i]) for i in range(n * r)]
+            rollout_outputs = states_to_outputs(states)
             metadata = _make_metadata(
                 env_id=config.env_id,
                 model=config.model,
                 sampling_args=config.sampling_args,
                 num_examples=n,
                 rollouts_per_example=r,
             )
-            return GenerateOutputs(states=states, metadata=metadata)
+            return GenerateOutputs(outputs=rollout_outputs, metadata=metadata)
 
         monkeypatch.setattr(
             verifiers.utils.eval_utils, "run_evaluation", fake_run_evaluation

tests/test_eval_utils.py

@@ -5,6 +5,7 @@
 """
 
 from verifiers.types import GenerateOutputs
+from verifiers.utils.save_utils import states_to_outputs
 
 
 def test_print_results_rollout_indexing(capsys, make_metadata, make_state, make_input):
@@ -39,9 +40,10 @@ def test_print_results_rollout_indexing(capsys, make_metadata, make_state, make_
         make_state(**input, reward=reward, metrics={"test_metric": metric_value})
         for input, reward, metric_value in zip(inputs, rewards, metric_values)
     ]
+    rollout_outputs = states_to_outputs(states)
 
-    outputs = GenerateOutputs(states=states, metadata=metadata)
-    print_results(outputs)
+    results = GenerateOutputs(outputs=rollout_outputs, metadata=metadata)
+    print_results(results)
     captured = capsys.readouterr()
 
     # Verify rollout groupings are correct
@@ -72,8 +74,9 @@ def test_print_results_single_rollout(capsys, make_metadata, make_state, make_in
         make_state(**make_input(example_id=example_id), reward=reward)
         for example_id, reward in zip(example_ids, rewards)
     ]
+    rollout_outputs = states_to_outputs(states)
 
-    results = GenerateOutputs(states=states, metadata=metadata)
+    results = GenerateOutputs(outputs=rollout_outputs, metadata=metadata)
 
     print_results(results)
     captured = capsys.readouterr()
@@ -97,11 +100,12 @@ def test_print_results_three_rollouts(capsys, make_metadata, make_state, make_in
     states = [
         make_state(**input, reward=reward) for input, reward in zip(inputs, rewards)
     ]
+    rollout_outputs = states_to_outputs(states)
     metadata = make_metadata(
         num_examples=num_examples, rollouts_per_example=rollouts_per_example
     )
 
-    results = GenerateOutputs(states=states, metadata=metadata)
+    results = GenerateOutputs(outputs=rollout_outputs, metadata=metadata)
 
     print_results(results)
     captured = capsys.readouterr()

tests/test_gym_env.py

@@ -147,8 +147,10 @@ def test_basic_rollout_and_reward_sum(toy_env_class, client):
         num_eval_episodes=1,
     )
 
-    outputs = env.evaluate_sync(client=client, model="mock")
-    st = outputs["states"][0]
+    outputs = env.evaluate_sync(
+        client=client, model="mock", state_columns=["trajectory", "gym_done"]
+    )
+    st = outputs["outputs"][0]
     steps = st.get("trajectory", [])
 
     assert len(steps) > 0
@@ -173,8 +175,10 @@ def bad_parser(_txt: str) -> int:
         num_eval_episodes=1,
     )
 
-    res = env.evaluate_sync(client=client, model="mock")
-    st = res["states"][0]
+    res = env.evaluate_sync(
+        client=client, model="mock", state_columns=["trajectory", "gym_done"]
+    )
+    st = res["outputs"][0]
     steps = st.get("trajectory", [])
 
     assert st.get("gym_done") is True
@@ -200,8 +204,10 @@ def step(self, action: int):
         num_train_episodes=0,
         num_eval_episodes=1,
     )
-    res = env.evaluate_sync(client=client, model="mock")
-    st = res["states"][0]
+    res = env.evaluate_sync(
+        client=client, model="mock", state_columns=["trajectory", "gym_done"]
+    )
+    st = res["outputs"][0]
     steps = st.get("trajectory", [])
 
     assert len(steps) == 3
@@ -229,7 +235,7 @@ def test_system_prompt_and_few_shot(toy_env_class, client):
         num_eval_episodes=1,
     )
     res = env.evaluate_sync(client=client, model="mock")
-    st = res["states"][0]
+    st = res["outputs"][0]
     first_prompt = st["prompt"]
 
     roles = [m["role"] for m in first_prompt]
@@ -256,8 +262,10 @@ def step(self, action: int):
         num_train_episodes=0,
         num_eval_episodes=1,
     )
-    res = env.evaluate_sync(client=client, model="mock")
-    st = res["states"][0]
+    res = env.evaluate_sync(
+        client=client, model="mock", state_columns=["trajectory", "gym_done"]
+    )
+    st = res["outputs"][0]
     steps = st.get("trajectory", [])
 
     assert steps[0]["extras"]["gym_info"] == {"info": "done"}
@@ -275,7 +283,7 @@ def test_env_kwargs_passed_to_env(toy_env_class, client):
         num_eval_episodes=1,
     )
     res = env.evaluate_sync(client=client, model="mock")
-    st = res["states"][0]
+    st = res["outputs"][0]
 
     first_obs_msg = st["prompt"][-1]["content"]
     assert first_obs_msg == "x=5"
@@ -303,7 +311,7 @@ def obs_to_text(self, obs: Any) -> str:
         num_eval_episodes=1,
     )
     res = env.evaluate_sync(client=client, model="mock")
-    st = res["states"][0]
+    st = res["outputs"][0]
     assert st["prompt"][-1]["content"] == "obs_is_0"
 
 
@@ -324,7 +332,7 @@ def test_completion_mode(toy_env_class, client):
     )
 
     res = env.evaluate_sync(client=client, model="mock")
-    st = res["states"][0]
+    st = res["outputs"][0]
 
     assert isinstance(st["prompt"], str)
     assert st["prompt"] == "x=0"