BREAKING CHANGE: Change type of 'source' field on EvaluationResult (#2388)

Co-authored-by: Douwe Maan <[email protected]>

Author: dmontagu

docs/changelog.md

@@ -12,6 +12,10 @@ Pydantic AI is still pre-version 1, so breaking changes will occur, however:
 !!! note
     Here's a filtered list of the breaking changes for each version to help you upgrade Pydantic AI.
 
+### v0.5.0 (2025-08-04)
+
+See [#2388](https://github.com/pydantic/pydantic-ai/pull/2388) - The `source` field of an `EvaluationResult` is now of type `EvaluatorSpec` rather than the actual source `Evaluator` instance, to help with serialization/deserialization.
+
 ### v0.4.0 (2025-07-08)
 
 See [#1799](https://github.com/pydantic/pydantic-ai/pull/1799) - Pydantic Evals `EvaluationReport` and `ReportCase` are now generic dataclasses instead of Pydantic models. If you were serializing them using `model_dump()`, you will now need to use the `EvaluationReportAdapter` and `ReportCaseAdapter` type adapters instead.

pydantic_evals/pydantic_evals/dataset.py

@@ -38,9 +38,9 @@
 from ._utils import get_unwrapped_function_name, task_group_gather
 from .evaluators import EvaluationResult, Evaluator
 from .evaluators._run_evaluator import run_evaluator
-from .evaluators._spec import EvaluatorSpec
 from .evaluators.common import DEFAULT_EVALUATORS
 from .evaluators.context import EvaluatorContext
+from .evaluators.spec import EvaluatorSpec
 from .otel import SpanTree
 from .otel._context_subtree import context_subtree
 from .reporting import EvaluationReport, ReportCase, ReportCaseAggregate

pydantic_evals/pydantic_evals/evaluators/__init__.py

@@ -10,7 +10,7 @@
     Python,
 )
 from .context import EvaluatorContext
-from .evaluator import EvaluationReason, EvaluationResult, Evaluator, EvaluatorOutput
+from .evaluator import EvaluationReason, EvaluationResult, Evaluator, EvaluatorOutput, EvaluatorSpec
 
 __all__ = (
     # common
@@ -27,7 +27,8 @@
     'EvaluatorContext',
     # evaluator
     'Evaluator',
-    'EvaluationReason',
     'EvaluatorOutput',
+    'EvaluatorSpec',
+    'EvaluationReason',
     'EvaluationResult',
 )

pydantic_evals/pydantic_evals/evaluators/_run_evaluator.py

@@ -48,7 +48,9 @@ async def run_evaluator(
     for name, result in results.items():
         if not isinstance(result, EvaluationReason):
             result = EvaluationReason(value=result)
-        details.append(EvaluationResult(name=name, value=result.value, reason=result.reason, source=evaluator))
+        details.append(
+            EvaluationResult(name=name, value=result.value, reason=result.reason, source=evaluator.as_spec())
+        )
 
     return details
 

pydantic_evals/pydantic_evals/evaluators/evaluator.py

@@ -17,15 +17,16 @@
 from pydantic_ai import _utils
 
 from .._utils import get_event_loop
-from ._spec import EvaluatorSpec
 from .context import EvaluatorContext
+from .spec import EvaluatorSpec
 
 __all__ = (
     'EvaluationReason',
     'EvaluationResult',
     'EvaluationScalar',
     'Evaluator',
     'EvaluatorOutput',
+    'EvaluatorSpec',
 )
 
 EvaluationScalar = Union[bool, int, float, str]
@@ -71,13 +72,13 @@ class EvaluationResult(Generic[EvaluationScalarT]):
         name: The name of the evaluation.
         value: The scalar result of the evaluation.
         reason: An optional explanation of the evaluation result.
-        source: The evaluator that produced this result.
+        source: The spec of the evaluator that produced this result.
     """
 
     name: str
     value: EvaluationScalarT
     reason: str | None
-    source: Evaluator
+    source: EvaluatorSpec
 
     def downcast(self, *value_types: type[T]) -> EvaluationResult[T] | None:
         """Attempt to downcast this result to a more specific type.
@@ -246,6 +247,13 @@ def serialize(self, info: SerializationInfo) -> Any:
         Returns:
             A JSON-serializable representation of this evaluator as an EvaluatorSpec.
         """
+        return to_jsonable_python(
+            self.as_spec(),
+            context=info.context,
+            serialize_unknown=True,
+        )
+
+    def as_spec(self) -> EvaluatorSpec:
         raw_arguments = self.build_serialization_arguments()
 
         arguments: None | tuple[Any,] | dict[str, Any]
@@ -255,11 +263,8 @@ def serialize(self, info: SerializationInfo) -> Any:
             arguments = (next(iter(raw_arguments.values())),)
         else:
             arguments = raw_arguments
-        return to_jsonable_python(
-            EvaluatorSpec(name=self.get_serialization_name(), arguments=arguments),
-            context=info.context,
-            serialize_unknown=True,
-        )
+
+        return EvaluatorSpec(name=self.get_serialization_name(), arguments=arguments)
 
     def build_serialization_arguments(self) -> dict[str, Any]:
         """Build the arguments for serialization.

pydantic_evals/pydantic_evals/evaluators/_spec.py renamed to pydantic_evals/pydantic_evals/evaluators/spec.py

@@ -30,12 +30,6 @@ class EvaluatorSpec(BaseModel):
     * `'MyEvaluator'` - Just the (string) name of the Evaluator subclass is used if its `__init__` takes no arguments
     * `{'MyEvaluator': first_arg}` - A single argument is passed as the first positional argument to `MyEvaluator.__init__`
     * `{'MyEvaluator': {k1: v1, k2: v2}}` - Multiple kwargs are passed to `MyEvaluator.__init__`
-
-    Args:
-        name: The serialization name of the evaluator class returned by `EvaluatorClass.get_serialization_name()`;
-            this is usually just the class name itself.
-        arguments: The arguments to pass to the evaluator's constructor. Can be None (for no arguments),
-            a tuple (for a single positional argument), or a dict (for multiple keyword arguments).
     """
 
     name: str

pydantic_evals/pydantic_evals/reporting/__init__.py

@@ -669,7 +669,11 @@ def build_diff_row(
             row.append(scores_diff)
 
         if self.include_labels:  # pragma: no branch
-            labels_diff = self._render_dicts_diff(baseline.labels, new_case.labels, self.label_renderers)
+            labels_diff = self._render_dicts_diff(
+                {k: v.value for k, v in baseline.labels.items()},
+                {k: v.value for k, v in new_case.labels.items()},
+                self.label_renderers,
+            )
             row.append(labels_diff)
 
         if self.include_metrics:  # pragma: no branch

tests/evals/test_dataset.py

@@ -9,7 +9,7 @@
 import pytest
 from dirty_equals import HasRepr, IsNumber
 from inline_snapshot import snapshot
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 
 from ..conftest import IsStr, try_import
 from .utils import render_table
@@ -20,7 +20,7 @@
 
     from pydantic_evals import Case, Dataset
     from pydantic_evals.dataset import increment_eval_metric, set_eval_attribute
-    from pydantic_evals.evaluators import EvaluationResult, Evaluator, EvaluatorOutput, LLMJudge, Python
+    from pydantic_evals.evaluators import EvaluationResult, Evaluator, EvaluatorOutput, EvaluatorSpec, LLMJudge, Python
     from pydantic_evals.evaluators.context import EvaluatorContext
 
     @dataclass
@@ -32,7 +32,7 @@ class MockEvaluator(Evaluator[object, object, object]):
         def evaluate(self, ctx: EvaluatorContext[object, object, object]) -> EvaluatorOutput:
             return self.output
 
-    from pydantic_evals.reporting import ReportCase, ReportCaseAdapter
+    from pydantic_evals.reporting import EvaluationReport, ReportCase, ReportCaseAdapter
 
 pytestmark = [pytest.mark.skipif(not imports_successful(), reason='pydantic-evals not installed'), pytest.mark.anyio]
 
@@ -456,13 +456,13 @@ async def my_task(inputs: TaskInput) -> TaskOutput:
                 scores={},
                 labels={
                     'output': EvaluationResult(
-                        name='output', value='a', reason=None, source=MockEvaluator(output={'output': 'a'})
+                        name='output', value='a', reason=None, source=MockEvaluator(output={'output': 'a'}).as_spec()
                     ),
                     'output_2': EvaluationResult(
-                        name='output', value='b', reason=None, source=MockEvaluator(output={'output': 'b'})
+                        name='output', value='b', reason=None, source=MockEvaluator(output={'output': 'b'}).as_spec()
                     ),
                     'output_3': EvaluationResult(
-                        name='output', value='c', reason=None, source=MockEvaluator(output={'output': 'c'})
+                        name='output', value='c', reason=None, source=MockEvaluator(output={'output': 'c'}).as_spec()
                     ),
                 },
                 assertions={},
@@ -482,13 +482,13 @@ async def my_task(inputs: TaskInput) -> TaskOutput:
                 scores={},
                 labels={
                     'output': EvaluationResult(
-                        name='output', value='a', reason=None, source=MockEvaluator(output={'output': 'a'})
+                        name='output', value='a', reason=None, source=MockEvaluator(output={'output': 'a'}).as_spec()
                     ),
                     'output_2': EvaluationResult(
-                        name='output', value='b', reason=None, source=MockEvaluator(output={'output': 'b'})
+                        name='output', value='b', reason=None, source=MockEvaluator(output={'output': 'b'}).as_spec()
                     ),
                     'output_3': EvaluationResult(
-                        name='output', value='c', reason=None, source=MockEvaluator(output={'output': 'c'})
+                        name='output', value='c', reason=None, source=MockEvaluator(output={'output': 'c'}).as_spec()
                     ),
                 },
                 assertions={},
@@ -501,6 +501,73 @@ async def my_task(inputs: TaskInput) -> TaskOutput:
     )
 
 
+async def test_report_round_trip_serialization(example_dataset: Dataset[TaskInput, TaskOutput, TaskMetadata]):
+    """Test the increment_eval_metric function."""
+
+    async def my_task(inputs: TaskInput) -> TaskOutput:
+        return TaskOutput(answer=f'answer to {inputs.query}')
+
+    example_dataset.add_evaluator(MockEvaluator({'output': 'a'}))
+
+    report = await example_dataset.evaluate(my_task)
+    assert report == snapshot(
+        EvaluationReport(
+            name='my_task',
+            cases=[
+                ReportCase(
+                    name='case1',
+                    inputs=TaskInput(query='What is 2+2?'),
+                    metadata=TaskMetadata(difficulty='easy', category='general'),
+                    expected_output=TaskOutput(answer='4', confidence=1.0),
+                    output=TaskOutput(answer='answer to What is 2+2?', confidence=1.0),
+                    metrics={},
+                    attributes={},
+                    scores={},
+                    labels={
+                        'output': EvaluationResult(
+                            name='output',
+                            value='a',
+                            reason=None,
+                            source=EvaluatorSpec(name='MockEvaluator', arguments=({'output': 'a'},)),
+                        )
+                    },
+                    assertions={},
+                    task_duration=1.0,
+                    total_duration=6.0,
+                    trace_id='00000000000000000000000000000001',
+                    span_id='0000000000000003',
+                ),
+                ReportCase(
+                    name='case2',
+                    inputs=TaskInput(query='What is the capital of France?'),
+                    metadata=TaskMetadata(difficulty='medium', category='geography'),
+                    expected_output=TaskOutput(answer='Paris', confidence=1.0),
+                    output=TaskOutput(answer='answer to What is the capital of France?', confidence=1.0),
+                    metrics={},
+                    attributes={},
+                    scores={},
+                    labels={
+                        'output': EvaluationResult(
+                            name='output',
+                            value='a',
+                            reason=None,
+                            source=EvaluatorSpec(name='MockEvaluator', arguments=({'output': 'a'},)),
+                        )
+                    },
+                    assertions={},
+                    task_duration=1.0,
+                    total_duration=4.0,
+                    trace_id='00000000000000000000000000000001',
+                    span_id='0000000000000007',
+                ),
+            ],
+        )
+    )
+
+    report_adapter = TypeAdapter(EvaluationReport[TaskInput, TaskOutput, TaskMetadata])
+    assert report == report_adapter.validate_json(report_adapter.dump_json(report, indent=2))
+
+
 async def test_genai_attribute_collection(example_dataset: Dataset[TaskInput, TaskOutput, TaskMetadata]):
     async def my_task(inputs: TaskInput) -> TaskOutput:
         with logfire.span(

tests/evals/test_evaluator_base.py

@@ -52,11 +52,11 @@ def evaluate(self, ctx: EvaluatorContext) -> bool:
     evaluator = DummyEvaluator()
 
     # Test basic result
-    result = EvaluationResult(name='test', value=True, reason='Success', source=evaluator)
+    result = EvaluationResult(name='test', value=True, reason='Success', source=evaluator.as_spec())
     assert result.name == 'test'
     assert result.value is True
     assert result.reason == 'Success'
-    assert result.source == evaluator
+    assert result.source == evaluator.as_spec()
 
     # Test downcast with matching type
     downcast = result.downcast(bool)

tests/evals/test_evaluator_spec.py

@@ -6,7 +6,7 @@
 from ..conftest import try_import
 
 with try_import() as imports_successful:
-    from pydantic_evals.evaluators._spec import (
+    from pydantic_evals.evaluators.spec import (
         EvaluatorSpec,
         _SerializedEvaluatorSpec,  # pyright: ignore[reportPrivateUsage]
     )