Merge pull request #4199 from tybug/legibility

Error message and warning improvements

hypothesis-python/RELEASE.rst

@@ -0,0 +1,7 @@
+RELEASE_TYPE: patch
+
+This patch improves our error and warning messages.
+
+- Add a warning for ``st.text("ascii")`` - you probably meant ``st.text(st.characters(codec="ascii"))``. Similarly for ``"utf-8"``.
+- Recommend remedies in the error message of ``Unsatisfiable``.
+- When ``@given`` errors because it was given an extra keyword argument, and the keyword matches a setting name like ``max_examples``, recommend ``@settings(max_examples=...)`` instead.

hypothesis-python/src/hypothesis/core.py

@@ -45,6 +45,7 @@
     HealthCheck,
     Phase,
     Verbosity,
+    all_settings,
     local_settings,
     settings as Settings,
 )
@@ -405,9 +406,12 @@ def is_invalid_test(test, original_sig, given_arguments, given_kwargs):
     ]
     if extra_kwargs and (params == [] or params[-1].kind is not params[-1].VAR_KEYWORD):
         arg = extra_kwargs[0]
+        extra = ""
+        if arg in all_settings:
+            extra = f". Did you mean @settings({arg}={given_kwargs[arg]!r})?"
         return invalid(
             f"{test.__name__}() got an unexpected keyword argument {arg!r}, "
-            f"from `{arg}={given_kwargs[arg]!r}` in @given"
+            f"from `{arg}={given_kwargs[arg]!r}` in @given{extra}"
         )
     if any(p.default is not p.empty for p in params):
         return invalid("Cannot apply @given to a function with defaults.")
@@ -1227,8 +1231,33 @@ def run_engine(self):
             )
         else:
             if runner.valid_examples == 0:
+                explanations = []
+                # use a somewhat arbitrary cutoff to avoid recommending spurious
+                # fixes.
+                # eg, a few invalid examples from internal filters when the
+                # problem is the user generating large inputs, or a
+                # few overruns during internal mutation when the problem is
+                # impossible user filters/assumes.
+                if runner.invalid_examples > min(20, runner.call_count // 5):
+                    explanations.append(
+                        f"{runner.invalid_examples} of {runner.call_count} "
+                        "examples failed a .filter() or assume() condition. Try "
+                        "making your filters or assumes less strict, or rewrite "
+                        "using strategy parameters: "
+                        "st.integers().filter(lambda x: x > 0) fails less often "
+                        "(that is, never) when rewritten as st.integers(min_value=1)."
+                    )
+                if runner.overrun_examples > min(20, runner.call_count // 5):
+                    explanations.append(
+                        f"{runner.overrun_examples} of {runner.call_count} "
+                        "examples were too large to finish generating; try "
+                        "reducing the typical size of your inputs?"
+                    )
                 rep = get_pretty_function_description(self.test)
-                raise Unsatisfiable(f"Unable to satisfy assumptions of {rep}")
+                raise Unsatisfiable(
+                    f"Unable to satisfy assumptions of {rep}. "
+                    f"{' Also, '.join(explanations)}"
+                )
 
         # If we have not traced executions, warn about that now (but only when
         # we'd expect to do so reliably, i.e. on CPython>=3.12)

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -228,6 +228,8 @@ def __init__(
         self.call_count: int = 0
         self.misaligned_count: int = 0
         self.valid_examples: int = 0
+        self.invalid_examples = 0
+        self.overrun_examples = 0
         self.random: Random = random or Random(getrandbits(128))
         self.database_key: Optional[bytes] = database_key
         self.ignore_limits: bool = ignore_limits
@@ -567,8 +569,12 @@ def test_function(self, data: ConjectureData) -> None:
                     assert not isinstance(data_as_result, _Overrun)
                     self.best_examples_of_observed_targets[k] = data_as_result
 
-        if data.status == Status.VALID:
+        if data.status is Status.VALID:
             self.valid_examples += 1
+        if data.status is Status.INVALID:
+            self.invalid_examples += 1
+        if data.status is Status.OVERRUN:
+            self.overrun_examples += 1
 
         if data.status == Status.INTERESTING:
             if not self.using_hypothesis_backend:

hypothesis-python/src/hypothesis/strategies/_internal/core.py

@@ -793,6 +793,19 @@ def text(
                 "The following elements in alphabet are not of length one, "
                 f"which leads to violation of size constraints:  {not_one_char!r}"
             )
+        if alphabet in ["ascii", "utf-8"]:
+            warnings.warn(
+                f"st.text({alphabet!r}): it seems like you are trying to use the "
+                f"codec {alphabet!r}. st.text({alphabet!r}) instead generates "
+                f"strings using the literal characters {list(alphabet)!r}. To specify "
+                f"the {alphabet} codec, use st.text(st.characters(codec={alphabet!r})). "
+                "If you intended to use character literals, you can silence this "
+                "warning by reordering the characters.",
+                HypothesisWarning,
+                # this stacklevel is of course incorrect, but breaking out of the
+                # levels of LazyStrategy and validation isn't worthwhile.
+                stacklevel=1,
+            )
         char_strategy = (
             characters(categories=(), include_characters=alphabet)
             if alphabet

hypothesis-python/tests/cover/test_given_error_conditions.py

@@ -11,6 +11,7 @@
 import pytest
 
 from hypothesis import assume, given, reject, settings
+from hypothesis._settings import all_settings
 from hypothesis.errors import InvalidArgument, Unsatisfiable
 from hypothesis.strategies import booleans, integers, nothing
 
@@ -112,3 +113,28 @@ async def foo(x):
         match="Hypothesis doesn't know how to run async test functions",
     ):
         foo()
+
+
+@pytest.mark.parametrize("setting_name", all_settings.keys())
+def test_suggests_at_settings_if_extra_kwarg_matches_setting_name(setting_name):
+    val = 1
+
+    # dynamically create functions with an extra kwarg argument which happens to
+    # match a settings variable. The user probably meant @settings.
+    # exec is pretty cursed here, but it does work.
+    namespace = {}
+    exec(
+        f"""
+@given(a=1, {setting_name}={val})
+def foo(a):
+    pass
+    """,
+        globals(),
+        namespace,
+    )
+
+    with pytest.raises(
+        InvalidArgument,
+        match=rf"Did you mean @settings\({setting_name}={val}\)\?",
+    ):
+        namespace["foo"]()

hypothesis-python/tests/cover/test_testdecorators.py

@@ -12,7 +12,17 @@
 import threading
 from collections import namedtuple
 
-from hypothesis import HealthCheck, Verbosity, assume, given, note, reporting, settings
+from hypothesis import (
+    HealthCheck,
+    Verbosity,
+    assume,
+    given,
+    note,
+    reporting,
+    settings,
+    strategies as st,
+)
+from hypothesis.errors import Unsatisfiable
 from hypothesis.strategies import (
     binary,
     booleans,
@@ -484,3 +494,41 @@ def test_given_usable_inline_on_lambdas():
     given(booleans())(lambda x: xs.append(x))()
     assert len(xs) == 2
     assert set(xs) == {False, True}
+
+
+def test_notes_high_filter_rates_in_unsatisfiable_error():
+    @given(st.integers())
+    @settings(suppress_health_check=[HealthCheck.filter_too_much])
+    def f(v):
+        assume(False)
+
+    with raises(
+        Unsatisfiable,
+        match=(
+            r"Unable to satisfy assumptions of f\. 1000 of 1000 examples "
+            r"failed a \.filter\(\) or assume\(\)"
+        ),
+    ):
+        f()
+
+
+def test_notes_high_overrun_rates_in_unsatisfiable_error():
+    @given(st.binary(min_size=9000))
+    @settings(
+        suppress_health_check=[
+            HealthCheck.data_too_large,
+            HealthCheck.too_slow,
+            HealthCheck.large_base_example,
+        ]
+    )
+    def f(v):
+        pass
+
+    with raises(
+        Unsatisfiable,
+        match=(
+            r"1000 of 1000 examples were too large to finish generating; "
+            r"try reducing the typical size of your inputs\?"
+        ),
+    ):
+        f()

hypothesis-python/tests/cover/test_validation.py

@@ -9,11 +9,12 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import functools
+import warnings
 
 import pytest
 
-from hypothesis import find, given
-from hypothesis.errors import InvalidArgument
+from hypothesis import find, given, strategies as st
+from hypothesis.errors import HypothesisWarning, InvalidArgument
 from hypothesis.internal.validation import check_type
 from hypothesis.strategies import (
     SearchStrategy as ActualSearchStrategy,
@@ -275,3 +276,27 @@ def test_check_strategy_might_suggest_sampled_from():
     with pytest.raises(InvalidArgument, match="such as st.sampled_from"):
         check_strategy_((1, 2, 3))
     check_strategy_(integers(), "passes for our custom coverage check")
+
+
+@pytest.mark.parametrize("codec", ["ascii", "utf-8"])
+def test_warn_on_strings_matching_common_codecs(codec):
+    with pytest.warns(
+        HypothesisWarning,
+        match=f"it seems like you are trying to use the codec {codec!r}",
+    ):
+
+        @given(st.text(codec))
+        def f(s):
+            pass
+
+        f()
+
+    # if we reorder, it doesn't warn anymore
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+
+        @given(st.text(codec[1:] + codec[:1]))
+        def f(s):
+            pass
+
+        f()