diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..d5f141b11a --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,7 @@ +RELEASE_TYPE: patch + +This patch improves our error and warning messages. + +- Add a warning for ``st.text("ascii")`` - you probably meant ``st.text(st.characters(codec="ascii"))``. Similarly for ``"utf-8"``. +- Recommend remedies in the error message of ``Unsatisfiable``. +- When ``@given`` errors because it was given an extra keyword argument, and the keyword matches a setting name like ``max_examples``, recommend ``@settings(max_examples=...)`` instead. diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 34a274a6ea..cfb569e4c8 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -45,6 +45,7 @@ HealthCheck, Phase, Verbosity, + all_settings, local_settings, settings as Settings, ) @@ -405,9 +406,12 @@ def is_invalid_test(test, original_sig, given_arguments, given_kwargs): ] if extra_kwargs and (params == [] or params[-1].kind is not params[-1].VAR_KEYWORD): arg = extra_kwargs[0] + extra = "" + if arg in all_settings: + extra = f". Did you mean @settings({arg}={given_kwargs[arg]!r})?" return invalid( f"{test.__name__}() got an unexpected keyword argument {arg!r}, " - f"from `{arg}={given_kwargs[arg]!r}` in @given" + f"from `{arg}={given_kwargs[arg]!r}` in @given{extra}" ) if any(p.default is not p.empty for p in params): return invalid("Cannot apply @given to a function with defaults.") @@ -1227,8 +1231,33 @@ def run_engine(self): ) else: if runner.valid_examples == 0: + explanations = [] + # use a somewhat arbitrary cutoff to avoid recommending spurious + # fixes. + # eg, a few invalid examples from internal filters when the + # problem is the user generating large inputs, or a + # few overruns during internal mutation when the problem is + # impossible user filters/assumes. + if runner.invalid_examples > min(20, runner.call_count // 5): + explanations.append( + f"{runner.invalid_examples} of {runner.call_count} " + "examples failed a .filter() or assume() condition. Try " + "making your filters or assumes less strict, or rewrite " + "using strategy parameters: " + "st.integers().filter(lambda x: x > 0) fails less often " + "(that is, never) when rewritten as st.integers(min_value=1)." + ) + if runner.overrun_examples > min(20, runner.call_count // 5): + explanations.append( + f"{runner.overrun_examples} of {runner.call_count} " + "examples were too large to finish generating; try " + "reducing the typical size of your inputs?" + ) rep = get_pretty_function_description(self.test) - raise Unsatisfiable(f"Unable to satisfy assumptions of {rep}") + raise Unsatisfiable( + f"Unable to satisfy assumptions of {rep}. " + f"{' Also, '.join(explanations)}" + ) # If we have not traced executions, warn about that now (but only when # we'd expect to do so reliably, i.e. on CPython>=3.12) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index a5c388683b..894a8b67ee 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -228,6 +228,8 @@ def __init__( self.call_count: int = 0 self.misaligned_count: int = 0 self.valid_examples: int = 0 + self.invalid_examples = 0 + self.overrun_examples = 0 self.random: Random = random or Random(getrandbits(128)) self.database_key: Optional[bytes] = database_key self.ignore_limits: bool = ignore_limits @@ -567,8 +569,12 @@ def test_function(self, data: ConjectureData) -> None: assert not isinstance(data_as_result, _Overrun) self.best_examples_of_observed_targets[k] = data_as_result - if data.status == Status.VALID: + if data.status is Status.VALID: self.valid_examples += 1 + if data.status is Status.INVALID: + self.invalid_examples += 1 + if data.status is Status.OVERRUN: + self.overrun_examples += 1 if data.status == Status.INTERESTING: if not self.using_hypothesis_backend: diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py index 671ee955c8..6133231ba6 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py @@ -793,6 +793,19 @@ def text( "The following elements in alphabet are not of length one, " f"which leads to violation of size constraints: {not_one_char!r}" ) + if alphabet in ["ascii", "utf-8"]: + warnings.warn( + f"st.text({alphabet!r}): it seems like you are trying to use the " + f"codec {alphabet!r}. st.text({alphabet!r}) instead generates " + f"strings using the literal characters {list(alphabet)!r}. To specify " + f"the {alphabet} codec, use st.text(st.characters(codec={alphabet!r})). " + "If you intended to use character literals, you can silence this " + "warning by reordering the characters.", + HypothesisWarning, + # this stacklevel is of course incorrect, but breaking out of the + # levels of LazyStrategy and validation isn't worthwhile. + stacklevel=1, + ) char_strategy = ( characters(categories=(), include_characters=alphabet) if alphabet diff --git a/hypothesis-python/tests/cover/test_given_error_conditions.py b/hypothesis-python/tests/cover/test_given_error_conditions.py index e417a6fc1a..fe1c3ff396 100644 --- a/hypothesis-python/tests/cover/test_given_error_conditions.py +++ b/hypothesis-python/tests/cover/test_given_error_conditions.py @@ -11,6 +11,7 @@ import pytest from hypothesis import assume, given, reject, settings +from hypothesis._settings import all_settings from hypothesis.errors import InvalidArgument, Unsatisfiable from hypothesis.strategies import booleans, integers, nothing @@ -112,3 +113,28 @@ async def foo(x): match="Hypothesis doesn't know how to run async test functions", ): foo() + + +@pytest.mark.parametrize("setting_name", all_settings.keys()) +def test_suggests_at_settings_if_extra_kwarg_matches_setting_name(setting_name): + val = 1 + + # dynamically create functions with an extra kwarg argument which happens to + # match a settings variable. The user probably meant @settings. + # exec is pretty cursed here, but it does work. + namespace = {} + exec( + f""" +@given(a=1, {setting_name}={val}) +def foo(a): + pass + """, + globals(), + namespace, + ) + + with pytest.raises( + InvalidArgument, + match=rf"Did you mean @settings\({setting_name}={val}\)\?", + ): + namespace["foo"]() diff --git a/hypothesis-python/tests/cover/test_testdecorators.py b/hypothesis-python/tests/cover/test_testdecorators.py index 7597bb164e..106650e002 100644 --- a/hypothesis-python/tests/cover/test_testdecorators.py +++ b/hypothesis-python/tests/cover/test_testdecorators.py @@ -12,7 +12,17 @@ import threading from collections import namedtuple -from hypothesis import HealthCheck, Verbosity, assume, given, note, reporting, settings +from hypothesis import ( + HealthCheck, + Verbosity, + assume, + given, + note, + reporting, + settings, + strategies as st, +) +from hypothesis.errors import Unsatisfiable from hypothesis.strategies import ( binary, booleans, @@ -484,3 +494,41 @@ def test_given_usable_inline_on_lambdas(): given(booleans())(lambda x: xs.append(x))() assert len(xs) == 2 assert set(xs) == {False, True} + + +def test_notes_high_filter_rates_in_unsatisfiable_error(): + @given(st.integers()) + @settings(suppress_health_check=[HealthCheck.filter_too_much]) + def f(v): + assume(False) + + with raises( + Unsatisfiable, + match=( + r"Unable to satisfy assumptions of f\. 1000 of 1000 examples " + r"failed a \.filter\(\) or assume\(\)" + ), + ): + f() + + +def test_notes_high_overrun_rates_in_unsatisfiable_error(): + @given(st.binary(min_size=9000)) + @settings( + suppress_health_check=[ + HealthCheck.data_too_large, + HealthCheck.too_slow, + HealthCheck.large_base_example, + ] + ) + def f(v): + pass + + with raises( + Unsatisfiable, + match=( + r"1000 of 1000 examples were too large to finish generating; " + r"try reducing the typical size of your inputs\?" + ), + ): + f() diff --git a/hypothesis-python/tests/cover/test_validation.py b/hypothesis-python/tests/cover/test_validation.py index 831c199b8e..883dc9791f 100644 --- a/hypothesis-python/tests/cover/test_validation.py +++ b/hypothesis-python/tests/cover/test_validation.py @@ -9,11 +9,12 @@ # obtain one at https://mozilla.org/MPL/2.0/. import functools +import warnings import pytest -from hypothesis import find, given -from hypothesis.errors import InvalidArgument +from hypothesis import find, given, strategies as st +from hypothesis.errors import HypothesisWarning, InvalidArgument from hypothesis.internal.validation import check_type from hypothesis.strategies import ( SearchStrategy as ActualSearchStrategy, @@ -275,3 +276,27 @@ def test_check_strategy_might_suggest_sampled_from(): with pytest.raises(InvalidArgument, match="such as st.sampled_from"): check_strategy_((1, 2, 3)) check_strategy_(integers(), "passes for our custom coverage check") + + +@pytest.mark.parametrize("codec", ["ascii", "utf-8"]) +def test_warn_on_strings_matching_common_codecs(codec): + with pytest.warns( + HypothesisWarning, + match=f"it seems like you are trying to use the codec {codec!r}", + ): + + @given(st.text(codec)) + def f(s): + pass + + f() + + # if we reorder, it doesn't warn anymore + with warnings.catch_warnings(): + warnings.simplefilter("error") + + @given(st.text(codec[1:] + codec[:1])) + def f(s): + pass + + f()