diff --git a/HISTORY.rst b/HISTORY.rst index e326df90..bd9ae854 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,6 +4,9 @@ History 1.3.0 (UNRELEASED) ------------------ +* ``cattrs`` now has a benchmark suite to help make and keep cattrs the fastest it can be. The instructions on using it can be found under the `Benchmarking ` section in the docs. + (`#123 `_) + 1.2.0 (2021-01-31) ------------------ diff --git a/Makefile b/Makefile index b9d2c7a4..9da5a71c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean clean-test clean-pyc clean-build docs help +.PHONY: clean clean-test clean-pyc clean-build docs help bench bench-cmp .DEFAULT_GOAL := help define BROWSER_PYSCRIPT import os, webbrowser, sys @@ -51,7 +51,7 @@ lint: ## check style with flake8 flake8 src/cattr tests test: ## run tests quickly with the default Python - pytest -x --ff + pytest -x --ff tests test-all: ## run tests on every Python version with tox @@ -87,3 +87,9 @@ dist: clean ## builds source and wheel package install: clean ## install the package to the active Python's site-packages python setup.py install + +bench-cmp: + pytest bench --benchmark-compare + +bench: + pytest bench --benchmark-save base \ No newline at end of file diff --git a/README.rst b/README.rst index bebf7f03..da3ae27d 100644 --- a/README.rst +++ b/README.rst @@ -169,7 +169,7 @@ characteristic_. ``cattrs`` is tested with Hypothesis_, by David R. MacIver. -``cattrs`` is benchmarked using perf_, by Victor Stinner. +``cattrs`` is benchmarked using perf_ and pytest-benchmark_. This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. @@ -177,6 +177,7 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack .. _characteristic: https://github.com/hynek/characteristic .. _Hypothesis: http://hypothesis.readthedocs.io/en/latest/ .. _perf: https://github.com/haypo/perf +.. _pytest-benchmark: https://pytest-benchmark.readthedocs.io/en/latest/index.html .. _Cookiecutter: https://github.com/audreyr/cookiecutter .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage diff --git a/bench/__init__.py b/bench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bench/bench.py b/bench/bench.py deleted file mode 100644 index 2c66e12b..00000000 --- a/bench/bench.py +++ /dev/null @@ -1,189 +0,0 @@ -from typing import Optional, List -import enum -import attr -import cattr -import cProfile - - -@attr.s(slots=True, frozen=True) -class Lorem: - ipsum = attr.ib() - dolor = attr.ib() - sit = attr.ib() - amet = attr.ib() - consectetur = attr.ib() - adipiscing = attr.ib() - - -@attr.s(slots=True, frozen=True) -class Fugiat: - eiusmod = attr.ib() - tempor = attr.ib() - incididunt = attr.ib() - labore = attr.ib() - dolore = attr.ib() - magna = attr.ib() - aliqua = attr.ib() - veniam = attr.ib() - nostrud = attr.ib() - exercitation = attr.ib() - ullamco = attr.ib() - laboris = attr.ib() - commodo = attr.ib() - consequat = attr.ib() - aute = attr.ib() - - -@attr.s(slots=True, frozen=True) -class Invenire: - irure = attr.ib() - reprehenderit = attr.ib() - voluptate = attr.ib() - velit = attr.ib() - esse = attr.ib() - cillum = attr.ib() - eepcillum = attr.ib() - - -@attr.s(slots=True, frozen=True) -class Tritani: - nulla = attr.ib() - name = attr.ib() - value = attr.ib() - pariatur = attr.ib() - exceptuer = attr.ib() - - -@attr.s(slots=True, frozen=True) -class Laborum: - type = attr.ib() - urangulal = attr.ib() - ipsumal = attr.ib() - occaecat = attr.ib() - cupidatat = attr.ib() - proident = attr.ib() - - -class Aliquip(enum.IntEnum): - Aliquip1 = 1 - Aliquip2 = 2 - Aliquip3 = 3 - Aliquip4 = 4 - Aliquip5 = 5 - - -@attr.s(slots=True, frozen=True) -class Assentior: - aliquip = attr.ib(type=Aliquip) - culpa = attr.ib() - fugiat = attr.ib(type=Optional[Fugiat]) - invenire = attr.ib(type=Optional[Invenire]) - deserunt = attr.ib() - lorem = attr.ib(type=Optional[Lorem]) - mollit = attr.ib() - laborums = attr.ib(type=Optional[List[Laborum]]) - tantas = attr.ib() - nominati = attr.ib() - fabulas = attr.ib() - tritani = attr.ib(type=Optional[Tritani]) - - -@attr.s(slots=True, frozen=True) -class Dignissim: - assentior = attr.ib(type=Assentior) - new_cupidatat = attr.ib() - laoreet = attr.ib() - rationibus = attr.ib() - - -obj = Dignissim( - assentior=Assentior( - aliquip=Aliquip.Aliquip1, - culpa=6, - fugiat=Fugiat( - eiusmod="aaaaaaaaaaaaaaaa", - tempor="bbbbbb", - incididunt="CCCCCCCCCCCCCCCCCCC", - labore="dddddddddd", - dolore="eeeeeeeeee", - magna="fffffffffff", - aliqua="gggggggggggggg", - veniam=None, - nostrud=None, - exercitation=None, - ullamco=None, - laboris=None, - commodo=None, - consequat=None, - aute=None, - ), - invenire=Invenire( - irure=53, - reprehenderit=153, - voluptate=242, - velit=100, - esse=5035, - cillum=53, - eepcillum=422, - ), - deserunt=True, - lorem=Lorem( - ipsum=b"", - dolor=[ - b';\xcd\xe5\xbf\x98\xbc\xd7\x12\xadp\xd9"#g\xdc\x1b;\n\xbc\xbd\x81\x0c\xaay\xe5$\x08\x0e\x8ch', - b"\x9f\xe7\xa2{\xc5(\x1bget\xf3\xb38\xf8\xe4v\x1c\xe3SL:\x04\xb6\xc7k\xef\xfeX\xa0\x18", - b"\xfd\x00\x92\xa5\x9d\xae\x1d\xdc'\xd9\x9d\xb5#w_6{\xb4\xa1\xc0\xfb\xdb\x9b\xc4Ww@\xa4V\x85" - b"\x91fe\xa4\xe0\xcd\xde\xdd\xa6%\x89\x15\xcbT\xc3g\x8bjZ\xfe\xacU\x0c\xc7H\xdc\xdaHk1" - b"\x11a\xbb&", - ], - sit=[ - b"\xc0/\x14\xd2\xfa\x1eGc\x84\xb4\x06\x91\x8c8\x0fS\xd1\xf0\xaa\x97RXd6\xee\xc2\x9d\xc4D/", - b"\t\x0eM\xec\xce\x01%\xd6\rv\x95\x93d\xa9\x02\xac\xcc\x8f\xcav\x8a\x99\xc9\x15\x17\x93Q\xd7\x13\xb3", - b"\xe57\xd9zm\xef8\xda\xe1h\x14\xf9-\x8f\xa9\xbc\x00\xc0\x07)i\xde\xc6;X!+{\xdb4", - b"\x97\xbe(\x89\x9d\xc6\xb9\xf3Z\xfb\x0e\x02+f\xa4\x88\xc5\xfc\xba\xe6\x01\x9f\xb7\x87\xbc\xda\xaa\x83wC", - ], - amet=[ - b'L_;\x12\xf5\xf9\xcc\xae6\x9e\x98$s_\xd9\xca\x92\xfd\xdbs\x83\x04"\x86t+\xbb\xf69g', - b" \xc7\xde\xff\xe3r**\x08?J\x0ba7\x9c\xf3\xaf\x99\xcc6\xe4\xbb\x9a\\\xb5q?ey\x9b", - b"\x84\xe8'\xb7\xd6\xdcR\x135\x00\x96\xa3\xea\xffIc\x9a\xf2\xa7\t\xe2\xb4\x07\x9e\xf49-\"\x1d\xa3", - b"J\n\xf7\xedcB]\r\xb2L\xaf\xbc\x9b\x92\xfe\xb4\x95L\xde\xf3\xe7r\xdf\x16\xbcID\x8f\x07\x91", - b"PDf\x91\x01?)G\x8d\xe1T\r\x1b\x8aL=\xffe\xcc\xa1\xab\x9a\xf8\xdeN^\x06\xdf\xc2\x95", - b"\x8e\x9e\\$\xed\xa2p\x12,=\x8c\x8d\x84J\xe6\xfc\xe1\x88y#\x9a'\xfc\x04\xba\x13\x10\xa3\xf5\xba", - b"\xa9\xc6 \xf3\xee;\x94\xe7\xeb\xb28\x1d\x93\nt\xa5H\x06\xcc\xd3\xf3\x9e%\x93\x89\x9d\xe4]!E", - b"\xef\xfa\x04\xa9 \x8cI\xa7*\x98\xc7+O\xba\x833^\x0fw\x95\x89Y\x932\x1f-\xaa#\x08U", - ], - consectetur=b"\x17\xfe\xf9\x1b\x8a\xc9\xbc\x95\xc4\xdc8\xcb\x9b{\x9eF\x8b\x89\xf8\x07`\x8eo\x11\xc9\x98\x07I\xd2\x1b", - adipiscing=b"wy\xe9\xd9^\x7f<\x14\xae\x86\xf33Y\xcd/\xb4b\x85\x18\xd9~,\xb6@\xd3g\x17\xa4\xf0\xbc", - ), - mollit=4294967294, - laborums=[ - Laborum( - type=23, - urangulal=b"\xd1c\xe0\x1dT\xf1\xde\x8f\xeb\x9d\xfd\xcf\x88\xe0\xcc\xda\x9er\xbdqJ/\xf0\x11\x97\\'&\xa6>", - ipsumal=None, - occaecat=b"\x13\xa9f{dr\x1a/\x15\xbc\xcb/7ax\xc9\x98\xb9\xd8s\xc8%\x9a\xf6wH\xf6\x0bg&", - cupidatat=13, - proident=None, - ), - ], - tantas="sdfsdlxcv49249sdfs90sdf==", - nominati=-1, - fabulas=False, - tritani=None, - ), - new_cupidatat=13, - laoreet=1, - rationibus=False, -) - - -converter = cattr.Converter() - - -def bench(): - unstructured = converter.unstructure_attrs_asdict(obj) - converter.structure_attrs_fromdict(unstructured, obj.__class__) - - -cProfile.run("""for i in range(25000): bench()""", sort="tottime") diff --git a/bench/test_attrs_collections.py b/bench/test_attrs_collections.py new file mode 100644 index 00000000..34f93e64 --- /dev/null +++ b/bench/test_attrs_collections.py @@ -0,0 +1,97 @@ +from enum import IntEnum +from typing import List + +import attr +import pytest + +from cattr import Converter, GenConverter, UnstructureStrategy + + +@pytest.mark.parametrize( + "converter_cls", + [Converter, GenConverter], +) +@pytest.mark.parametrize( + "unstructure_strat", + [UnstructureStrategy.AS_DICT, UnstructureStrategy.AS_TUPLE], +) +def test_unstructure_attrs_lists(benchmark, converter_cls, unstructure_strat): + """ + Benchmark a large (30 attributes) attrs class containing lists of + primitives. + """ + + class E(IntEnum): + ONE = 1 + TWO = 2 + + @attr.define + class C: + a: List[int] + b: List[float] + c: List[str] + d: List[bytes] + e: List[E] + f: List[int] + g: List[float] + h: List[str] + i: List[bytes] + j: List[E] + k: List[int] + l: List[float] + m: List[str] + n: List[bytes] + o: List[E] + p: List[int] + q: List[float] + r: List[str] + s: List[bytes] + t: List[E] + u: List[int] + v: List[float] + w: List[str] + x: List[bytes] + y: List[E] + z: List[int] + aa: List[float] + ab: List[str] + ac: List[bytes] + ad: List[E] + + c = converter_cls(unstruct_strat=unstructure_strat) + + benchmark( + c.unstructure, + C( + [1] * 3, + [1.0] * 3, + ["a small string"] * 3, + ["test".encode()] * 3, + [E.ONE] * 3, + [2] * 3, + [2.0] * 3, + ["a small string"] * 3, + ["test".encode()] * 3, + [E.TWO] * 3, + [3] * 3, + [3.0] * 3, + ["a small string"] * 3, + ["test".encode()] * 3, + [E.ONE] * 3, + [4] * 3, + [4.0] * 3, + ["a small string"] * 3, + ["test".encode()] * 3, + [E.TWO] * 3, + [5] * 3, + [5.0] * 3, + ["a small string"] * 3, + ["test".encode()] * 3, + [E.ONE] * 3, + [6] * 3, + [6.0] * 3, + ["a small string"] * 3, + ["test".encode()] * 3, + [E.TWO] * 3, + ), + ) diff --git a/bench/test_attrs_nested.py b/bench/test_attrs_nested.py new file mode 100644 index 00000000..45188dea --- /dev/null +++ b/bench/test_attrs_nested.py @@ -0,0 +1,70 @@ +"""Benchmark attrs containing other attrs classes.""" +import attr +import pytest + +from cattr import Converter, GenConverter, UnstructureStrategy + + +@pytest.mark.parametrize( + "converter_cls", + [Converter, GenConverter], +) +@pytest.mark.parametrize( + "unstructure_strat", + [UnstructureStrategy.AS_DICT, UnstructureStrategy.AS_TUPLE], +) +def test_unstructure_attrs_nested(benchmark, converter_cls, unstructure_strat): + c = converter_cls(unstruct_strat=unstructure_strat) + + @attr.define + class InnerA: + a: int + b: float + c: str + d: bytes + + @attr.define + class InnerB: + a: int + b: float + c: str + d: bytes + + @attr.define + class InnerC: + a: int + b: float + c: str + d: bytes + + @attr.define + class InnerD: + a: int + b: float + c: str + d: bytes + + @attr.define + class InnerE: + a: int + b: float + c: str + d: bytes + + @attr.define + class Outer: + a: InnerA + b: InnerB + c: InnerC + d: InnerD + e: InnerE + + inst = Outer( + InnerA(1, 1.0, "one", "one".encode()), + InnerB(2, 2.0, "two", "two".encode()), + InnerC(3, 3.0, "three", "three".encode()), + InnerD(4, 4.0, "four", "four".encode()), + InnerE(5, 5.0, "five", "five".encode()), + ) + + benchmark(c.unstructure, inst) diff --git a/bench/test_attrs_primitives.py b/bench/test_attrs_primitives.py new file mode 100644 index 00000000..55fde147 --- /dev/null +++ b/bench/test_attrs_primitives.py @@ -0,0 +1,95 @@ +from enum import IntEnum + +import attr +import pytest + +from cattr import Converter, GenConverter, UnstructureStrategy + + +@pytest.mark.parametrize( + "converter_cls", + [Converter, GenConverter], +) +@pytest.mark.parametrize( + "unstructure_strat", + [UnstructureStrategy.AS_DICT, UnstructureStrategy.AS_TUPLE], +) +def test_unstructure_attrs_primitives( + benchmark, converter_cls, unstructure_strat +): + """Benchmark a large (30 attributes) attrs class containing primitives.""" + + class E(IntEnum): + ONE = 1 + TWO = 2 + + @attr.define + class C: + a: int + b: float + c: str + d: bytes + e: E + f: int + g: float + h: str + i: bytes + j: E + k: int + l: float + m: str + n: bytes + o: E + p: int + q: float + r: str + s: bytes + t: E + u: int + v: float + w: str + x: bytes + y: E + z: int + aa: float + ab: str + ac: bytes + ad: E + + c = converter_cls(unstruct_strat=unstructure_strat) + + benchmark( + c.unstructure, + C( + 1, + 1.0, + "a small string", + "test".encode(), + E.ONE, + 2, + 2.0, + "a small string", + "test".encode(), + E.TWO, + 3, + 3.0, + "a small string", + "test".encode(), + E.ONE, + 4, + 4.0, + "a small string", + "test".encode(), + E.TWO, + 5, + 5.0, + "a small string", + "test".encode(), + E.ONE, + 6, + 6.0, + "a small string", + "test".encode(), + E.TWO, + ), + ) diff --git a/bench/test_primitives.py b/bench/test_primitives.py new file mode 100644 index 00000000..b21c683c --- /dev/null +++ b/bench/test_primitives.py @@ -0,0 +1,17 @@ +import pytest + +from cattr import Converter, GenConverter + + +@pytest.mark.parametrize("converter_cls", [Converter, GenConverter]) +def test_unstructure_int(benchmark, converter_cls): + c = converter_cls() + + benchmark(c.unstructure, 5) + + +@pytest.mark.parametrize("converter_cls", [Converter, GenConverter]) +def test_unstructure_float(benchmark, converter_cls): + c = converter_cls() + + benchmark(c.unstructure, 15.0) diff --git a/docs/benchmarking.rst b/docs/benchmarking.rst new file mode 100644 index 00000000..658a059b --- /dev/null +++ b/docs/benchmarking.rst @@ -0,0 +1,27 @@ +============ +Benchmarking +============ + +cattrs includes a benchmarking suite to help detect performance regressions and +guide performance optimizations. + +The suite is based on pytest and pytest-benchmark. Benchmarks are similar to +tests, with the exception of being stored in the `bench/` directory and being +used to verify performance instead of correctness. + +A Sample Workflow +~~~~~~~~~~~~~~~~~ + +First, ensure the system you're benchmarking on is as stable as possible. For +example, the pyperf library has a `system tune` command that can tweak +CPU frequency governors. You also might want to quit as many applications as +possible and run the benchmark suite on isolated CPU cores (`taskset` can be +used for this purpose on Linux). + +Then, generate a baseline using `make bench`. This will run the benchmark suite +and save it into a file. + +Following that, implement the changes you have in mind. Run the test suite to +ensure correctness. Then, compare the performance of the new code to the saved +baseline using `make bench-cmp`. If the code is still correct but faster, +congratulations! \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 7ad9dbfe..10f9b91d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,6 +19,7 @@ Contents: unstructuring customizing unions + benchmarking contributing history diff --git a/setup.cfg b/setup.cfg index dbc97083..200ba512 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,4 +18,4 @@ source = license_file = LICENSE [tool:pytest] -addopts = -l \ No newline at end of file +addopts = -l --benchmark-sort=fullname --benchmark-warmup=true --benchmark-warmup-iterations=5 --benchmark-group-by=fullname \ No newline at end of file diff --git a/setup.py b/setup.py index fe61d37e..190d7d56 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ "coverage", "Sphinx", "pytest", + "pytest-benchmark", "hypothesis", "pendulum", "isort", diff --git a/tox.ini b/tox.ini index bc86ab0a..8bd018af 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ setenv = extras = dev commands = pip install -U pip - coverage run --source cattr -m pytest + coverage run --source cattr -m pytest tests passenv = CI [testenv:docs]