From 60d3fd50f163ba220ce19b951032310a0647fd44 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 1 Nov 2019 23:56:59 +0100 Subject: [PATCH 01/29] configuration for cosmic-ray --- cosmic-ray.sh | 9 +++++++++ cosmic-ray.toml | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 cosmic-ray.sh create mode 100644 cosmic-ray.toml diff --git a/cosmic-ray.sh b/cosmic-ray.sh new file mode 100644 index 00000000..43684650 --- /dev/null +++ b/cosmic-ray.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +cosmic-ray init cosmic-ray.toml session.sqlite +cosmic-ray baseline --report session.sqlite +cr-report --show-output session.baseline.sqlite +cosmic-ray exec session.sqlite +cr-report session.sqlite +cr-html session.sqlite > session.html diff --git a/cosmic-ray.toml b/cosmic-ray.toml new file mode 100644 index 00000000..b3ee9e1f --- /dev/null +++ b/cosmic-ray.toml @@ -0,0 +1,18 @@ +[cosmic-ray] +module-path = "src" +python-version = "" +timeout = 120.0 +exclude-modules = ['src/ecdsa/_version.py', 'src/ecdsa/test*'] +test-command = "pytest -x src/" + +[cosmic-ray.execution-engine] +name = "local" + +[cosmic-ray.cloning] +method = "copy" +commands = [] + +[cosmic-ray.interceptors] +enabled = [ "spor", "pragma_no_mutate", "operators-filter",] + +[cosmic-ray.operators-filter] From aa7e7b36b9eedecbbc0698e80ea9d59882e95bed Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sat, 2 Nov 2019 19:24:22 +0100 Subject: [PATCH 02/29] add quick mutation testing to travis --- .travis.yml | 37 +++++++++++++++++++++++++++++++------ cosmic-ray.sh | 1 + cosmic-ray.toml | 2 +- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index f31a7197..009656c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,13 @@ addons: - libmpc-dev before_cache: - rm -f $HOME/.cache/pip/log/debug.log -# place the slowest (instrumental and py2.6) first +# place the slowest (instrumental, mutation and py2.6) first matrix: include: + - python: 3.9 + dist: bionic + sudo: true + env: MUTATION=yes - python: 2.7 env: INSTRUMENTAL=yes dist: bionic @@ -125,15 +129,16 @@ install: else travis_retry pip install -r build-requirements.txt; fi - - if [[ $TOX_ENV =~ gmpy2 ]] || [[ $INSTRUMENTAL ]]; then travis_retry pip install gmpy2; fi + - if [[ $TOX_ENV =~ gmpy2 ]] || [[ $INSTRUMENTAL ]] || [[ $MUTATION ]]; then travis_retry pip install gmpy2; fi - if [[ $TOX_ENV =~ gmpyp ]]; then travis_retry pip install gmpy; fi - if [[ $INSTRUMENTAL ]]; then travis_retry pip install instrumental; fi + - if [[ $MUTATION ]]; then travis_retry pip install cosmic-ray; fi - pip list script: - if [[ $TOX_ENV ]]; then tox -e $TOX_ENV; fi - - if [[ $TOX_ENV =~ gmpy2 ]]; then tox -e speedgmpy2; fi - - if [[ $TOX_ENV =~ gmpyp ]]; then tox -e speedgmpy; fi - - if ! [[ $TOX_ENV =~ gmpy ]]; then tox -e speed; fi + - if [[ $TOX_ENV =~ gmpy2 ]] && [[ -z $MUTATION ]]; then tox -e speedgmpy2; fi + - if [[ $TOX_ENV =~ gmpyp ]] && [[ -z $MUTATION ]]; then tox -e speedgmpy; fi + - if ! [[ $TOX_ENV =~ gmpy ]] && [[ -z $MUTATION ]]; then tox -e speed; fi - | if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then git checkout $PR_FIRST^ @@ -155,6 +160,26 @@ script: if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then instrumental -f .instrumental.cov -s | python diff-instrumental.py --read .diff-instrumental --fail-under 70 --max-difference -0.1 fi + # cosmic-ray (mutation testing) runs + - if [[ $MUTATION ]]; then cosmic-ray init cosmic-ray.toml session.sqlite; fi + - if [[ $MUTATION ]]; then cosmic-ray baseline --report session.sqlite; fi + - if [[ $MUTATION ]]; then cr-report --show-output session.baseline.sqlite; fi + - | + if [[ $MUTATION ]]; then + cosmic-ray exec session.sqlite & + COSMIC_PID=$! + # make sure that we output something every 5 minutes (otherwise travis will kill us) + while kill -s 0 $COSMIC_PID; do + sleep 300 + cr-report session.sqlite | tail -n 3; + done & + REPORT_PID=$! + # kill exec after 25 minutes + (sleep $((60*25)); kill $COSMIC_PID) & + fi + - if [[ $MUTATION ]]; then wait $COSMIC_PID ; kill $REPORT_PID ; true; fi + - if [[ $MUTATION ]]; then cr-report --show-output session.sqlite | tail -n 40; fi + - if [[ $MUTATION ]]; then cr-rate --estimate --fail-over 25 --confidence 99.9 session.sqlite; fi after_success: - - if [[ -z $INSTRUMENTAL ]]; then coveralls; fi + - if [[ -z $INSTRUMENTAL && -z $MUTATION ]]; then coveralls; fi diff --git a/cosmic-ray.sh b/cosmic-ray.sh index 43684650..ba512d50 100644 --- a/cosmic-ray.sh +++ b/cosmic-ray.sh @@ -7,3 +7,4 @@ cr-report --show-output session.baseline.sqlite cosmic-ray exec session.sqlite cr-report session.sqlite cr-html session.sqlite > session.html +cr-rate --estimate --fail-over 25 --confidence 99.9 session.sqlite diff --git a/cosmic-ray.toml b/cosmic-ray.toml index b3ee9e1f..c9a234e0 100644 --- a/cosmic-ray.toml +++ b/cosmic-ray.toml @@ -1,7 +1,7 @@ [cosmic-ray] module-path = "src" python-version = "" -timeout = 120.0 +timeout = 30.0 exclude-modules = ['src/ecdsa/_version.py', 'src/ecdsa/test*'] test-command = "pytest -x src/" From 880d3ccfc4afa1b47841b10f8cbc54e01968d2c6 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 12 Nov 2020 01:33:07 +0100 Subject: [PATCH 03/29] add --fast option to run hypothesis tests faster --- conftest.py | 4 ++++ cosmic-ray.toml | 2 +- src/ecdsa/test_der.py | 11 ++++++++++- src/ecdsa/test_malformed_sigs.py | 13 +++++++++---- src/ecdsa/test_numbertheory.py | 14 ++++++++++++-- src/ecdsa/test_pyecdsa.py | 11 ++++++++++- 6 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..13387580 --- /dev/null +++ b/conftest.py @@ -0,0 +1,4 @@ +def pytest_addoption(parser): + parser.addoption( + "--fast", action="store_true", default=False, help="run tests fast" + ) diff --git a/cosmic-ray.toml b/cosmic-ray.toml index c9a234e0..f5de84bf 100644 --- a/cosmic-ray.toml +++ b/cosmic-ray.toml @@ -3,7 +3,7 @@ module-path = "src" python-version = "" timeout = 30.0 exclude-modules = ['src/ecdsa/_version.py', 'src/ecdsa/test*'] -test-command = "pytest -x src/" +test-command = "pytest -x --fast src/" [cosmic-ray.execution-engine] name = "local" diff --git a/src/ecdsa/test_der.py b/src/ecdsa/test_der.py index 833d12d8..f450baf8 100644 --- a/src/ecdsa/test_der.py +++ b/src/ecdsa/test_der.py @@ -7,9 +7,10 @@ import unittest2 as unittest except ImportError: import unittest +import sys from six import b import hypothesis.strategies as st -from hypothesis import given +from hypothesis import given, settings import pytest from ._compat import str_idx_as_int from .curves import NIST256p, NIST224p @@ -462,6 +463,14 @@ def st_oid(draw, max_value=2**512, max_size=50): return (first, second) + tuple(rest) +HYP_SETTINGS = {} + + +if "--fast" in sys.argv: + HYP_SETTINGS["max_examples"] = 2 + + +@settings(**HYP_SETTINGS) @given(st_oid()) def test_oids(ids): encoded_oid = encode_oid(*ids) diff --git a/src/ecdsa/test_malformed_sigs.py b/src/ecdsa/test_malformed_sigs.py index 20912376..2a318a44 100644 --- a/src/ecdsa/test_malformed_sigs.py +++ b/src/ecdsa/test_malformed_sigs.py @@ -154,12 +154,17 @@ def st_fuzzed_sig(draw, keys_and_sigs): HealthCheck.filter_too_much, HealthCheck.too_slow, ] +if "--fast" in sys.argv: + params["max_examples"] = 20 slow_params = dict(params) -slow_params["max_examples"] = 10 +if "--fast" in sys.argv: + slow_params["max_examples"] = 2 +else: + slow_params["max_examples"] = 10 -@settings(**params) +@settings(**slow_params) @given(st_fuzzed_sig(keys_and_sigs)) def test_fuzzed_der_signatures(args): verifying_key, sig = args @@ -303,7 +308,7 @@ def st_der(): ) -@settings(**params) +@settings(**slow_params) @given(st.sampled_from(keys_and_sigs), st_der()) def test_random_der_as_signature(params, der): """Check if random DER structures are rejected as signature""" @@ -360,7 +365,7 @@ def test_random_bytes_as_signature(params, der): ] -@settings(**params) +@settings(**slow_params) @given(st_fuzzed_sig(keys_and_string_sigs)) def test_fuzzed_string_signatures(params): verifying_key, sig = params diff --git a/src/ecdsa/test_numbertheory.py b/src/ecdsa/test_numbertheory.py index 239eaa8c..dcaa6d92 100644 --- a/src/ecdsa/test_numbertheory.py +++ b/src/ecdsa/test_numbertheory.py @@ -1,5 +1,6 @@ import operator from functools import reduce +import sys try: import unittest2 as unittest @@ -251,9 +252,15 @@ def st_comp_no_com_fac(draw): # the factorization() sometimes takes a long time to finish HYP_SETTINGS["deadline"] = 5000 +if "--fast" in sys.argv: + HYP_SETTINGS["max_examples"] = 20 + HYP_SLOW_SETTINGS = dict(HYP_SETTINGS) -HYP_SLOW_SETTINGS["max_examples"] = 10 +if "--fast" in sys.argv: + HYP_SLOW_SETTINGS["max_examples"] = 2 +else: + HYP_SLOW_SETTINGS["max_examples"] = 20 class TestIsPrime(unittest.TestCase): @@ -320,6 +327,7 @@ def test_gcd_with_uncom_factor(self, numbers): n = gcd(numbers) assert n == 1 + @settings(**HYP_SLOW_SETTINGS) @given( st.lists( st.integers(min_value=1, max_value=2**8192), @@ -338,6 +346,7 @@ def test_lcm(self): assert lcm([3, 5 * 3, 7 * 3]) == 3 * 5 * 7 assert lcm(3) == 3 + @settings(**HYP_SLOW_SETTINGS) @given( st.lists( st.integers(min_value=1, max_value=2**8192), @@ -356,7 +365,7 @@ def test_lcm_with_random_numbers(self, numbers): "meet requirements (like `is_prime()`), the test " "case times-out on it", ) - @settings(**HYP_SETTINGS) + @settings(**HYP_SLOW_SETTINGS) @given(st_num_square_prime()) def test_square_root_mod_prime(self, vals): square, prime = vals @@ -420,6 +429,7 @@ def test_jacobi(self, mod): c *= jacobi(a, i[0]) ** i[1] assert c == jacobi(a, mod) + @settings(**HYP_SETTINGS) @given(st_two_nums_rel_prime()) def test_inverse_mod(self, nums): num, mod = nums diff --git a/src/ecdsa/test_pyecdsa.py b/src/ecdsa/test_pyecdsa.py index 3ce3e04f..1a428478 100644 --- a/src/ecdsa/test_pyecdsa.py +++ b/src/ecdsa/test_pyecdsa.py @@ -9,11 +9,12 @@ import shutil import subprocess import pytest +import sys from binascii import hexlify, unhexlify import hashlib from functools import partial -from hypothesis import given +from hypothesis import given, settings import hypothesis.strategies as st from six import b, print_, binary_type @@ -75,6 +76,13 @@ class SubprocessError(Exception): pass +HYP_SETTINGS = {} + + +if "--fast" in sys.argv: + HYP_SETTINGS["max_examples"] = 20 + + def run_openssl(cmd): OPENSSL = "openssl" p = subprocess.Popen( @@ -1811,6 +1819,7 @@ def test_trytryagain_single(self): else: self.assertEqual(n, 18) + @settings(**HYP_SETTINGS) @given(st.integers(min_value=0, max_value=10**200)) def test_randrange(self, i): # util.randrange does not provide long-term stability: we might From f433d49df7ecc5f2ab2365eff3ab390cbe36d7dc Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 12 Nov 2020 01:43:50 +0100 Subject: [PATCH 04/29] add mark @slow to allow skipping slowest tests as calling openssl is very slow, allow skipping tests that do that --- conftest.py | 7 +++++++ cosmic-ray.toml | 2 +- src/ecdsa/test_ecdh.py | 1 + src/ecdsa/test_jacobi.py | 2 +- src/ecdsa/test_pyecdsa.py | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 13387580..6c163acf 100644 --- a/conftest.py +++ b/conftest.py @@ -2,3 +2,10 @@ def pytest_addoption(parser): parser.addoption( "--fast", action="store_true", default=False, help="run tests fast" ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "slow: mark test as slow to run (deselect with '-m \"not slow\"')", + ) diff --git a/cosmic-ray.toml b/cosmic-ray.toml index f5de84bf..3d27d1b4 100644 --- a/cosmic-ray.toml +++ b/cosmic-ray.toml @@ -3,7 +3,7 @@ module-path = "src" python-version = "" timeout = 30.0 exclude-modules = ['src/ecdsa/_version.py', 'src/ecdsa/test*'] -test-command = "pytest -x --fast src/" +test-command = "pytest -x --fast -m 'not slow' src/" [cosmic-ray.execution-engine] name = "local" diff --git a/src/ecdsa/test_ecdh.py b/src/ecdsa/test_ecdh.py index 872d4d14..3395a212 100644 --- a/src/ecdsa/test_ecdh.py +++ b/src/ecdsa/test_ecdh.py @@ -366,6 +366,7 @@ def run_openssl(cmd): ) +@pytest.mark.slow @pytest.mark.parametrize( "vcurve", curves, diff --git a/src/ecdsa/test_jacobi.py b/src/ecdsa/test_jacobi.py index 322b568c..7afe19af 100644 --- a/src/ecdsa/test_jacobi.py +++ b/src/ecdsa/test_jacobi.py @@ -348,7 +348,7 @@ def test_add_same_scale_points_static(self): self.assertEqual(c, x + y) - @settings(max_examples=14) + @settings(max_examples=10) @given( st.integers( min_value=1, max_value=int(generator_brainpoolp160r1.order() - 1) diff --git a/src/ecdsa/test_pyecdsa.py b/src/ecdsa/test_pyecdsa.py index 1a428478..cdaa03e2 100644 --- a/src/ecdsa/test_pyecdsa.py +++ b/src/ecdsa/test_pyecdsa.py @@ -1049,6 +1049,7 @@ def get_openssl_messagedigest_arg(self, hash_name): # vk: 3:OpenSSL->python 4:python->OpenSSL # sig: 5:OpenSSL->python 6:python->OpenSSL + @pytest.mark.slow @pytest.mark.skipif( "secp112r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp112r1", @@ -1084,6 +1085,7 @@ def test_from_openssl_secp160r1(self): def test_from_openssl_nist192p(self): return self.do_test_from_openssl(NIST192p) + @pytest.mark.slow @pytest.mark.skipif( "prime192v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime192v1", @@ -1091,6 +1093,7 @@ def test_from_openssl_nist192p(self): def test_from_openssl_nist192p_sha256(self): return self.do_test_from_openssl(NIST192p, "SHA256") + @pytest.mark.slow @pytest.mark.skipif( "secp224r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp224r1", @@ -1098,6 +1101,7 @@ def test_from_openssl_nist192p_sha256(self): def test_from_openssl_nist224p(self): return self.do_test_from_openssl(NIST224p) + @pytest.mark.slow @pytest.mark.skipif( "prime256v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime256v1", @@ -1105,6 +1109,7 @@ def test_from_openssl_nist224p(self): def test_from_openssl_nist256p(self): return self.do_test_from_openssl(NIST256p) + @pytest.mark.slow @pytest.mark.skipif( "prime256v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime256v1", @@ -1112,6 +1117,7 @@ def test_from_openssl_nist256p(self): def test_from_openssl_nist256p_sha384(self): return self.do_test_from_openssl(NIST256p, "SHA384") + @pytest.mark.slow @pytest.mark.skipif( "prime256v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime256v1", @@ -1119,6 +1125,7 @@ def test_from_openssl_nist256p_sha384(self): def test_from_openssl_nist256p_sha512(self): return self.do_test_from_openssl(NIST256p, "SHA512") + @pytest.mark.slow @pytest.mark.skipif( "secp384r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp384r1", @@ -1126,6 +1133,7 @@ def test_from_openssl_nist256p_sha512(self): def test_from_openssl_nist384p(self): return self.do_test_from_openssl(NIST384p) + @pytest.mark.slow @pytest.mark.skipif( "secp521r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp521r1", @@ -1133,6 +1141,7 @@ def test_from_openssl_nist384p(self): def test_from_openssl_nist521p(self): return self.do_test_from_openssl(NIST521p) + @pytest.mark.slow @pytest.mark.skipif( "secp256k1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp256k1", @@ -1140,6 +1149,7 @@ def test_from_openssl_nist521p(self): def test_from_openssl_secp256k1(self): return self.do_test_from_openssl(SECP256k1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP160r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP160r1", @@ -1147,6 +1157,7 @@ def test_from_openssl_secp256k1(self): def test_from_openssl_brainpoolp160r1(self): return self.do_test_from_openssl(BRAINPOOLP160r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP192r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP192r1", @@ -1154,6 +1165,7 @@ def test_from_openssl_brainpoolp160r1(self): def test_from_openssl_brainpoolp192r1(self): return self.do_test_from_openssl(BRAINPOOLP192r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP224r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP224r1", @@ -1161,6 +1173,7 @@ def test_from_openssl_brainpoolp192r1(self): def test_from_openssl_brainpoolp224r1(self): return self.do_test_from_openssl(BRAINPOOLP224r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP256r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP256r1", @@ -1168,6 +1181,7 @@ def test_from_openssl_brainpoolp224r1(self): def test_from_openssl_brainpoolp256r1(self): return self.do_test_from_openssl(BRAINPOOLP256r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP320r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP320r1", @@ -1175,6 +1189,7 @@ def test_from_openssl_brainpoolp256r1(self): def test_from_openssl_brainpoolp320r1(self): return self.do_test_from_openssl(BRAINPOOLP320r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP384r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP384r1", @@ -1182,6 +1197,7 @@ def test_from_openssl_brainpoolp320r1(self): def test_from_openssl_brainpoolp384r1(self): return self.do_test_from_openssl(BRAINPOOLP384r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP512r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP512r1", @@ -1290,6 +1306,7 @@ def do_test_from_openssl(self, curve, hash_name="SHA1"): sk_from_p8 = SigningKey.from_pem(privkey_p8_pem) self.assertEqual(sk, sk_from_p8) + @pytest.mark.slow @pytest.mark.skipif( "secp112r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp112r1", @@ -1325,6 +1342,7 @@ def test_to_openssl_secp160r1(self): def test_to_openssl_nist192p(self): self.do_test_to_openssl(NIST192p) + @pytest.mark.slow @pytest.mark.skipif( "prime192v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime192v1", @@ -1332,6 +1350,7 @@ def test_to_openssl_nist192p(self): def test_to_openssl_nist192p_sha256(self): self.do_test_to_openssl(NIST192p, "SHA256") + @pytest.mark.slow @pytest.mark.skipif( "secp224r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp224r1", @@ -1339,6 +1358,7 @@ def test_to_openssl_nist192p_sha256(self): def test_to_openssl_nist224p(self): self.do_test_to_openssl(NIST224p) + @pytest.mark.slow @pytest.mark.skipif( "prime256v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime256v1", @@ -1346,6 +1366,7 @@ def test_to_openssl_nist224p(self): def test_to_openssl_nist256p(self): self.do_test_to_openssl(NIST256p) + @pytest.mark.slow @pytest.mark.skipif( "prime256v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime256v1", @@ -1353,6 +1374,7 @@ def test_to_openssl_nist256p(self): def test_to_openssl_nist256p_sha384(self): self.do_test_to_openssl(NIST256p, "SHA384") + @pytest.mark.slow @pytest.mark.skipif( "prime256v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime256v1", @@ -1360,6 +1382,7 @@ def test_to_openssl_nist256p_sha384(self): def test_to_openssl_nist256p_sha512(self): self.do_test_to_openssl(NIST256p, "SHA512") + @pytest.mark.slow @pytest.mark.skipif( "secp384r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp384r1", @@ -1367,6 +1390,7 @@ def test_to_openssl_nist256p_sha512(self): def test_to_openssl_nist384p(self): self.do_test_to_openssl(NIST384p) + @pytest.mark.slow @pytest.mark.skipif( "secp521r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp521r1", @@ -1374,6 +1398,7 @@ def test_to_openssl_nist384p(self): def test_to_openssl_nist521p(self): self.do_test_to_openssl(NIST521p) + @pytest.mark.slow @pytest.mark.skipif( "secp256k1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp256k1", @@ -1381,6 +1406,7 @@ def test_to_openssl_nist521p(self): def test_to_openssl_secp256k1(self): self.do_test_to_openssl(SECP256k1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP160r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP160r1", @@ -1388,6 +1414,7 @@ def test_to_openssl_secp256k1(self): def test_to_openssl_brainpoolp160r1(self): self.do_test_to_openssl(BRAINPOOLP160r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP192r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP192r1", @@ -1395,6 +1422,7 @@ def test_to_openssl_brainpoolp160r1(self): def test_to_openssl_brainpoolp192r1(self): self.do_test_to_openssl(BRAINPOOLP192r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP224r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP224r1", @@ -1402,6 +1430,7 @@ def test_to_openssl_brainpoolp192r1(self): def test_to_openssl_brainpoolp224r1(self): self.do_test_to_openssl(BRAINPOOLP224r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP256r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP256r1", @@ -1409,6 +1438,7 @@ def test_to_openssl_brainpoolp224r1(self): def test_to_openssl_brainpoolp256r1(self): self.do_test_to_openssl(BRAINPOOLP256r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP320r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP320r1", @@ -1416,6 +1446,7 @@ def test_to_openssl_brainpoolp256r1(self): def test_to_openssl_brainpoolp320r1(self): self.do_test_to_openssl(BRAINPOOLP320r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP384r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP384r1", @@ -1423,6 +1454,7 @@ def test_to_openssl_brainpoolp320r1(self): def test_to_openssl_brainpoolp384r1(self): self.do_test_to_openssl(BRAINPOOLP384r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP512r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP512r1", From 260cc296738cf7f81e70cf94b83f257fb941aef2 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Dec 2020 16:13:38 +0100 Subject: [PATCH 05/29] more test speed-ups --- src/ecdsa/test_ecdsa.py | 5 ++- src/ecdsa/test_ellipticcurve.py | 2 +- src/ecdsa/test_jacobi.py | 46 +++++++++++++++++++++---- src/ecdsa/test_malformed_sigs.py | 4 +-- src/ecdsa/test_numbertheory.py | 10 +++--- src/ecdsa/test_pyecdsa.py | 58 ++++++++++++++------------------ 6 files changed, 79 insertions(+), 46 deletions(-) diff --git a/src/ecdsa/test_ecdsa.py b/src/ecdsa/test_ecdsa.py index 2af527b6..a23373fd 100644 --- a/src/ecdsa/test_ecdsa.py +++ b/src/ecdsa/test_ecdsa.py @@ -634,7 +634,10 @@ def st_random_gen_key_msg_nonce(draw): SIG_VER_SETTINGS = dict(HYP_SETTINGS) -SIG_VER_SETTINGS["max_examples"] = 10 +if "--fast" in sys.argv: + SIG_VER_SETTINGS["max_examples"] = 1 +else: + SIG_VER_SETTINGS["max_examples"] = 10 @settings(**SIG_VER_SETTINGS) diff --git a/src/ecdsa/test_ellipticcurve.py b/src/ecdsa/test_ellipticcurve.py index f46fd9ea..33248f6e 100644 --- a/src/ecdsa/test_ellipticcurve.py +++ b/src/ecdsa/test_ellipticcurve.py @@ -40,7 +40,7 @@ HYP_SLOW_SETTINGS = dict(HYP_SETTINGS) -HYP_SLOW_SETTINGS["max_examples"] = 10 +HYP_SLOW_SETTINGS["max_examples"] = 2 @settings(**HYP_SLOW_SETTINGS) diff --git a/src/ecdsa/test_jacobi.py b/src/ecdsa/test_jacobi.py index 7afe19af..a4750455 100644 --- a/src/ecdsa/test_jacobi.py +++ b/src/ecdsa/test_jacobi.py @@ -1,4 +1,5 @@ import pickle +import sys try: import unittest2 as unittest @@ -32,6 +33,13 @@ NO_OLD_SETTINGS["deadline"] = 5000 +SLOW_SETTINGS = {} +if "--fast" in sys.argv: + SLOW_SETTINGS["max_examples"] = 2 +else: + SLOW_SETTINGS["max_examples"] = 10 + + class TestJacobi(unittest.TestCase): def test___init__(self): curve = object() @@ -199,7 +207,7 @@ def test_compare_double_with_multiply(self): self.assertEqual(dbl, mlpl) - @settings(max_examples=10) + @settings(**SLOW_SETTINGS) @given( st.integers( min_value=0, max_value=int(generator_brainpoolp160r1.order() - 1) @@ -214,7 +222,7 @@ def test_multiplications(self, mul): self.assertEqual((pj.x(), pj.y()), (pw.x(), pw.y())) self.assertEqual(pj, pw) - @settings(max_examples=10) + @settings(**SLOW_SETTINGS) @given( st.integers( min_value=0, max_value=int(generator_brainpoolp160r1.order() - 1) @@ -232,7 +240,7 @@ def test_precompute(self, mul): self.assertEqual(a, b) - @settings(max_examples=10) + @settings(**SLOW_SETTINGS) @given( st.integers( min_value=1, max_value=int(generator_brainpoolp160r1.order() - 1) @@ -251,7 +259,7 @@ def test_add_scaled_points(self, a_mul, b_mul): self.assertEqual(c, j_g * (a_mul + b_mul)) - @settings(max_examples=10) + @settings(**SLOW_SETTINGS) @given( st.integers( min_value=1, max_value=int(generator_brainpoolp160r1.order() - 1) @@ -283,7 +291,8 @@ def test_add_one_scaled_point(self, a_mul, b_mul, new_z): self.assertEqual(c, j_g * (a_mul + b_mul)) - @settings(max_examples=10) + @pytest.mark.slow + @settings(**SLOW_SETTINGS) @given( st.integers( min_value=1, max_value=int(generator_brainpoolp160r1.order() - 1) @@ -348,7 +357,8 @@ def test_add_same_scale_points_static(self): self.assertEqual(c, x + y) - @settings(max_examples=10) + @pytest.mark.slow + @settings(**SLOW_SETTINGS) @given( st.integers( min_value=1, max_value=int(generator_brainpoolp160r1.order() - 1) @@ -423,6 +433,30 @@ def test_add_different_scale_points_static(self): self.assertEqual(c, x + y) + def test_add_different_scale_points_static(self): + j_g = generator_brainpoolp160r1 + p = curve_brainpoolp160r1.p() + a = j_g * 11 + a.scale() + z1 = 13 + x = PointJacobi( + curve_brainpoolp160r1, + a.x() * z1**2 % p, + a.y() * z1**3 % p, + z1, + ) + z2 = 29 + y = PointJacobi( + curve_brainpoolp160r1, + a.x() * z2**2 % p, + a.y() * z2**3 % p, + z2, + ) + + c = a + a + + self.assertEqual(c, x + y) + def test_add_point_3_times(self): j_g = PointJacobi.from_affine(generator_256) diff --git a/src/ecdsa/test_malformed_sigs.py b/src/ecdsa/test_malformed_sigs.py index 2a318a44..e822a0fd 100644 --- a/src/ecdsa/test_malformed_sigs.py +++ b/src/ecdsa/test_malformed_sigs.py @@ -159,7 +159,7 @@ def st_fuzzed_sig(draw, keys_and_sigs): slow_params = dict(params) if "--fast" in sys.argv: - slow_params["max_examples"] = 2 + slow_params["max_examples"] = 1 else: slow_params["max_examples"] = 10 @@ -318,7 +318,7 @@ def test_random_der_as_signature(params, der): verifying_key.verify(der, example_data, sigdecode=sigdecode_der) -@settings(**params) +@settings(**slow_params) @given(st.sampled_from(keys_and_sigs), st.binary(max_size=1024**2)) @example( keys_and_sigs[0], encode_sequence(encode_integer(0), encode_integer(0)) diff --git a/src/ecdsa/test_numbertheory.py b/src/ecdsa/test_numbertheory.py index dcaa6d92..643a0148 100644 --- a/src/ecdsa/test_numbertheory.py +++ b/src/ecdsa/test_numbertheory.py @@ -67,6 +67,7 @@ def test_next_prime_with_nums_less_2(val): assert next_prime(val) == 2 +@pytest.mark.slow @pytest.mark.parametrize("prime", smallprimes) def test_square_root_mod_prime_for_small_primes(prime): squares = set() @@ -258,7 +259,7 @@ def st_comp_no_com_fac(draw): HYP_SLOW_SETTINGS = dict(HYP_SETTINGS) if "--fast" in sys.argv: - HYP_SLOW_SETTINGS["max_examples"] = 2 + HYP_SLOW_SETTINGS["max_examples"] = 1 else: HYP_SLOW_SETTINGS["max_examples"] = 20 @@ -373,7 +374,8 @@ def test_square_root_mod_prime(self, vals): calc = square_root_mod_prime(square, prime) assert calc * calc % prime == square - @settings(**HYP_SETTINGS) + @pytest.mark.slow + @settings(**HYP_SLOW_SETTINGS) @given(st.integers(min_value=1, max_value=10**12)) @example(265399 * 1526929) @example(373297**2 * 553991) @@ -410,7 +412,7 @@ def test_jacobi_with_zero(self): def test_jacobi_with_one(self): assert jacobi(1, 3) == 1 - @settings(**HYP_SETTINGS) + @settings(**HYP_SLOW_SETTINGS) @given(st.integers(min_value=3, max_value=1000).filter(lambda x: x % 2)) def test_jacobi(self, mod): if is_prime(mod): @@ -429,7 +431,7 @@ def test_jacobi(self, mod): c *= jacobi(a, i[0]) ** i[1] assert c == jacobi(a, mod) - @settings(**HYP_SETTINGS) + @settings(**HYP_SLOW_SETTINGS) @given(st_two_nums_rel_prime()) def test_inverse_mod(self, nums): num, mod = nums diff --git a/src/ecdsa/test_pyecdsa.py b/src/ecdsa/test_pyecdsa.py index cdaa03e2..9335631b 100644 --- a/src/ecdsa/test_pyecdsa.py +++ b/src/ecdsa/test_pyecdsa.py @@ -80,7 +80,7 @@ class SubprocessError(Exception): if "--fast" in sys.argv: - HYP_SETTINGS["max_examples"] = 20 + HYP_SETTINGS["max_examples"] = 2 def run_openssl(cmd): @@ -146,41 +146,13 @@ def test_bad_usage(self): self.assertRaises(TypeError, SigningKey) self.assertRaises(TypeError, VerifyingKey) - def test_lengths(self): + def test_lengths_default(self): default = NIST192p priv = SigningKey.generate() pub = priv.get_verifying_key() self.assertEqual(len(pub.to_string()), default.verifying_key_length) - sig = priv.sign(b("data")) + sig = priv.sign(b"data") self.assertEqual(len(sig), default.signature_length) - for curve in ( - NIST192p, - NIST224p, - NIST256p, - NIST384p, - NIST521p, - BRAINPOOLP160r1, - BRAINPOOLP192r1, - BRAINPOOLP224r1, - BRAINPOOLP256r1, - BRAINPOOLP320r1, - BRAINPOOLP384r1, - BRAINPOOLP512r1, - BRAINPOOLP160t1, - BRAINPOOLP192t1, - BRAINPOOLP224t1, - BRAINPOOLP256t1, - BRAINPOOLP320t1, - BRAINPOOLP384t1, - BRAINPOOLP512t1, - ): - priv = SigningKey.generate(curve=curve) - pub1 = priv.get_verifying_key() - pub2 = VerifyingKey.from_string(pub1.to_string(), curve) - self.assertEqual(pub1.to_string(), pub2.to_string()) - self.assertEqual(len(pub1.to_string()), curve.verifying_key_length) - sig = priv.sign(b("data")) - self.assertEqual(len(sig), curve.signature_length) def test_serialize(self): seed = b("secret") @@ -1013,6 +985,23 @@ def test_VerifyingKey_encode_decode(curve, encoding): assert vk.pubkey.point == from_enc.pubkey.point +if "--fast" in sys.argv: + params = [NIST192p, BRAINPOOLP160r1] +else: + params = curves + + +@pytest.mark.parametrize("curve", params) +def test_lengths(curve): + priv = SigningKey.generate(curve=curve) + pub1 = priv.get_verifying_key() + pub2 = VerifyingKey.from_string(pub1.to_string(), curve) + assert pub1.to_string() == pub2.to_string() + assert len(pub1.to_string()) == curve.verifying_key_length + sig = priv.sign(b"data") + assert len(sig) == curve.signature_length + + class OpenSSL(unittest.TestCase): # test interoperability with OpenSSL tools. Note that openssl's ECDSA # sign/verify arguments changed between 0.9.8 and 1.0.0: the early @@ -1819,6 +1808,7 @@ def test_constructed(self): class Util(unittest.TestCase): + @pytest.mark.slow def test_trytryagain(self): tta = util.randrange_from_seed__trytryagain for i in range(1000): @@ -1846,7 +1836,7 @@ def test_trytryagain_single(self): seed = b"text" n = tta(seed, order) # known issue: https://github.com/warner/python-ecdsa/issues/221 - if sys.version_info < (3, 0): # pragma: no branch + if sys.version_info < (3, 0): self.assertEqual(n, 228) else: self.assertEqual(n, 18) @@ -2184,6 +2174,7 @@ def test_brainpoolP256r1(self): ), ) + @pytest.mark.slow def test_brainpoolP384r1(self): self._do( curve=curve_brainpoolp384r1, @@ -2230,6 +2221,7 @@ def test_brainpoolP384r1(self): ), ) + @pytest.mark.slow def test_brainpoolP512r1(self): self._do( curve=curve_brainpoolp512r1, @@ -2334,6 +2326,7 @@ def test_brainpoolP256r1(self): ), ) + @pytest.mark.slow def test_brainpoolP384r1(self): self._do( curve=curve_brainpoolp384r1, @@ -2380,6 +2373,7 @@ def test_brainpoolP384r1(self): ), ) + @pytest.mark.slow def test_brainpoolP512r1(self): self._do( curve=curve_brainpoolp512r1, From 724535a14588b7e04e3308ff8761b00f5dc86560 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Dec 2020 16:46:41 +0100 Subject: [PATCH 06/29] extend the cosmic-ray testing --- .travis.yml | 4 ++-- cosmic-ray.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 009656c5..931b1ad7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -174,8 +174,8 @@ script: cr-report session.sqlite | tail -n 3; done & REPORT_PID=$! - # kill exec after 25 minutes - (sleep $((60*25)); kill $COSMIC_PID) & + # kill exec after 40 minutes + (sleep $((60*40)); kill $COSMIC_PID) & fi - if [[ $MUTATION ]]; then wait $COSMIC_PID ; kill $REPORT_PID ; true; fi - if [[ $MUTATION ]]; then cr-report --show-output session.sqlite | tail -n 40; fi diff --git a/cosmic-ray.toml b/cosmic-ray.toml index 3d27d1b4..09f364aa 100644 --- a/cosmic-ray.toml +++ b/cosmic-ray.toml @@ -1,7 +1,7 @@ [cosmic-ray] module-path = "src" python-version = "" -timeout = 30.0 +timeout = 20.0 exclude-modules = ['src/ecdsa/_version.py', 'src/ecdsa/test*'] test-command = "pytest -x --fast -m 'not slow' src/" From cbf086e602f04c873b64b4b4f0a3174db8bb3857 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Dec 2020 17:46:00 +0100 Subject: [PATCH 07/29] how to run cosmic-ray comfortably --- cosmic-ray.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cosmic-ray.sh b/cosmic-ray.sh index ba512d50..d9aea619 100644 --- a/cosmic-ray.sh +++ b/cosmic-ray.sh @@ -4,6 +4,8 @@ set -e cosmic-ray init cosmic-ray.toml session.sqlite cosmic-ray baseline --report session.sqlite cr-report --show-output session.baseline.sqlite +# some mutations cause huge memory use, so put it in a cgroup +# systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec session.sqlite cosmic-ray exec session.sqlite cr-report session.sqlite cr-html session.sqlite > session.html From 6ee5257f7a34e202d55fad17931aee0008ab608a Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 6 Dec 2020 20:40:43 +0100 Subject: [PATCH 08/29] increase the survival rate limit currently we have 28.88% survival rate, so set that as the baseline --- .travis.yml | 2 +- cosmic-ray.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 931b1ad7..8729ca80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -179,7 +179,7 @@ script: fi - if [[ $MUTATION ]]; then wait $COSMIC_PID ; kill $REPORT_PID ; true; fi - if [[ $MUTATION ]]; then cr-report --show-output session.sqlite | tail -n 40; fi - - if [[ $MUTATION ]]; then cr-rate --estimate --fail-over 25 --confidence 99.9 session.sqlite; fi + - if [[ $MUTATION ]]; then cr-rate --estimate --fail-over 29 --confidence 99.9 session.sqlite; fi after_success: - if [[ -z $INSTRUMENTAL && -z $MUTATION ]]; then coveralls; fi diff --git a/cosmic-ray.sh b/cosmic-ray.sh index d9aea619..384750a3 100644 --- a/cosmic-ray.sh +++ b/cosmic-ray.sh @@ -9,4 +9,4 @@ cr-report --show-output session.baseline.sqlite cosmic-ray exec session.sqlite cr-report session.sqlite cr-html session.sqlite > session.html -cr-rate --estimate --fail-over 25 --confidence 99.9 session.sqlite +cr-rate --estimate --fail-over 29 --confidence 99.9 session.sqlite From 082a403b108b3cca81bfc976bd34f8f820d42572 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 7 Dec 2020 00:31:37 +0100 Subject: [PATCH 09/29] waive test coverage for mutation speedups the branches in test coverage that are there to speed up mutation testing don't have to be covered by regular testing so waive them --- src/ecdsa/test_der.py | 2 +- src/ecdsa/test_ecdsa.py | 2 +- src/ecdsa/test_jacobi.py | 2 +- src/ecdsa/test_malformed_sigs.py | 4 ++-- src/ecdsa/test_numbertheory.py | 4 ++-- src/ecdsa/test_pyecdsa.py | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/ecdsa/test_der.py b/src/ecdsa/test_der.py index f450baf8..f08ea573 100644 --- a/src/ecdsa/test_der.py +++ b/src/ecdsa/test_der.py @@ -466,7 +466,7 @@ def st_oid(draw, max_value=2**512, max_size=50): HYP_SETTINGS = {} -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover HYP_SETTINGS["max_examples"] = 2 diff --git a/src/ecdsa/test_ecdsa.py b/src/ecdsa/test_ecdsa.py index a23373fd..29f05b8f 100644 --- a/src/ecdsa/test_ecdsa.py +++ b/src/ecdsa/test_ecdsa.py @@ -634,7 +634,7 @@ def st_random_gen_key_msg_nonce(draw): SIG_VER_SETTINGS = dict(HYP_SETTINGS) -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover SIG_VER_SETTINGS["max_examples"] = 1 else: SIG_VER_SETTINGS["max_examples"] = 10 diff --git a/src/ecdsa/test_jacobi.py b/src/ecdsa/test_jacobi.py index a4750455..ed598bb3 100644 --- a/src/ecdsa/test_jacobi.py +++ b/src/ecdsa/test_jacobi.py @@ -34,7 +34,7 @@ SLOW_SETTINGS = {} -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover SLOW_SETTINGS["max_examples"] = 2 else: SLOW_SETTINGS["max_examples"] = 10 diff --git a/src/ecdsa/test_malformed_sigs.py b/src/ecdsa/test_malformed_sigs.py index e822a0fd..d7050d2f 100644 --- a/src/ecdsa/test_malformed_sigs.py +++ b/src/ecdsa/test_malformed_sigs.py @@ -154,11 +154,11 @@ def st_fuzzed_sig(draw, keys_and_sigs): HealthCheck.filter_too_much, HealthCheck.too_slow, ] -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover params["max_examples"] = 20 slow_params = dict(params) -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover slow_params["max_examples"] = 1 else: slow_params["max_examples"] = 10 diff --git a/src/ecdsa/test_numbertheory.py b/src/ecdsa/test_numbertheory.py index 643a0148..983039e5 100644 --- a/src/ecdsa/test_numbertheory.py +++ b/src/ecdsa/test_numbertheory.py @@ -253,12 +253,12 @@ def st_comp_no_com_fac(draw): # the factorization() sometimes takes a long time to finish HYP_SETTINGS["deadline"] = 5000 -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover HYP_SETTINGS["max_examples"] = 20 HYP_SLOW_SETTINGS = dict(HYP_SETTINGS) -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover HYP_SLOW_SETTINGS["max_examples"] = 1 else: HYP_SLOW_SETTINGS["max_examples"] = 20 diff --git a/src/ecdsa/test_pyecdsa.py b/src/ecdsa/test_pyecdsa.py index 9335631b..579a5738 100644 --- a/src/ecdsa/test_pyecdsa.py +++ b/src/ecdsa/test_pyecdsa.py @@ -79,7 +79,7 @@ class SubprocessError(Exception): HYP_SETTINGS = {} -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover HYP_SETTINGS["max_examples"] = 2 @@ -985,7 +985,7 @@ def test_VerifyingKey_encode_decode(curve, encoding): assert vk.pubkey.point == from_enc.pubkey.point -if "--fast" in sys.argv: +if "--fast" in sys.argv: # pragma: no cover params = [NIST192p, BRAINPOOLP160r1] else: params = curves @@ -1836,9 +1836,9 @@ def test_trytryagain_single(self): seed = b"text" n = tta(seed, order) # known issue: https://github.com/warner/python-ecdsa/issues/221 - if sys.version_info < (3, 0): + if sys.version_info < (3, 0): # pragma: no branch self.assertEqual(n, 228) - else: + else: # pragma: no branch self.assertEqual(n, 18) @settings(**HYP_SETTINGS) From c2a1c1631f70f3b9273f061be49bb94e33a103c0 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Mon, 7 Dec 2020 03:28:58 +0100 Subject: [PATCH 10/29] ensure that the equality tests are collected by instrumental --- src/ecdsa/test_keys.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ecdsa/test_keys.py b/src/ecdsa/test_keys.py index 4d9e976e..a41a3dbc 100644 --- a/src/ecdsa/test_keys.py +++ b/src/ecdsa/test_keys.py @@ -249,13 +249,13 @@ def test_array_array_of_bytes_memoryview(self): self.assertEqual(self.vk.to_string(), vk.to_string()) def test_equality_on_verifying_keys(self): - self.assertEqual(self.vk, self.sk.get_verifying_key()) + self.assertTrue(self.vk == self.sk.get_verifying_key()) def test_inequality_on_verifying_keys(self): - self.assertNotEqual(self.vk, self.vk2) + self.assertFalse(self.vk == self.vk2) def test_inequality_on_verifying_keys_not_implemented(self): - self.assertNotEqual(self.vk, None) + self.assertFalse(self.vk == None) def test_VerifyingKey_inequality_on_same_curve(self): self.assertNotEqual(self.vk, self.sk2.verifying_key) @@ -264,7 +264,7 @@ def test_SigningKey_inequality_on_same_curve(self): self.assertNotEqual(self.sk, self.sk2) def test_inequality_on_wrong_types(self): - self.assertNotEqual(self.vk, self.sk) + self.assertFalse(self.vk == self.sk) def test_from_public_point_old(self): pj = self.vk.pubkey.point @@ -272,7 +272,7 @@ def test_from_public_point_old(self): vk = VerifyingKey.from_public_point(point, self.vk.curve) - self.assertEqual(vk, self.vk) + self.assertTrue(vk == self.vk) def test_ed25519_VerifyingKey_repr__(self): sk = SigningKey.from_string(Ed25519.generator.to_bytes(), Ed25519) @@ -970,14 +970,14 @@ def test_VerifyingKey_inequality_with_different_curves(): sk1 = SigningKey.from_secret_exponent(2, BRAINPOOLP160r1) sk2 = SigningKey.from_secret_exponent(2, NIST256p) - assert sk1.verifying_key != sk2.verifying_key + assert not (sk1.verifying_key == sk2.verifying_key) def test_VerifyingKey_inequality_with_different_secret_points(): sk1 = SigningKey.from_secret_exponent(2, BRAINPOOLP160r1) sk2 = SigningKey.from_secret_exponent(3, BRAINPOOLP160r1) - assert sk1.verifying_key != sk2.verifying_key + assert not (sk1.verifying_key == sk2.verifying_key) def test_SigningKey_from_pem_pkcs8v2_EdDSA(): From c17d7e7a002cba75d807f20aebe1874a285ff8de Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 28 Dec 2023 18:04:51 +0100 Subject: [PATCH 11/29] test verification with plain Point PointJacobi has a mul_add() method, but Point doesn't, verify that verifies() works with both point encodings --- src/ecdsa/test_ecdsa.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/ecdsa/test_ecdsa.py b/src/ecdsa/test_ecdsa.py index 29f05b8f..6798a7e6 100644 --- a/src/ecdsa/test_ecdsa.py +++ b/src/ecdsa/test_ecdsa.py @@ -27,6 +27,7 @@ generator_112r2, int_to_string, ) +from .ellipticcurve import Point HYP_SETTINGS = {} @@ -76,6 +77,19 @@ def test_verification(self): def test_rejection(self): assert not self.pubk.verifies(self.msg - 1, self.sig) + def test_verification_with_regular_point(self): + pubk = Public_key( + Point( + generator_192.curve(), + generator_192.x(), + generator_192.y(), + generator_192.order(), + ), + self.pubk.point, + ) + + assert pubk.verifies(self.msg, self.sig) + class TestPublicKey(unittest.TestCase): def test_equality_public_keys(self): From c98364927e406af037d8f9652ff18094f207dcec Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 28 Dec 2023 18:45:59 +0100 Subject: [PATCH 12/29] deprecate digest_integer() and related methods --- src/ecdsa/ecdsa.py | 26 +++++++++++++++++++++++--- src/ecdsa/test_ecdsa.py | 22 +++++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/src/ecdsa/ecdsa.py b/src/ecdsa/ecdsa.py index 5a891ec9..7ac76c2f 100644 --- a/src/ecdsa/ecdsa.py +++ b/src/ecdsa/ecdsa.py @@ -64,6 +64,7 @@ modified as part of the python-ecdsa package. """ +import warnings from six import int2byte, b from . import ellipticcurve from . import numbertheory @@ -269,8 +270,14 @@ def sign(self, hash, random_k): return Signature(r, s) -def int_to_string(x): +def int_to_string(x): # pragma: no cover """Convert integer x into a string of bytes, as per X9.62.""" + # deprecated in 0.19 + warnings.warn( + "Function is unused in library code. If you use this code, " + "change to util.string_to_number.", + DeprecationWarning, + ) assert x >= 0 if x == 0: return b("\0") @@ -284,8 +291,14 @@ def int_to_string(x): return b("").join(result) -def string_to_int(s): +def string_to_int(s): # pragma: no cover """Convert a string of bytes into an integer, as per X9.62.""" + # deprecated in 0.19 + warnings.warn( + "Function is unused in library code. If you use this code, " + "change to util.number_to_string.", + DeprecationWarning, + ) result = 0 for c in s: if not isinstance(c, int): @@ -294,9 +307,16 @@ def string_to_int(s): return result -def digest_integer(m): +def digest_integer(m): # pragma: no cover """Convert an integer into a string of bytes, compute its SHA-1 hash, and convert the result to an integer.""" + # deprecated in 0.19 + warnings.warn( + "Function is unused in library code. If you use this code, " + "change to a one-liner with util.number_to_string and " + "util.string_to_number methods.", + DeprecationWarning, + ) # # I don't expect this function to be used much. I wrote # it in order to be able to duplicate the examples diff --git a/src/ecdsa/test_ecdsa.py b/src/ecdsa/test_ecdsa.py index 6798a7e6..c1e25829 100644 --- a/src/ecdsa/test_ecdsa.py +++ b/src/ecdsa/test_ecdsa.py @@ -598,7 +598,13 @@ def test_signature_validity(gen, msg, qx, qy, r, s, expected): elliptic curve of `gen`, `r` and `s` are the signature, and `expected` is True iff the signature is expected to be valid.""" pubk = Public_key(gen, ellipticcurve.Point(gen.curve(), qx, qy)) - assert expected == pubk.verifies(digest_integer(msg), Signature(r, s)) + with pytest.warns(DeprecationWarning) as warns: + msg_dgst = digest_integer(msg) + assert len(warns) == 3 + assert "unused" in warns[0].message.args[0] + assert "unused" in warns[1].message.args[0] + assert "unused" in warns[2].message.args[0] + assert expected == pubk.verifies(msg_dgst, Signature(r, s)) @pytest.mark.parametrize( @@ -607,7 +613,13 @@ def test_signature_validity(gen, msg, qx, qy, r, s, expected): def test_pk_recovery(gen, msg, r, s, qx, qy, expected): del expected sign = Signature(r, s) - pks = sign.recover_public_keys(digest_integer(msg), gen) + with pytest.warns(DeprecationWarning) as warns: + msg_dgst = digest_integer(msg) + assert len(warns) == 3 + assert "unused" in warns[0].message.args[0] + assert "unused" in warns[1].message.args[0] + assert "unused" in warns[2].message.args[0] + pks = sign.recover_public_keys(msg_dgst, gen) assert pks @@ -675,4 +687,8 @@ def test_sig_verify(args): def test_int_to_string_with_zero(): - assert int_to_string(0) == b"\x00" + with pytest.warns(DeprecationWarning) as warns: + assert int_to_string(0) == b"\x00" + + assert len(warns) == 1 + assert "unused" in warns[0].message.args[0] From f369620060ed254954d43f56e5b1967ea8a0231c Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 28 Dec 2023 18:58:59 +0100 Subject: [PATCH 13/29] test coverage for Edwards key public_point() method --- src/ecdsa/test_eddsa.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/ecdsa/test_eddsa.py b/src/ecdsa/test_eddsa.py index a125b94e..d398d033 100644 --- a/src/ecdsa/test_eddsa.py +++ b/src/ecdsa/test_eddsa.py @@ -259,6 +259,22 @@ def test_add_wrong_point_type(self): self.assertIn("different curve", str(e.exception)) +def test_generate_with_point(): + x1 = int( + "427838232691226969392843410947554224151809796397784248136826" + "78720006717057747" + ) + y1 = int( + "463168356949264781694283940034751631413079938662562256157830" + "33603165251855960" + ) + p = PointEdwards(curve_ed25519, x1, y1, 1, x1 * y1) + + pk = PublicKey(generator_ed25519, b"0" * 32, public_point=p) + + assert pk.public_point() == p + + def test_ed25519_mul_to_order_min_1(): x1 = int( "427838232691226969392843410947554224151809796397784248136826" From c5b7c7acc9bce4b5cb08e357ea9cf6aa547ee407 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 28 Dec 2023 20:39:56 +0100 Subject: [PATCH 14/29] test coverage for CurveFp.__str__ --- src/ecdsa/ellipticcurve.py | 10 ++++++++-- src/ecdsa/test_ellipticcurve.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/ecdsa/ellipticcurve.py b/src/ecdsa/ellipticcurve.py index af2aaeab..942094a0 100644 --- a/src/ecdsa/ellipticcurve.py +++ b/src/ecdsa/ellipticcurve.py @@ -136,11 +136,17 @@ def contains_point(self, x, y): return (y * y - ((x * x + self.__a) * x + self.__b)) % self.__p == 0 def __str__(self): - return "CurveFp(p=%d, a=%d, b=%d, h=%d)" % ( + if self.__h is not None: + return "CurveFp(p={0}, a={1}, b={2}, h={3})".format( + self.__p, + self.__a, + self.__b, + self.__h, + ) + return "CurveFp(p={0}, a={1}, b={2})".format( self.__p, self.__a, self.__b, - self.__h, ) diff --git a/src/ecdsa/test_ellipticcurve.py b/src/ecdsa/test_ellipticcurve.py index 33248f6e..0e07c25e 100644 --- a/src/ecdsa/test_ellipticcurve.py +++ b/src/ecdsa/test_ellipticcurve.py @@ -97,6 +97,13 @@ def test_conflation_curves(self): self.assertDictEqual({c_23: None}, {eq1: None}) self.assertIn(eq2, {eq3: None}) + def test___str__(self): + self.assertEqual(str(self.c_23), "CurveFp(p=23, a=1, b=1)") + + def test___str___with_cofactor(self): + c = CurveFp(23, 1, 1, 4) + self.assertEqual(str(c), "CurveFp(p=23, a=1, b=1, h=4)") + class TestPoint(unittest.TestCase): @classmethod From 8125ca2b8e1eb24d159f80f7c734b63b40621c09 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 28 Dec 2023 20:47:43 +0100 Subject: [PATCH 15/29] test coverage for CurveEdTw() __str__ --- src/ecdsa/ellipticcurve.py | 10 ++++++++-- src/ecdsa/test_ellipticcurve.py | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/ecdsa/ellipticcurve.py b/src/ecdsa/ellipticcurve.py index 942094a0..067a149c 100644 --- a/src/ecdsa/ellipticcurve.py +++ b/src/ecdsa/ellipticcurve.py @@ -225,11 +225,17 @@ def cofactor(self): return self.__h def __str__(self): - return "CurveEdTw(p={0}, a={1}, d={2}, h={3})".format( + if self.__h is not None: + return "CurveEdTw(p={0}, a={1}, d={2}, h={3})".format( + self.__p, + self.__a, + self.__d, + self.__h, + ) + return "CurveEdTw(p={0}, a={1}, d={2})".format( self.__p, self.__a, self.__d, - self.__h, ) diff --git a/src/ecdsa/test_ellipticcurve.py b/src/ecdsa/test_ellipticcurve.py index 0e07c25e..6e121546 100644 --- a/src/ecdsa/test_ellipticcurve.py +++ b/src/ecdsa/test_ellipticcurve.py @@ -14,7 +14,7 @@ except ImportError: # pragma: no cover HC_PRESENT = False from .numbertheory import inverse_mod -from .ellipticcurve import CurveFp, INFINITY, Point +from .ellipticcurve import CurveFp, INFINITY, Point, CurveEdTw HYP_SETTINGS = {} @@ -105,6 +105,19 @@ def test___str___with_cofactor(self): self.assertEqual(str(c), "CurveFp(p=23, a=1, b=1, h=4)") +class TestCurveEdTw(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.c_23 = CurveEdTw(23, 1, 1) + + def test___str__(self): + self.assertEqual(str(self.c_23), "CurveEdTw(p=23, a=1, d=1)") + + def test___str___with_cofactor(self): + c = CurveEdTw(23, 1, 1, 4) + self.assertEqual(str(c), "CurveEdTw(p=23, a=1, d=1, h=4)") + + class TestPoint(unittest.TestCase): @classmethod def setUpClass(cls): From 6b56848ba366465013dec7fdc7f52089481a3260 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 28 Dec 2023 21:45:45 +0100 Subject: [PATCH 16/29] ensure CurveEdTw hashability --- src/ecdsa/test_ellipticcurve.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ecdsa/test_ellipticcurve.py b/src/ecdsa/test_ellipticcurve.py index 6e121546..9fac7eed 100644 --- a/src/ecdsa/test_ellipticcurve.py +++ b/src/ecdsa/test_ellipticcurve.py @@ -117,6 +117,12 @@ def test___str___with_cofactor(self): c = CurveEdTw(23, 1, 1, 4) self.assertEqual(str(c), "CurveEdTw(p=23, a=1, d=1, h=4)") + def test_usability_in_a_hashed_collection_curves(self): + {self.c_23: None} + + def test_hashability_curves(self): + hash(self.c_23) + class TestPoint(unittest.TestCase): @classmethod From 27b13109404449bd05a674c86cfea391099b77c0 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 28 Dec 2023 21:54:12 +0100 Subject: [PATCH 17/29] more complete test coverage for Jacobi points without hypothesis --- src/ecdsa/test_jacobi.py | 67 ++++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/src/ecdsa/test_jacobi.py b/src/ecdsa/test_jacobi.py index ed598bb3..efe298e8 100644 --- a/src/ecdsa/test_jacobi.py +++ b/src/ecdsa/test_jacobi.py @@ -23,6 +23,7 @@ generator_brainpoolp160r1, curve_brainpoolp160r1, generator_112r2, + curve_112r2, ) from .numbertheory import inverse_mod from .util import randrange @@ -433,30 +434,78 @@ def test_add_different_scale_points_static(self): self.assertEqual(c, x + y) - def test_add_different_scale_points_static(self): + def test_add_different_points_same_scale_static(self): j_g = generator_brainpoolp160r1 p = curve_brainpoolp160r1.p() a = j_g * 11 a.scale() - z1 = 13 + b = j_g * 12 + z = 13 x = PointJacobi( curve_brainpoolp160r1, - a.x() * z1**2 % p, - a.y() * z1**3 % p, - z1, + a.x() * z**2 % p, + a.y() * z**3 % p, + z, ) - z2 = 29 y = PointJacobi( curve_brainpoolp160r1, - a.x() * z2**2 % p, - a.y() * z2**3 % p, - z2, + b.x() * z**2 % p, + b.y() * z**3 % p, + z, + ) + + c = a + b + + self.assertEqual(c, x + y) + + def test_add_same_point_different_scale_second_z_1_static(self): + j_g = generator_112r2 + p = curve_112r2.p() + z = 11 + a = j_g * z + a.scale() + + x = PointJacobi( + curve_112r2, + a.x() * z**2 % p, + a.y() * z**3 % p, + z, + ) + y = PointJacobi( + curve_112r2, + a.x(), + a.y(), + 1, ) c = a + a self.assertEqual(c, x + y) + def test_add_to_infinity_static(self): + j_g = generator_112r2 + + z = 11 + a = j_g * z + a.scale() + + b = -a + + x = PointJacobi( + curve_112r2, + a.x(), + a.y(), + 1, + ) + y = PointJacobi( + curve_112r2, + b.x(), + b.y(), + 1, + ) + + self.assertEqual(INFINITY, x + y) + def test_add_point_3_times(self): j_g = PointJacobi.from_affine(generator_256) From 55d73214d372ff41e6c7901170ecb3b5785eb0ad Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Thu, 28 Dec 2023 23:28:13 +0100 Subject: [PATCH 18/29] mark multithreaded tests as slow --- src/ecdsa/test_jacobi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ecdsa/test_jacobi.py b/src/ecdsa/test_jacobi.py index efe298e8..2100cc16 100644 --- a/src/ecdsa/test_jacobi.py +++ b/src/ecdsa/test_jacobi.py @@ -640,6 +640,7 @@ def test_pickle(self): pj = PointJacobi(curve=CurveFp(23, 1, 1, 1), x=2, y=3, z=1, order=1) self.assertEqual(pickle.loads(pickle.dumps(pj)), pj) + @pytest.mark.slow @settings(**NO_OLD_SETTINGS) @pytest.mark.skipif( platform.python_implementation() == "PyPy", @@ -678,6 +679,7 @@ def runner(generator): generator_112r2._PointJacobi__precompute, ) + @pytest.mark.slow @pytest.mark.skipif( platform.system() == "Windows" or platform.python_implementation() == "PyPy", From 0cfc121ca29de56cb6159ab09e5a285d1a05524a Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 29 Dec 2023 01:07:13 +0100 Subject: [PATCH 19/29] mark all openssl calling tests as slow --- src/ecdsa/test_pyecdsa.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/ecdsa/test_pyecdsa.py b/src/ecdsa/test_pyecdsa.py index 579a5738..069f3e0c 100644 --- a/src/ecdsa/test_pyecdsa.py +++ b/src/ecdsa/test_pyecdsa.py @@ -1002,6 +1002,7 @@ def test_lengths(curve): assert len(sig) == curve.signature_length +@pytest.mark.slow class OpenSSL(unittest.TestCase): # test interoperability with OpenSSL tools. Note that openssl's ECDSA # sign/verify arguments changed between 0.9.8 and 1.0.0: the early @@ -1046,6 +1047,7 @@ def get_openssl_messagedigest_arg(self, hash_name): def test_from_openssl_secp112r1(self): return self.do_test_from_openssl(SECP112r1) + @pytest.mark.slow @pytest.mark.skipif( "secp112r2" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp112r2", @@ -1053,6 +1055,7 @@ def test_from_openssl_secp112r1(self): def test_from_openssl_secp112r2(self): return self.do_test_from_openssl(SECP112r2) + @pytest.mark.slow @pytest.mark.skipif( "secp128r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp128r1", @@ -1060,6 +1063,7 @@ def test_from_openssl_secp112r2(self): def test_from_openssl_secp128r1(self): return self.do_test_from_openssl(SECP128r1) + @pytest.mark.slow @pytest.mark.skipif( "secp160r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp160r1", @@ -1067,6 +1071,7 @@ def test_from_openssl_secp128r1(self): def test_from_openssl_secp160r1(self): return self.do_test_from_openssl(SECP160r1) + @pytest.mark.slow @pytest.mark.skipif( "prime192v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime192v1", @@ -1194,6 +1199,7 @@ def test_from_openssl_brainpoolp384r1(self): def test_from_openssl_brainpoolp512r1(self): return self.do_test_from_openssl(BRAINPOOLP512r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP160t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP160t1", @@ -1201,6 +1207,7 @@ def test_from_openssl_brainpoolp512r1(self): def test_from_openssl_brainpoolp160t1(self): return self.do_test_from_openssl(BRAINPOOLP160t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP192t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP192t1", @@ -1208,6 +1215,7 @@ def test_from_openssl_brainpoolp160t1(self): def test_from_openssl_brainpoolp192t1(self): return self.do_test_from_openssl(BRAINPOOLP192t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP224t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP224t1", @@ -1215,6 +1223,7 @@ def test_from_openssl_brainpoolp192t1(self): def test_from_openssl_brainpoolp224t1(self): return self.do_test_from_openssl(BRAINPOOLP224t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP256t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP256t1", @@ -1222,6 +1231,7 @@ def test_from_openssl_brainpoolp224t1(self): def test_from_openssl_brainpoolp256t1(self): return self.do_test_from_openssl(BRAINPOOLP256t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP320t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP320t1", @@ -1229,6 +1239,7 @@ def test_from_openssl_brainpoolp256t1(self): def test_from_openssl_brainpoolp320t1(self): return self.do_test_from_openssl(BRAINPOOLP320t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP384t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP384t1", @@ -1236,6 +1247,7 @@ def test_from_openssl_brainpoolp320t1(self): def test_from_openssl_brainpoolp384t1(self): return self.do_test_from_openssl(BRAINPOOLP384t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP512t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP512t1", @@ -1303,6 +1315,7 @@ def do_test_from_openssl(self, curve, hash_name="SHA1"): def test_to_openssl_secp112r1(self): self.do_test_to_openssl(SECP112r1) + @pytest.mark.slow @pytest.mark.skipif( "secp112r2" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp112r2", @@ -1310,6 +1323,7 @@ def test_to_openssl_secp112r1(self): def test_to_openssl_secp112r2(self): self.do_test_to_openssl(SECP112r2) + @pytest.mark.slow @pytest.mark.skipif( "secp128r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp128r1", @@ -1317,6 +1331,7 @@ def test_to_openssl_secp112r2(self): def test_to_openssl_secp128r1(self): self.do_test_to_openssl(SECP128r1) + @pytest.mark.slow @pytest.mark.skipif( "secp160r1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support secp160r1", @@ -1324,6 +1339,7 @@ def test_to_openssl_secp128r1(self): def test_to_openssl_secp160r1(self): self.do_test_to_openssl(SECP160r1) + @pytest.mark.slow @pytest.mark.skipif( "prime192v1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support prime192v1", @@ -1451,6 +1467,7 @@ def test_to_openssl_brainpoolp384r1(self): def test_to_openssl_brainpoolp512r1(self): self.do_test_to_openssl(BRAINPOOLP512r1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP160t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP160t1", @@ -1458,6 +1475,7 @@ def test_to_openssl_brainpoolp512r1(self): def test_to_openssl_brainpoolp160t1(self): self.do_test_to_openssl(BRAINPOOLP160t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP192t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP192t1", @@ -1465,6 +1483,7 @@ def test_to_openssl_brainpoolp160t1(self): def test_to_openssl_brainpoolp192t1(self): self.do_test_to_openssl(BRAINPOOLP192t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP224t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP224t1", @@ -1472,6 +1491,7 @@ def test_to_openssl_brainpoolp192t1(self): def test_to_openssl_brainpoolp224t1(self): self.do_test_to_openssl(BRAINPOOLP224t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP256t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP256t1", @@ -1479,6 +1499,7 @@ def test_to_openssl_brainpoolp224t1(self): def test_to_openssl_brainpoolp256t1(self): self.do_test_to_openssl(BRAINPOOLP256t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP320t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP320t1", @@ -1486,6 +1507,7 @@ def test_to_openssl_brainpoolp256t1(self): def test_to_openssl_brainpoolp320t1(self): self.do_test_to_openssl(BRAINPOOLP320t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP384t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP384t1", @@ -1493,6 +1515,7 @@ def test_to_openssl_brainpoolp320t1(self): def test_to_openssl_brainpoolp384t1(self): self.do_test_to_openssl(BRAINPOOLP384t1) + @pytest.mark.slow @pytest.mark.skipif( "brainpoolP512t1" not in OPENSSL_SUPPORTED_CURVES, reason="system openssl does not support brainpoolP512t1", @@ -1633,6 +1656,7 @@ def do_eddsa_test_to_openssl(self, curve): # in practice at least OpenSSL 3.0.0 is needed to make EdDSA signatures # earlier versions support EdDSA only in X.509 certificates + @pytest.mark.slow @pytest.mark.skipif( "ed25519" not in OPENSSL_SUPPORTED_TYPES, reason="system openssl does not support signing with Ed25519", @@ -1640,6 +1664,7 @@ def do_eddsa_test_to_openssl(self, curve): def test_to_openssl_ed25519(self): return self.do_eddsa_test_to_openssl(Ed25519) + @pytest.mark.slow @pytest.mark.skipif( "ed448" not in OPENSSL_SUPPORTED_TYPES, reason="system openssl does not support signing with Ed448", @@ -1683,6 +1708,7 @@ def do_eddsa_test_from_openssl(self, curve): shutil.rmtree("t") + @pytest.mark.slow @pytest.mark.skipif( "ed25519" not in OPENSSL_SUPPORTED_TYPES, reason="system openssl does not support signing with Ed25519", @@ -1690,6 +1716,7 @@ def do_eddsa_test_from_openssl(self, curve): def test_from_openssl_ed25519(self): return self.do_eddsa_test_from_openssl(Ed25519) + @pytest.mark.slow @pytest.mark.skipif( "ed448" not in OPENSSL_SUPPORTED_TYPES, reason="system openssl does not support signing with Ed448", From 00563070d1726052eed1c9945a7d9be78a506ec7 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 29 Dec 2023 01:46:50 +0100 Subject: [PATCH 20/29] quicker encode-decode tests --- src/ecdsa/test_eddsa.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ecdsa/test_eddsa.py b/src/ecdsa/test_eddsa.py index d398d033..1a35fb32 100644 --- a/src/ecdsa/test_eddsa.py +++ b/src/ecdsa/test_eddsa.py @@ -1,3 +1,4 @@ +import sys import pickle import hashlib import pytest @@ -663,7 +664,10 @@ def test_invalid_r_value(self): HYP_SETTINGS = dict() -HYP_SETTINGS["max_examples"] = 10 +if "--fast" in sys.argv: + HYP_SETTINGS["max_examples"] = 2 +else: + HYP_SETTINGS["max_examples"] = 10 @settings(**HYP_SETTINGS) From 159be34d0d5858d6218a298ff9e97c2e601577f4 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 29 Dec 2023 19:40:03 +0100 Subject: [PATCH 21/29] update examples for the new cosmic-ray --- cosmic-ray-12way.sh | 13 +++++++++++++ cosmic-ray-12way.toml | 28 ++++++++++++++++++++++++++++ cosmic-ray.sh | 6 +++--- cosmic-ray.toml | 15 ++++----------- 4 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 cosmic-ray-12way.sh create mode 100644 cosmic-ray-12way.toml diff --git a/cosmic-ray-12way.sh b/cosmic-ray-12way.sh new file mode 100644 index 00000000..7b16fb2d --- /dev/null +++ b/cosmic-ray-12way.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +cosmic-ray init cosmic-ray-12way.toml session.sqlite +cosmic-ray baseline --session-file session.baseline.sqlite cosmic-ray-12way.toml +cr-report --show-output session.baseline.sqlite +# some mutations cause huge memory use, so put it in a cgroup +# systemd-run --user --scope -p MemoryMax=8G -p MemoryHigh=8G cr-http-workers cosmic-ray-12way.toml . +cr-http-workers cosmic-ray-12way.toml . +cosmic-ray exec cosmic-ray-12way.toml session.sqlite +cr-report session.sqlite +cr-html session.sqlite > session.html +cr-rate --estimate --fail-over 29 --confidence 99.9 session.sqlite diff --git a/cosmic-ray-12way.toml b/cosmic-ray-12way.toml new file mode 100644 index 00000000..78ee99c3 --- /dev/null +++ b/cosmic-ray-12way.toml @@ -0,0 +1,28 @@ +[cosmic-ray] +module-path = "src" +timeout = 20.0 +excluded-modules = ['src/ecdsa/_sha3.py', 'src/ecdsa/_version.py', 'src/ecdsa/test*'] +test-command = "pytest -x --fast -m 'not slow' src/" + +[cosmic-ray.distributor] +name = "http" + +[cosmic-ray.distributor.http] +worker-urls = [ + "http://localhost:9870", + "http://localhost:9871", + "http://localhost:9872", + "http://localhost:9873", + "http://localhost:9874", + "http://localhost:9875", + "http://localhost:9876", + "http://localhost:9877", + "http://localhost:9878", + "http://localhost:9879", + "http://localhost:9880", + "http://localhost:9881", + "http://localhost:9882" +] + +[cosmic-ray.filters.git-filter] +branch = "master" diff --git a/cosmic-ray.sh b/cosmic-ray.sh index 384750a3..51181a3a 100644 --- a/cosmic-ray.sh +++ b/cosmic-ray.sh @@ -2,11 +2,11 @@ set -e cosmic-ray init cosmic-ray.toml session.sqlite -cosmic-ray baseline --report session.sqlite +cosmic-ray baseline --session-file session.baseline.sqlite cosmic-ray.toml cr-report --show-output session.baseline.sqlite # some mutations cause huge memory use, so put it in a cgroup -# systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec session.sqlite -cosmic-ray exec session.sqlite +# systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec cosmic-ray.toml session.sqlite +cosmic-ray exec cosmic-ray.toml session.sqlite cr-report session.sqlite cr-html session.sqlite > session.html cr-rate --estimate --fail-over 29 --confidence 99.9 session.sqlite diff --git a/cosmic-ray.toml b/cosmic-ray.toml index 09f364aa..31c95f9f 100644 --- a/cosmic-ray.toml +++ b/cosmic-ray.toml @@ -1,18 +1,11 @@ [cosmic-ray] module-path = "src" -python-version = "" timeout = 20.0 -exclude-modules = ['src/ecdsa/_version.py', 'src/ecdsa/test*'] +exclude-modules = ['src/ecdsa/_sha3.py', 'src/ecdsa/_version.py', 'src/ecdsa/test*'] test-command = "pytest -x --fast -m 'not slow' src/" -[cosmic-ray.execution-engine] +[cosmic-ray.distributor] name = "local" -[cosmic-ray.cloning] -method = "copy" -commands = [] - -[cosmic-ray.interceptors] -enabled = [ "spor", "pragma_no_mutate", "operators-filter",] - -[cosmic-ray.operators-filter] +[cosmic-ray.filters.git-filter] +branch = "master" From ad564c04ad1a3309079be11a86c4a656c7eebdb6 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 29 Dec 2023 21:33:30 +0100 Subject: [PATCH 22/29] don't run speed tests on mutation and codechecks tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3b218cc..d9d2eac1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -260,7 +260,7 @@ jobs: run: pip list - name: Test native speed # tox uses pip to install dependenceis, so it breaks on py2.6 - if: ${{ !contains(matrix.tox-env, 'gmpy') && matrix.python-version != '2.6'}} + if: ${{ !contains(matrix.tox-env, 'gmpy') && matrix.python-version != '2.6' && ! matrix.mutation && !contains(matrix.tox-env, 'codechecks') }} run: tox -e speed - name: Test speed with gmpy if: ${{ contains(matrix.tox-env, 'gmpyp') }} From da9d8bb93ecdcccec21416f23895a3a131bd5c58 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 29 Dec 2023 20:03:37 +0100 Subject: [PATCH 23/29] add mutation testing to PR --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9d2eac1..5e7b8fea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,10 @@ jobs: os: ubuntu-latest python-version: 3.9 tox-env: codechecks + - name: mutation testing + os: ubuntu-latest + python-version: '3.11' + mutation: 'true' steps: - uses: actions/checkout@v2 if: ${{ !matrix.container }} @@ -149,6 +153,11 @@ jobs: run: | apt-get update apt-get install -y git make python-is-python3 python3 curl wget python3-distutils python3-pip + + - name: Dependencies for mutation testing + if: ${{ matrix.mutation == 'true' }} + run: | + sudo apt-get install -y sqlite3 - name: workaround git failures with py3.10 run: | git config --global --add safe.directory /__w/python-ecdsa/python-ecdsa @@ -256,6 +265,10 @@ jobs: else pip install -r build-requirements.txt; fi + - name: Install mutation testing dependencies + if: ${{ matrix.mutation == 'true' }} + run: | + pip install cosmic-ray - name: Display installed python package versions run: pip list - name: Test native speed @@ -277,6 +290,26 @@ jobs: - name: Run unit tests if: ${{ matrix.tox-env }} run: tox -e ${{ matrix.tox-env }} + - name: Init for mutation testing in PR + if: ${{ matrix.mutation == 'true' && github.event.pull_request }} + run: | + cosmic-ray init cosmic-ray.toml session-vs-master.sqlite + git branch master origin/master + git rev-parse HEAD + git rev-parse master + cr-filter-git --config cosmic-ray.toml session-vs-master.sqlite + - name: Exec mutation testing for PR + if: ${{ matrix.mutation == 'true' && github.event.pull_request }} + run: | + cosmic-ray exec cosmic-ray.toml session-vs-master.sqlite + - name: Check test coverage for PR + if: ${{ matrix.mutation == 'true' && github.event.pull_request }} + run: | + # remove not-executed results + sqlite3 session-vs-master.sqlite "DELETE from work_results WHERE work_results.worker_outcome = 'NO_TEST' OR work_results.worker_outcome = 'SKIPPED'" + cr-report session-vs-master.sqlite | tail -n 5 + # check if executed have at most 5% survival rate + cr-rate --fail-over 5 session-vs-master.sqlite - name: instrumental test coverage on PR if: ${{ contains(matrix.opt-deps, 'instrumental') && github.event.pull_request }} env: From 6f09910c89996313fff1223ccff34e362986a7f1 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 29 Dec 2023 22:53:44 +0100 Subject: [PATCH 24/29] correct exclude configuration --- cosmic-ray.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmic-ray.toml b/cosmic-ray.toml index 31c95f9f..6f1c54fa 100644 --- a/cosmic-ray.toml +++ b/cosmic-ray.toml @@ -1,7 +1,7 @@ [cosmic-ray] module-path = "src" timeout = 20.0 -exclude-modules = ['src/ecdsa/_sha3.py', 'src/ecdsa/_version.py', 'src/ecdsa/test*'] +excluded-modules = ['src/ecdsa/_sha3.py', 'src/ecdsa/_version.py', 'src/ecdsa/test*'] test-command = "pytest -x --fast -m 'not slow' src/" [cosmic-ray.distributor] From 09ffa0f326bf66f273d4c7152cdcff88b3beb663 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Fri, 29 Dec 2023 23:01:49 +0100 Subject: [PATCH 25/29] log number of tests to run --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e7b8fea..4b15bc54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,9 +295,8 @@ jobs: run: | cosmic-ray init cosmic-ray.toml session-vs-master.sqlite git branch master origin/master - git rev-parse HEAD - git rev-parse master cr-filter-git --config cosmic-ray.toml session-vs-master.sqlite + cr-report session-vs-master.sqlite | tail -n 5 - name: Exec mutation testing for PR if: ${{ matrix.mutation == 'true' && github.event.pull_request }} run: | @@ -306,7 +305,7 @@ jobs: if: ${{ matrix.mutation == 'true' && github.event.pull_request }} run: | # remove not-executed results - sqlite3 session-vs-master.sqlite "DELETE from work_results WHERE work_results.worker_outcome = 'NO_TEST' OR work_results.worker_outcome = 'SKIPPED'" + sqlite3 session-vs-master.sqlite "DELETE from work_results WHERE work_results.worker_outcome = 'SKIPPED'" cr-report session-vs-master.sqlite | tail -n 5 # check if executed have at most 5% survival rate cr-rate --fail-over 5 session-vs-master.sqlite From cae3082265ef725584f5c0a27a965bd53b8a6669 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 31 Dec 2023 02:10:30 +0100 Subject: [PATCH 26/29] Run mutation testing on multiple workers in parallel --- .github/workflows/ci.yml | 255 +++++++++++++++++++++++++++++++++++++++ sql/combine.sql | 7 ++ sql/create_to_del.sql | 2 + sql/shard-db.sql | 3 + 4 files changed, 267 insertions(+) create mode 100644 sql/combine.sql create mode 100644 sql/create_to_del.sql create mode 100644 sql/shard-db.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b15bc54..562ca7ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -369,3 +369,258 @@ jobs: COVERALLS_SERVICE_NAME: github run: | coveralls --finish + + mutation-prepare: + name: Prepare job files for the mutation runners + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + if: ${{ !matrix.container }} + with: + fetch-depth: 50 + - name: save session objects + uses: actions/cache@v3 + with: + path: | + sessions/ + key: sessions-${{ github.sha }} + - name: Install cosmic-ray + run: | + pip3 install cosmic-ray + - name: Install dependencies + run: | + sudo apt-get install -y sqlite3 + - name: Display Python version + run: python -c "import sys; print(sys.version)" + - name: Create list of mutations + run: | + cosmic-ray init cosmic-ray.toml session.sqlite + - name: Log number of jobs created + run: | + cr-report session.sqlite | tail -n 3 + - name: Split up mutations to workers + run: | + cp session.sqlite session-to_del.sqlite + sqlite3 session-to_del.sqlite "$(cat sql/create_to_del.sql)" + mkdir sessions + for i in $(seq 0 19); do + sed "s/%SHARD%/$i/" < sql/shard-db.sql > shard.sql + cp session-to_del.sqlite session-$i.sqlite + sqlite3 session-$i.sqlite "$(cat shard.sql)" + mv session-$i.sqlite sessions/ + done + mutation-execute: + name: Execute mutation testing + needs: mutation-prepare + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + include: + - name: 0 + - name: 1 + - name: 2 + - name: 3 + - name: 4 + - name: 5 + - name: 6 + - name: 7 + - name: 8 + - name: 9 + - name: 10 + - name: 11 + - name: 12 + - name: 13 + - name: 14 + - name: 15 + - name: 16 + - name: 17 + - name: 18 + - name: 19 + steps: + - uses: actions/checkout@v2 + if: ${{ !matrix.container }} + with: + fetch-depth: 1 + - name: Session objects + uses: actions/cache@v3 + with: + path: | + sessions/ + key: sessions-${{ github.sha }} + - name: Session done objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-${{ matrix.name }}-done.sqlite + key: sessions-${{ github.sha }}-${{ matrix.name }}-done + - name: Install gmpy2 dependencies + run: sudo apt-get install -y libmpfr-dev libmpc-dev + - name: Install gmpy2 + run: pip install gmpy2 + - name: Install build dependencies + run: | + pip install -r build-requirements.txt + pip install cosmic-ray + - name: Run mutation testing + run: | + cp sessions/session-${{ matrix.name }}.sqlite session.sqlite + systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec cosmic-ray.toml session.sqlite & + cosmic_pid=$! + for i in $(seq 1 10); do + echo $i + sleep 60 + done + kill $cosmic_pid + mkdir sessions-done/ + cp session.sqlite sessions-done/session-${{ matrix.name }}-done.sqlite + - name: Report executed + run: | + cr-report session.sqlite | tail -n 3 + mutation-combine: + name: Combine mutation testing results + needs: mutation-execute + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + if: ${{ !matrix.container }} + with: + fetch-depth: 1 + - name: Session done 0 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-0-done.sqlite + key: sessions-${{ github.sha }}-0-done + - name: Session done 1 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-1-done.sqlite + key: sessions-${{ github.sha }}-1-done + - name: Session done 2 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-2-done.sqlite + key: sessions-${{ github.sha }}-2-done + - name: Session done 3 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-3-done.sqlite + key: sessions-${{ github.sha }}-3-done + - name: Session done 4 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-4-done.sqlite + key: sessions-${{ github.sha }}-4-done + - name: Session done 5 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-5-done.sqlite + key: sessions-${{ github.sha }}-5-done + - name: Session done 6 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-6-done.sqlite + key: sessions-${{ github.sha }}-6-done + - name: Session done 7 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-7-done.sqlite + key: sessions-${{ github.sha }}-7-done + - name: Session done 8 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-8-done.sqlite + key: sessions-${{ github.sha }}-8-done + - name: Session done 9 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-9-done.sqlite + key: sessions-${{ github.sha }}-9-done + - name: Session done 10 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-10-done.sqlite + key: sessions-${{ github.sha }}-10-done + - name: Session done 11 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-11-done.sqlite + key: sessions-${{ github.sha }}-11-done + - name: Session done 12 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-12-done.sqlite + key: sessions-${{ github.sha }}-12-done + - name: Session done 13 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-13-done.sqlite + key: sessions-${{ github.sha }}-13-done + - name: Session done 14 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-14-done.sqlite + key: sessions-${{ github.sha }}-14-done + - name: Session done 15 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-15-done.sqlite + key: sessions-${{ github.sha }}-15-done + - name: Session done 16 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-16-done.sqlite + key: sessions-${{ github.sha }}-16-done + - name: Session done 17 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-17-done.sqlite + key: sessions-${{ github.sha }}-17-done + - name: Session done 18 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-18-done.sqlite + key: sessions-${{ github.sha }}-18-done + - name: Session done 19 objects + uses: actions/cache@v3 + with: + path: | + sessions-done/session-19-done.sqlite + key: sessions-${{ github.sha }}-19-done + - name: Install cosmic-ray + run: | + pip3 install cosmic-ray + - name: Install dependencies + run: | + sudo apt-get install -y sqlite3 + - name: Combine worker results + run: | + cp sessions-done/session-0-done.sqlite session.sqlite + for i in $(seq 1 19); do + cp sessions-done/session-$i-done.sqlite session-to_merge.sqlite + sqlite3 session.sqlite "$(cat sql/combine.sql)" + done + - name: Report executed + run: | + cr-report session.sqlite | tail -n 3 + - name: Check mutation level + run: cr-rate --estimate --fail-over 32 --confidence 99.9 session.sqlite diff --git a/sql/combine.sql b/sql/combine.sql new file mode 100644 index 00000000..d2a4490d --- /dev/null +++ b/sql/combine.sql @@ -0,0 +1,7 @@ +attach 'session-to_merge.sqlite' as toMerge; +BEGIN; + insert into work_items select * from toMerge.work_items; + insert into mutation_specs select * from toMerge.mutation_specs; + insert into work_results select * from toMerge.work_results; +COMMIT; +detach toMerge; diff --git a/sql/create_to_del.sql b/sql/create_to_del.sql new file mode 100644 index 00000000..956d1a28 --- /dev/null +++ b/sql/create_to_del.sql @@ -0,0 +1,2 @@ +create table to_del (job_id VARCHAR NOT NULL, id INTEGER PRIMARY KEY); +insert into to_del select *, ROWID from work_items; diff --git a/sql/shard-db.sql b/sql/shard-db.sql new file mode 100644 index 00000000..7460bd31 --- /dev/null +++ b/sql/shard-db.sql @@ -0,0 +1,3 @@ +delete from mutation_specs where job_id in (select job_id from to_del where to_del.ID % 20 != %SHARD%); +delete from work_items where job_id in (select job_id from to_del where to_del.ID % 20 != %SHARD%); +drop table to_del; From 8df3bb87ef21c85f6946e4a2222dd14b1a47c3b4 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 31 Dec 2023 12:18:07 +0100 Subject: [PATCH 27/29] create mutation score badge --- .github/workflows/ci.yml | 15 ++++++++++++++- README.md | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 562ca7ea..ce34b5cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -622,5 +622,18 @@ jobs: - name: Report executed run: | cr-report session.sqlite | tail -n 3 - - name: Check mutation level + - name: Check survival estimate run: cr-rate --estimate --fail-over 32 --confidence 99.9 session.sqlite + - name: Get mutation score + run: echo "MUT_SCORE=$(python -c 'print(100-$(cr-rate session.sqlite))')" >> $GITHUB_ENV + - name: Create mutation score badge + uses: schneegans/dynamic-badges-action@v1.4.0 + with: + auth: ${{ secrets.GIST_SECRET }} + gistID: 9b6ca1f3410207fbeca785a178781651 + filename: python-ecdsa-mutation-score.json + label: mutation score + message: ${{ env.MUT_SCORE }}% + valColorRange: ${{ env.MUT_SCORE }} + maxColorRange: 100 + minColorRange: 0 diff --git a/README.md b/README.md index fbb41a63..6c43f85d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Documentation Status](https://readthedocs.org/projects/ecdsa/badge/?version=latest)](https://ecdsa.readthedocs.io/en/latest/?badge=latest) [![Coverage Status](https://coveralls.io/repos/github/tlsfuzzer/python-ecdsa/badge.svg?branch=master)](https://coveralls.io/github/tlsfuzzer/python-ecdsa?branch=master) ![condition coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/tomato42/9b6ca1f3410207fbeca785a178781651/raw/python-ecdsa-condition-coverage.json) +![mutation score](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/tomato42/9b6ca1f3410207fbeca785a178781651/raw/python-ecdsa-mutation-score.json) [![CodeQL](https://github.com/tlsfuzzer/python-ecdsa/actions/workflows/codeql.yml/badge.svg)](https://github.com/tlsfuzzer/python-ecdsa/actions/workflows/codeql.yml) [![Latest Version](https://img.shields.io/pypi/v/ecdsa.svg?style=flat)](https://pypi.python.org/pypi/ecdsa/) ![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=flat) From 1364e899d62b5962aeea3fcd91d31f798337e0b7 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 31 Dec 2023 12:42:52 +0100 Subject: [PATCH 28/29] error tolerance --- .github/workflows/ci.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce34b5cf..6594746a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -414,7 +414,7 @@ jobs: needs: mutation-prepare runs-on: ubuntu-latest strategy: - fail-fast: true + fail-fast: false matrix: include: - name: 0 @@ -467,7 +467,7 @@ jobs: cp sessions/session-${{ matrix.name }}.sqlite session.sqlite systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec cosmic-ray.toml session.sqlite & cosmic_pid=$! - for i in $(seq 1 10); do + for i in $(seq 1 2); do echo $i sleep 60 done @@ -616,16 +616,17 @@ jobs: run: | cp sessions-done/session-0-done.sqlite session.sqlite for i in $(seq 1 19); do - cp sessions-done/session-$i-done.sqlite session-to_merge.sqlite - sqlite3 session.sqlite "$(cat sql/combine.sql)" + cp sessions-done/session-$i-done.sqlite session-to_merge.sqlite && sqlite3 session.sqlite "$(cat sql/combine.sql)" || true done - name: Report executed run: | cr-report session.sqlite | tail -n 3 - - name: Check survival estimate - run: cr-rate --estimate --fail-over 32 --confidence 99.9 session.sqlite + - name: Log survival estimate + run: cr-rate --estimate --fail-over 32 --confidence 99.9 session.sqlite || true - name: Get mutation score - run: echo "MUT_SCORE=$(python -c 'print(100-$(cr-rate session.sqlite))')" >> $GITHUB_ENV + run: | + echo "print(100-$(cr-rate session.sqlite))" > print-score.py + echo "MUT_SCORE=$(python print-score.py)" >> $GITHUB_ENV - name: Create mutation score badge uses: schneegans/dynamic-badges-action@v1.4.0 with: From c4c6ff6c13ce71726dcf9140f6fbc87aa4f6d8e0 Mon Sep 17 00:00:00 2001 From: Hubert Kario Date: Sun, 31 Dec 2023 15:44:58 +0100 Subject: [PATCH 29/29] long execution for mutation testing --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6594746a..f25221df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -467,7 +467,7 @@ jobs: cp sessions/session-${{ matrix.name }}.sqlite session.sqlite systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec cosmic-ray.toml session.sqlite & cosmic_pid=$! - for i in $(seq 1 2); do + for i in $(seq 1 10); do echo $i sleep 60 done