Add mutation testing to CI #270
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GitHub CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| test: | |
| runs-on: ${{ matrix.os }} | |
| container: ${{ matrix.container }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: py2.6 | |
| os: ubuntu-latest | |
| container: centos:6 | |
| python-version: 2.6 | |
| - name: py3.10 with ossl3.0 | |
| os: ubuntu-latest | |
| container: ubuntu:22.04 | |
| python-version: "3.10" | |
| tox-env: py310 | |
| - name: py2.7 | |
| os: ubuntu-20.04 | |
| python-version: 2.7 | |
| tox-env: py27 | |
| - name: py2.7 with old gmpy | |
| os: ubuntu-20.04 | |
| python-version: 2.7 | |
| tox-env: py27_old_gmpy | |
| - name: py2.7 with old gmpy2 | |
| os: ubuntu-20.04 | |
| python-version: 2.7 | |
| tox-env: py27_old_gmpy2 | |
| - name: py2.7 with old six | |
| os: ubuntu-20.04 | |
| python-version: 2.7 | |
| tox-env: py27_old_six | |
| - name: py2.7 with gmpy | |
| os: ubuntu-20.04 | |
| python-version: 2.7 | |
| tox-env: gmpypy27 | |
| - name: py2.7 with gmpy2 | |
| os: ubuntu-20.04 | |
| python-version: 2.7 | |
| tox-env: gmpy2py27 | |
| - name: py3.5 | |
| os: ubuntu-20.04 | |
| python-version: 3.5 | |
| tox-env: py35 | |
| - name: py3.6 | |
| os: ubuntu-20.04 | |
| python-version: 3.6 | |
| tox-env: py36 | |
| - name: py3.7 | |
| os: ubuntu-latest | |
| python-version: 3.7 | |
| tox-env: py37 | |
| - name: py3.8 | |
| os: ubuntu-latest | |
| python-version: 3.8 | |
| tox-env: py38 | |
| - name: py3.9 | |
| os: ubuntu-latest | |
| python-version: 3.9 | |
| tox-env: py39 | |
| - name: py3.10 | |
| os: ubuntu-latest | |
| python-version: '3.10' | |
| tox-env: py310 | |
| - name: py3.10 with gmpy | |
| os: ubuntu-latest | |
| python-version: '3.10' | |
| tox-env: gmpypy310 | |
| - name: py3.10 with gmpy2 | |
| os: ubuntu-latest | |
| python-version: '3.10' | |
| tox-env: gmpy2py310 | |
| - name: py3.11 | |
| os: ubuntu-latest | |
| python-version: '3.11' | |
| tox-env: py311 | |
| - name: py3.12 | |
| os: ubuntu-latest | |
| python-version: '3.12.0-beta.1' | |
| tox-env: py312 | |
| - name: pypy | |
| os: ubuntu-latest | |
| python-version: pypy-2.7 | |
| tox-env: pypy | |
| - name: pypy3 | |
| os: ubuntu-latest | |
| python-version: pypy-3.7 | |
| tox-env: pypy3 | |
| # special configurations | |
| - name: py2.7 with instrumental | |
| os: ubuntu-20.04 | |
| python-version: 2.7 | |
| opt-deps: ['instrumental'] | |
| - name: code checks | |
| os: ubuntu-latest | |
| python-version: 3.9 | |
| tox-env: codechecks | |
| - name: mutation testing | |
| os: ubuntu-latest | |
| python-version: '3.11' | |
| mutation: 'true' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| if: ${{ !matrix.container }} | |
| with: | |
| fetch-depth: 50 | |
| - uses: actions/checkout@v1 | |
| # centos 6 doesn't have glibc new enough for the nodejs used by v2 | |
| if: ${{ matrix.container }} | |
| with: | |
| fetch-depth: 50 | |
| - name: Ensure dependencies on CentOS | |
| if: ${{ matrix.container == 'centos:6' }} | |
| run: | | |
| ls /etc/yum.repos.d/ | |
| cat /etc/yum.repos.d/CentOS-Base.repo | |
| rm /etc/yum.repos.d/CentOS-Base.repo | |
| cat > /etc/yum.repos.d/CentOS-Base.repo <<EOF | |
| [base] | |
| name=CentOS-$releasever - Base | |
| baseurl=https://vault.centos.org/6.10/os/x86_64/ | |
| gpgcheck=1 | |
| gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 | |
| [updates] | |
| name=CentOS-$releasever - Updates | |
| baseurl=https://vault.centos.org/6.10/updates/x86_64/ | |
| gpgcheck=1 | |
| gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 | |
| [extras] | |
| name=CentOS-$releasever - Extras | |
| baseurl=https://vault.centos.org/6.10/extras/x86_64/ | |
| gpgcheck=1 | |
| gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 | |
| EOF | |
| echo installing | |
| yum clean all | |
| yum repolist all | |
| yum install -y git make python curl gcc libffi-devel python-devel glibc-devel openssl-devel wget | |
| - name: Ensure dependencies on Ubuntu 22.04 | |
| if: ${{ matrix.container == 'ubuntu:22.04' }} | |
| run: | | |
| apt-get update | |
| apt-get install -y git make python-is-python3 python3 curl wget python3-distutils python3-pip | |
| - name: Dependencies for mutation testing | |
| if: ${{ matrix.mutation == 'true' }} | |
| run: | | |
| sudo apt-get install -y sqlite3 | |
| - name: workaround git failures with py3.10 | |
| run: | | |
| git config --global --add safe.directory /__w/python-ecdsa/python-ecdsa | |
| - name: Verify git status | |
| run: | | |
| git status | |
| git remote -v | |
| - name: Ensure we have baseline branch for quality coverage | |
| run: git fetch origin master:refs/remotes/origin/master | |
| - name: Set up Python ${{ matrix.python-version }} | |
| # we use containers to use the native python version from them | |
| if: ${{ !matrix.container && matrix.python-version != '2.7' }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Ensure python 2.7 | |
| if: matrix.python-version == '2.7' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| python2.7 python2.7-dev python-pip-whl | |
| sudo ln -sf python2.7 /usr/bin/python | |
| export PYTHONPATH=`echo /usr/share/python-wheels/pip-*py2*.whl` | |
| sudo --preserve-env=PYTHONPATH python -m pip install --upgrade pip setuptools wheel | |
| sudo chown -R $USER /usr/local/lib/python2.7 | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Display installed python package versions | |
| run: | | |
| pip list || : | |
| - name: Ensure working pip on 3.3 | |
| if: ${{ matrix.python-version == '3.3' }} | |
| run: | | |
| curl -o get-pip.py https://bootstrap.pypa.io/pip/3.3/get-pip.py | |
| python get-pip.py | |
| - name: Ensure working pip on 2.6 | |
| if: ${{ matrix.python-version == '2.6' }} | |
| run: | | |
| # pypi deprecated SNI-less access to the CDN, so we have to download | |
| # the packages manually | |
| curl -o get-pip.py https://bootstrap.pypa.io/pip/2.6/get-pip.py | |
| wget https://files.pythonhosted.org/packages/ac/95/a05b56bb975efa78d3557efa36acaf9cf5d2fd0ee0062060493687432e03/pip-9.0.3-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/27/f6/fabfc9c71c9b1b99d2ec4768a6e1f73b2e924f51c89d436302b8c2a25459/setuptools-36.8.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/8a/e9/8468cd68b582b06ef554be0b96b59f59779627131aad48f8a5bce4b13450/wheel-0.29.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl | |
| python get-pip.py pip-9.0.3-py2.py3-none-any.whl setuptools-36.8.0-py2.py3-none-any.whl wheel-0.29.0-py2.py3-none-any.whl argparse-1.4.0-py2.py3-none-any.whl | |
| pip list | |
| wget https://files.pythonhosted.org/packages/3b/7e/293d19ccd106119e35db4bf3e111b1895098f618b455b758aa636496cf03/setuptools-28.8.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/83/53/e120833aa2350db333df89a40dea3b310dd9dabf6f29eaa18934a597dc79/wheel-0.30.0a0-py2.py3-none-any.whl | |
| pip install setuptools-28.8.0-py2.py3-none-any.whl wheel-0.30.0a0-py2.py3-none-any.whl | |
| - name: Install instrumental | |
| if: ${{ contains(matrix.opt-deps, 'instrumental') }} | |
| run: pip install instrumental | |
| - name: Install gmpy | |
| if: ${{ contains(matrix.tox-env, 'gmpyp') }} | |
| run: pip install gmpy | |
| - name: Install gmpy2 dependencies | |
| if: ${{ contains(matrix.tox-env, 'gmpy2') || contains(matrix.tox-env, 'instrumental') || matrix.mutation == 'true' }} | |
| run: sudo apt-get install -y libmpfr-dev libmpc-dev | |
| - name: Install gmpy2 | |
| if: ${{ contains(matrix.tox-env, 'gmpy2') || contains(matrix.tox-env, 'instrumental') || matrix.mutation == 'true' }} | |
| run: pip install gmpy2 | |
| - name: Install build dependencies (2.6) | |
| if: ${{ matrix.python-version == '2.6' }} | |
| run: | | |
| wget https://files.pythonhosted.org/packages/1d/4e/20c679f8c5948f7c48591fde33d442e716af66a31a88f5791850a75041eb/tox-2.9.1-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/d9/9d/077582a4c6d771e3b742631e6c1d3688f48210626de488e032776242b3f2/inflect-0.3.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/79/db/7c0cfe4aa8341a5fab4638952520d8db6ab85ff84505e12c00ea311c3516/pyOpenSSL-17.5.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/2d/bf/960e5a422db3ac1a5e612cb35ca436c3fc985ed4b7ed13a1b4879006f450/cffi-1.13.2.tar.gz | |
| wget https://files.pythonhosted.org/packages/4b/2a/0276479a4b3caeb8a8c1af2f8e4355746a97fab05a372e4a2c6a6b876165/idna-2.7-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/a8/5a/5cf074e1c6681dcbb4e640113f58bed16955e7da9a6c8090b518031775e7/hypothesis-2.0.0.tar.gz | |
| wget https://files.pythonhosted.org/packages/85/d5/818d0e603685c4a613d56f065a721013e942088047ff1027a632948bdae6/coverage-4.5.4.tar.gz | |
| wget https://files.pythonhosted.org/packages/e6/35/f187bdf23be87092bd0f1200d43d23076cee4d0dec109f195173fd3ebc79/mock-2.0.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/ed/ea/e20b5cbebf45d3096e8138ab74eda139595d827677f38e9dd543e6015bdf/virtualenv-15.2.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/53/67/9620edf7803ab867b175e4fd23c7b8bd8eba11cb761514dcd2e726ef07da/py-1.4.34-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/65/26/32b8464df2a97e6dd1b656ed26b2c194606c16fe163c695a992b36c11cdf/six-1.13.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/4d/d1/e478b8a33230f85f38e35b386376fbd115219de2a2c4c8783610851ad1c3/pluggy-0.5.2-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/78/c5/7188f15a92413096c93053d5304718e1f6ba88b818357d05d19250ebff85/cryptography-2.1.4.tar.gz | |
| wget https://files.pythonhosted.org/packages/8c/2d/aad7f16146f4197a11f8e91fb81df177adcc2073d36a17b1491fd09df6ed/pycparser-2.18.tar.gz | |
| wget https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz | |
| wget https://files.pythonhosted.org/packages/65/47/7e02164a2a3db50ed6d8a6ab1d6d60b69c4c3fdf57a284257925dfc12bda/requests-2.19.1-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/17/0a/6ac05a3723017a967193456a2efa0aa9ac4b51456891af1e2353bb9de21e/traceback2-1.4.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/31/77/3781f65cafe55480b56914def99022a5d2965a4bb269655c89ef2f1de3cd/importlib-1.0.4.zip | |
| wget https://files.pythonhosted.org/packages/7d/b0/23d19892f8d91ec9c5b8a2035659bce23587fed419d68fa3d70b6abf8bcd/Counter-1.0.0.tar.gz | |
| wget https://files.pythonhosted.org/packages/69/cb/f5be453359271714c01b9bd06126eaf2e368f1fddfff30818754b5ac2328/funcsigs-1.0.2-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/fb/48/69046506f6ac61c1eaa9a0d42d22d54673b69e176d30ca98e3f61513e980/pbr-5.5.1-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/b5/a8/56be92dcd4a5bf1998705a9b4028249fe7c9a035b955fe93b6a3e5b829f8/asn1crypto-1.4.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/6f/2c/a9386903ece2ea85e9807e0e062174dc26fdce8b05f216d00491be29fad5/enum34-1.1.10-py2-none-any.whl | |
| wget https://files.pythonhosted.org/packages/c2/f8/49697181b1651d8347d24c095ce46c7346c37335ddc7d255833e7cde674d/ipaddress-1.0.23-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/bd/c9/6fdd990019071a4a32a5e7cb78a1d92c53851ef4f56f62a3486e6a7d8ffb/urllib3-1.23-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/5e/a0/5f06e1e1d463903cf0c0eebeb751791119ed7a4b3737fdc9a77f1cdfb51f/certifi-2020.12.5-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/c7/a3/c5da2a44c85bfbb6eebcfc1dde24933f8704441b98fdde6528f4831757a6/linecache2-1.0.0-py2.py3-none-any.whl | |
| wget https://files.pythonhosted.org/packages/53/25/ef88e8e45db141faa9598fbf7ad0062df8f50f881a36ed6a0073e1572126/ordereddict-1.1.tar.gz | |
| wget https://files.pythonhosted.org/packages/ef/41/d8a61f1b2ba308e96b36106e95024977e30129355fd12087f23e4b9852a1/pytest-3.2.5-py2.py3-none-any.whl | |
| pip install pycparser-2.18.tar.gz importlib-1.0.4.zip Counter-1.0.0.tar.gz tox-2.9.1-py2.py3-none-any.whl inflect-0.3.0-py2.py3-none-any.whl pyOpenSSL-17.5.0-py2.py3-none-any.whl cffi-1.13.2.tar.gz idna-2.7-py2.py3-none-any.whl unittest2-1.1.0-py2.py3-none-any.whl hypothesis-2.0.0.tar.gz coverage-4.5.4.tar.gz mock-2.0.0-py2.py3-none-any.whl virtualenv-15.2.0-py2.py3-none-any.whl py-1.4.34-py2.py3-none-any.whl six-1.13.0-py2.py3-none-any.whl pluggy-0.5.2-py2.py3-none-any.whl cryptography-2.1.4.tar.gz docopt-0.6.2.tar.gz requests-2.19.1-py2.py3-none-any.whl traceback2-1.4.0-py2.py3-none-any.whl funcsigs-1.0.2-py2.py3-none-any.whl pbr-5.5.1-py2.py3-none-any.whl asn1crypto-1.4.0-py2.py3-none-any.whl enum34-1.1.10-py2-none-any.whl ipaddress-1.0.23-py2.py3-none-any.whl chardet-3.0.4-py2.py3-none-any.whl urllib3-1.23-py2.py3-none-any.whl certifi-2020.12.5-py2.py3-none-any.whl linecache2-1.0.0-py2.py3-none-any.whl ordereddict-1.1.tar.gz pytest-3.2.5-py2.py3-none-any.whl git+https://github.com/tomato42/coveralls-python.git@add-py26#egg=coveralls | |
| - name: Install build dependencies | |
| if: ${{ matrix.python-version != '2.6' }} | |
| run: | | |
| PYTHON_VERSION=${{ matrix.python-version }} | |
| PYTHON_VERSION=${PYTHON_VERSION#pypy-} | |
| if [[ -e build-requirements-${PYTHON_VERSION}.txt ]]; then | |
| pip install -r build-requirements-${PYTHON_VERSION}.txt; | |
| else | |
| pip install -r build-requirements.txt; | |
| fi | |
| - name: Install mutation testing dependencies | |
| if: ${{ matrix.mutation == 'true' }} | |
| run: | | |
| pip install https://github.com/sixty-north/cosmic-ray/archive/master.zip | |
| pip install pytest-timeout | |
| - name: Display installed python package versions | |
| run: pip list | |
| - name: Test native speed | |
| # tox uses pip to install dependenceis, so it breaks on py2.6 | |
| if: ${{ !contains(matrix.tox-env, 'gmpy') && matrix.python-version != '2.6' && ! matrix.mutation && !contains(matrix.tox-env, 'codechecks') }} | |
| run: tox -e speed | |
| - name: Test speed with gmpy | |
| if: ${{ contains(matrix.tox-env, 'gmpyp') }} | |
| run: tox -e speedgmpy | |
| - name: Test speed with gmpy2 | |
| if: ${{ contains(matrix.tox-env, 'gmpy2') }} | |
| run: tox -e speedgmpy2 | |
| - name: Run unit tests (2.6) | |
| if: ${{ matrix.python-version == '2.6' }} | |
| run: | | |
| # because tox uses pip, and pip is broken on py2.6, we need run the | |
| # tests directly on the system | |
| coverage run --branch -m pytest | |
| - name: Run unit tests | |
| if: ${{ matrix.tox-env }} | |
| run: tox -e ${{ matrix.tox-env }} | |
| - name: Init for mutation testing in PR | |
| if: ${{ matrix.mutation == 'true' && github.event.pull_request }} | |
| run: | | |
| cosmic-ray init cosmic-ray.toml session-vs-master.sqlite | |
| git branch master origin/master | |
| cr-filter-git --config cosmic-ray.toml session-vs-master.sqlite | |
| cr-report session-vs-master.sqlite | tail -n 3 | |
| - name: Exec mutation testing for PR | |
| if: ${{ matrix.mutation == 'true' && github.event.pull_request }} | |
| run: | | |
| systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray --verbosity INFO exec cosmic-ray.toml session-vs-master.sqlite & | |
| cosmic_pid=$! | |
| for i in $(seq 1 600); do | |
| # wait for test execution at most 10 minutes | |
| kill -s 0 $cosmic_pid || break | |
| sleep 1 | |
| done | |
| kill $cosmic_pid || true | |
| wait $cosmic_pid || true | |
| - name: Check test coverage for PR | |
| if: ${{ matrix.mutation == 'true' && github.event.pull_request }} | |
| run: | | |
| # remove not-executed results | |
| sqlite3 session-vs-master.sqlite "DELETE from work_results WHERE work_results.worker_outcome = 'SKIPPED'" | |
| cr-report session-vs-master.sqlite | tail -n 3 | |
| - name: Generate html report | |
| if: ${{ matrix.mutation == 'true' && github.event.pull_request }} | |
| run: | | |
| cr-html session-vs-master.sqlite > cosmic-ray.html | |
| - name: Archive mutation testing results | |
| if: ${{ matrix.mutation == 'true' && github.event.pull_request }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: mutation-PR-coverage-report | |
| path: cosmic-ray.html | |
| - name: Check test coverage for PR | |
| if: ${{ matrix.mutation == 'true' && github.event.pull_request }} | |
| run: | | |
| # check if executed have at most 50% survival rate | |
| cr-rate --estimate --confidence 99.9 --fail-over 50 session-vs-master.sqlite | |
| - name: instrumental test coverage on PR | |
| if: ${{ contains(matrix.opt-deps, 'instrumental') && github.event.pull_request }} | |
| env: | |
| BASE_REF: ${{ github.event.pull_request.base.ref }} | |
| run: | | |
| git fetch origin $BASE_REF | |
| MERGE_BASE=$(git merge-base origin/$BASE_REF HEAD) | |
| echo "MERGE_BASE:" $MERGE_BASE | |
| git checkout $MERGE_BASE | |
| instrumental -t ecdsa -i '.*test_.*|.*_version|.*_compat|.*_sha3' `which pytest` src/ecdsa/test*.py | |
| instrumental -f .instrumental.cov -s | |
| instrumental -f .instrumental.cov -s | python diff-instrumental.py --save .diff-instrumental | |
| git checkout $GITHUB_SHA | |
| instrumental -t ecdsa -i '.*test_.*|.*_version|.*_compat|.*_sha3' `which pytest` src/ecdsa/test*.py | |
| instrumental -f .instrumental.cov -sr | |
| instrumental -f .instrumental.cov -s | python diff-instrumental.py --read .diff-instrumental --fail-under 70 --max-difference -0.1 | |
| - name: instrumental test coverage on push | |
| if: ${{ contains(matrix.opt-deps, 'instrumental') && !github.event.pull_request }} | |
| run: | | |
| instrumental -t ecdsa -i '.*test_.*|.*_version|.*_compat|.*_sha3' `which pytest` src/ecdsa | |
| instrumental -f .instrumental.cov -s | |
| # just log the values when merging | |
| instrumental -f .instrumental.cov -s | python diff-instrumental.py | |
| echo "COND_COV=$(instrumental -f .instrumental.cov -s | python diff-instrumental.py --raw)" >> $GITHUB_ENV | |
| - name: Create condition coverage badge | |
| uses: schneegans/[email protected] | |
| if: ${{ contains(matrix.opt-deps, 'instrumental') && !github.event.pull_request }} | |
| with: | |
| auth: ${{ secrets.GIST_SECRET }} | |
| gistID: 9b6ca1f3410207fbeca785a178781651 | |
| filename: python-ecdsa-condition-coverage.json | |
| label: condition coverage | |
| message: ${{ env.COND_COV }}% | |
| valColorRange: ${{ env.COND_COV }} | |
| maxColorRange: 100 | |
| minColorRange: 0 | |
| - name: Publish coverage to Coveralls | |
| if: ${{ !matrix.opt-deps && matrix.tox-env != 'codechecks' }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| COVERALLS_FLAG_NAME: ${{ matrix.name }} | |
| COVERALLS_PARALLEL: true | |
| COVERALLS_SERVICE_NAME: github | |
| run: coveralls | |
| coveralls: | |
| name: Indicate completion to coveralls.io | |
| needs: test | |
| runs-on: ubuntu-latest | |
| container: python:3-slim | |
| steps: | |
| - name: Install coveralls | |
| run: | | |
| pip3 install --upgrade coveralls | |
| - name: Send "finished" signal to coveralls | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| COVERALLS_SERVICE_NAME: github | |
| run: | | |
| coveralls --finish | |
| mutation-prepare: | |
| name: Prepare job files for the mutation runners | |
| # use runner minutes on mutation testing only after the PR passed basic | |
| # testing | |
| needs: coveralls | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| if: ${{ !matrix.container }} | |
| with: | |
| fetch-depth: 50 | |
| - name: save session objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions/ | |
| key: sessions-${{ github.sha }} | |
| - name: Install cosmic-ray | |
| run: | | |
| pip3 install https://github.com/sixty-north/cosmic-ray/archive/master.zip | |
| pip install pytest-timeout | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install -y sqlite3 | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Create list of mutations | |
| run: | | |
| cosmic-ray init cosmic-ray.toml session.sqlite | |
| - name: Log number of jobs created | |
| run: | | |
| cr-report session.sqlite | tail -n 3 | |
| - name: Split up mutations to workers | |
| run: | | |
| cp session.sqlite session-to_del.sqlite | |
| sqlite3 session-to_del.sqlite "$(cat sql/create_to_del.sql)" | |
| mkdir sessions | |
| for i in $(seq 0 19); do | |
| sed "s/%SHARD%/$i/" < sql/shard-db.sql > shard.sql | |
| cp session-to_del.sqlite session-$i.sqlite | |
| sqlite3 session-$i.sqlite "$(cat shard.sql)" | |
| mv session-$i.sqlite sessions/ | |
| done | |
| mutation-execute: | |
| name: Execute mutation testing | |
| needs: mutation-prepare | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: 0 | |
| - name: 1 | |
| - name: 2 | |
| - name: 3 | |
| - name: 4 | |
| - name: 5 | |
| - name: 6 | |
| - name: 7 | |
| - name: 8 | |
| - name: 9 | |
| - name: 10 | |
| - name: 11 | |
| - name: 12 | |
| - name: 13 | |
| - name: 14 | |
| - name: 15 | |
| - name: 16 | |
| - name: 17 | |
| - name: 18 | |
| - name: 19 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| if: ${{ !matrix.container }} | |
| with: | |
| fetch-depth: 1 | |
| - name: Session objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions/ | |
| key: sessions-${{ github.sha }} | |
| - name: Session done objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-${{ matrix.name }}-done.sqlite | |
| key: sessions-${{ github.sha }}-${{ matrix.name }}-done | |
| - name: Install gmpy2 dependencies | |
| run: sudo apt-get install -y libmpfr-dev libmpc-dev | |
| - name: Install gmpy2 | |
| run: pip install gmpy2 | |
| - name: Install build dependencies | |
| run: | | |
| pip install -r build-requirements.txt | |
| pip install https://github.com/sixty-north/cosmic-ray/archive/master.zip | |
| pip install pytest-timeout | |
| - name: Run mutation testing | |
| run: | | |
| cp sessions/session-${{ matrix.name }}.sqlite session.sqlite | |
| systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec cosmic-ray.toml session.sqlite & | |
| cosmic_pid=$! | |
| for i in $(seq 1 10); do | |
| echo $i | |
| sleep 60 | |
| done | |
| kill $cosmic_pid | |
| mkdir sessions-done/ | |
| cp session.sqlite sessions-done/session-${{ matrix.name }}-done.sqlite | |
| - name: Report executed | |
| run: | | |
| cr-report session.sqlite | tail -n 3 | |
| mutation-combine: | |
| name: Combine mutation testing results | |
| needs: mutation-execute | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| if: ${{ !matrix.container }} | |
| with: | |
| fetch-depth: 1 | |
| - name: Session done 0 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-0-done.sqlite | |
| key: sessions-${{ github.sha }}-0-done | |
| - name: Session done 1 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-1-done.sqlite | |
| key: sessions-${{ github.sha }}-1-done | |
| - name: Session done 2 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-2-done.sqlite | |
| key: sessions-${{ github.sha }}-2-done | |
| - name: Session done 3 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-3-done.sqlite | |
| key: sessions-${{ github.sha }}-3-done | |
| - name: Session done 4 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-4-done.sqlite | |
| key: sessions-${{ github.sha }}-4-done | |
| - name: Session done 5 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-5-done.sqlite | |
| key: sessions-${{ github.sha }}-5-done | |
| - name: Session done 6 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-6-done.sqlite | |
| key: sessions-${{ github.sha }}-6-done | |
| - name: Session done 7 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-7-done.sqlite | |
| key: sessions-${{ github.sha }}-7-done | |
| - name: Session done 8 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-8-done.sqlite | |
| key: sessions-${{ github.sha }}-8-done | |
| - name: Session done 9 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-9-done.sqlite | |
| key: sessions-${{ github.sha }}-9-done | |
| - name: Session done 10 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-10-done.sqlite | |
| key: sessions-${{ github.sha }}-10-done | |
| - name: Session done 11 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-11-done.sqlite | |
| key: sessions-${{ github.sha }}-11-done | |
| - name: Session done 12 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-12-done.sqlite | |
| key: sessions-${{ github.sha }}-12-done | |
| - name: Session done 13 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-13-done.sqlite | |
| key: sessions-${{ github.sha }}-13-done | |
| - name: Session done 14 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-14-done.sqlite | |
| key: sessions-${{ github.sha }}-14-done | |
| - name: Session done 15 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-15-done.sqlite | |
| key: sessions-${{ github.sha }}-15-done | |
| - name: Session done 16 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-16-done.sqlite | |
| key: sessions-${{ github.sha }}-16-done | |
| - name: Session done 17 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-17-done.sqlite | |
| key: sessions-${{ github.sha }}-17-done | |
| - name: Session done 18 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-18-done.sqlite | |
| key: sessions-${{ github.sha }}-18-done | |
| - name: Session done 19 objects | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| sessions-done/session-19-done.sqlite | |
| key: sessions-${{ github.sha }}-19-done | |
| - name: Install cosmic-ray | |
| run: | | |
| pip3 install https://github.com/sixty-north/cosmic-ray/archive/master.zip | |
| pip install pytest-timeout | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install -y sqlite3 | |
| - name: Combine worker results | |
| run: | | |
| cp sessions-done/session-0-done.sqlite session.sqlite | |
| for i in $(seq 1 19); do | |
| cp sessions-done/session-$i-done.sqlite session-to_merge.sqlite && sqlite3 session.sqlite "$(cat sql/combine.sql)" || true | |
| done | |
| - name: Report executed | |
| run: | | |
| cr-report session.sqlite | tail -n 3 | |
| - name: Generate html report | |
| run: | | |
| cr-html session.sqlite > cosmic-ray.html | |
| - name: Archive mutation testing results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: mutation-coverage-report | |
| path: cosmic-ray.html | |
| - name: Get mutation score | |
| run: | | |
| echo "print('{0:.2f}'.format(100-$(cr-rate session.sqlite)))" > print-score.py | |
| echo "MUT_SCORE=$(python print-score.py)" >> $GITHUB_ENV | |
| - name: Create mutation score badge | |
| if: ${{ !github.event.pull_request }} | |
| uses: schneegans/[email protected] | |
| with: | |
| auth: ${{ secrets.GIST_SECRET }} | |
| gistID: 9b6ca1f3410207fbeca785a178781651 | |
| filename: python-ecdsa-mutation-score.json | |
| label: mutation score | |
| message: ${{ env.MUT_SCORE }}% | |
| valColorRange: ${{ env.MUT_SCORE }} | |
| maxColorRange: 100 | |
| minColorRange: 0 | |
| - name: Check survival estimate | |
| run: cr-rate --estimate --fail-over 32 --confidence 99.9 session.sqlite |