Skip to content

Commit

Permalink
Merge pull request #141 from tomato42/instrumental
Browse files Browse the repository at this point in the history
Add condition coverage
  • Loading branch information
tomato42 authored Oct 22, 2019
2 parents 188f09e + 31a1f73 commit 4c92d31
Show file tree
Hide file tree
Showing 3 changed files with 125 additions and 3 deletions.
64 changes: 61 additions & 3 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@ language: python
cache: pip
before_cache:
- rm -f $HOME/.cache/pip/log/debug.log
# place the slowest (instrumental and py2.6) first
matrix:
include:
- python: 2.7
env: INSTRUMENTAL=yes
- python: 2.6
env: TOX_ENV=py26
- python: 2.7
Expand Down Expand Up @@ -39,13 +42,68 @@ matrix:
allow_failures:
- python: nightly

# for instrumental we're checking if the coverage changed from base branch
# so collect that info
before_install:
- |
echo -e "TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST\n" \
"TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG\n" \
"TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST\n" \
"TRAVIS_COMMIT=$TRAVIS_COMMIT\n" \
"TRAVIS_PYTHON_VERSION=$TRAVIS_PYTHON_VERSION"
- |
# workaround https://github.com/travis-ci/travis-ci/issues/2666
if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
URL="https://github.com/${TRAVIS_REPO_SLUG}/pull/${TRAVIS_PULL_REQUEST}.patch"
# `--location` makes curl follow redirects
PR_FIRST=$(curl --silent --show-error --location $URL | head -1 | grep -o -E '\b[0-9a-f]{40}\b' | tr -d '\n')
TRAVIS_COMMIT_RANGE=$PR_FIRST^..$TRAVIS_COMMIT
fi
# sanity check current commit
- git rev-parse HEAD
- echo "TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE"
- git fetch origin master:refs/remotes/origin/master


install:
- pip list
- if [[ -e build-requirements-${TRAVIS_PYTHON_VERSION}.txt ]]; then travis_retry pip install -r build-requirements-${TRAVIS_PYTHON_VERSION}.txt; else travis_retry pip install -r build-requirements.txt; fi
- |
if [[ -e build-requirements-${TRAVIS_PYTHON_VERSION}.txt ]]; then
travis_retry pip install -r build-requirements-${TRAVIS_PYTHON_VERSION}.txt;
else
travis_retry pip install -r build-requirements.txt;
fi
- if [[ $INSTRUMENTAL ]]; then travis_retry pip install instrumental; fi
- pip list
script:
- tox -e $TOX_ENV
- if [[ $TOX_ENV ]]; then tox -e $TOX_ENV; fi
- tox -e speed
- cp diff-instrumental.py diff-instrumental-2.py
- |
if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then
git checkout $PR_FIRST^
# exclude the super slow test_malformed_sigs.py, until #127 is merged
files="$(ls src/ecdsa/test*.py | grep -v test_malformed_sigs.py)"
instrumental -t ecdsa -i 'test.*|.*_version' `which pytest` $files
instrumental -f .instrumental.cov -s
instrumental -f .instrumental.cov -s | python diff-instrumental-2.py --save .diff-instrumental
git checkout $TRAVIS_COMMIT
instrumental -t ecdsa -i 'test.*|.*_version' `which pytest` $files
instrumental -f .instrumental.cov -sr
fi
- |
if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST == "false" ]]; then
# exclude the super slow test_malformed_sigs.py, until #127 is merged
files="$(ls src/ecdsa/test*.py | grep -v test_malformed_sigs.py)"
instrumental -t ecdsa -i 'test.*|.*_version' `which pytest` $files
instrumental -f .instrumental.cov -s
# just log the values when merging
instrumental -f .instrumental.cov -s | python diff-instrumental-2.py
fi
- |
if [[ $INSTRUMENTAL && $TRAVIS_PULL_REQUEST != "false" ]]; then
instrumental -f .instrumental.cov -s | python diff-instrumental-2.py --read .diff-instrumental --fail-under 70 --max-difference -0.1
fi
after_success:
- coveralls
- if [[ -z $INSTRUMENTAL ]]; then coveralls; fi

1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

[![build status](https://travis-ci.org/warner/python-ecdsa.png)](http://travis-ci.org/warner/python-ecdsa)
[![Coverage Status](https://coveralls.io/repos/warner/python-ecdsa/badge.svg)](https://coveralls.io/r/warner/python-ecdsa)
[![condition coverage](https://img.shields.io/badge/condition%20coverage-78%25-yellow)](https://travis-ci.org/warner/python-ecdsa/jobs/600079269#L1587)
[![Latest Version](https://img.shields.io/pypi/v/ecdsa.svg?style=flat)](https://pypi.python.org/pypi/ecdsa/)


Expand Down
63 changes: 63 additions & 0 deletions diff-instrumental.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
from __future__ import print_function
import sys
import getopt

fail_under = None
max_difference = 0
read_location = None
save_location = None

argv = sys.argv[1:]

opts, args = getopt.getopt(
argv, "s:r:",
["fail-under=", "max-difference=", "save=", "read="])
if args:
raise ValueError("Unexpected parameters: {0}".format(args))
for opt, arg in opts:
if opt == "-s" or opt == "--save":
save_location = arg
elif opt == "-r" or opt == "--read":
read_location = arg
elif opt == "--fail-under":
fail_under = float(arg)/100.0
elif opt == "--max-difference":
max_difference = float(arg)/100.0
else:
raise ValueError("Unknown option: {0}".format(opt))

total_hits = 0
total_count = 0

for line in sys.stdin.readlines():
if not line.startswith("ecdsa"):
continue

fields = line.split()
hit, count = fields[1].split('/')
total_hits += int(hit)
total_count += int(count)

coverage = total_hits * 1.0 / total_count

if read_location:
with open(read_location, "r") as f:
old_coverage = float(f.read())
print("Old coverage: {0:6.2f}%".format(old_coverage*100))

if save_location:
with open(save_location, "w") as f:
f.write("{0:1.40f}".format(coverage))

print("Coverage: {0:6.2f}%".format(coverage*100))

if read_location:
print("Difference: {0:6.2f}%".format((old_coverage - coverage)*100))

if fail_under and coverage < fail_under:
print("ERROR: Insufficient coverage.", file=sys.stderr)
sys.exit(1)

if read_location and coverage - old_coverage < max_difference:
print("ERROR: Too big decrease in coverage", file=sys.stderr)
sys.exit(1)

0 comments on commit 4c92d31

Please sign in to comment.