run-tests.sh

#!/usr/bin/env bash
# run-tests.sh
#
# Sets up and runs unittests in each python environment it detects in the build files.


# Print if log level is CRITICAL
critical() {
  log 0 "(C): " "$@"
}

# Print if log level is ERROR
error() {
  log 1 "(E): " "$@"
}

# Print if log level is WARN
warn() {
  log 2 "(W): " "$@"
}

# Print if log level is INFO
info() {
  log 3 "(I): " "$@"
}

# Print if log level is DEBUG
debug() {
  log 4 "(D): " "$@"
}

# Print if log level is >= $1
log() {
  __this_level=$1
  shift 1
  __prefix="$1"
  shift 1
  if [ $__log_level -ge $__this_level ] ; then
    echo -n "$__prefix"
    "$@"
  else
    (2>&1 >/dev/null "$@")
  fi
}


# Figure out the full interpreter path for a python version
__get_python_interpreter_path() {
  __python=python$1
  (> /dev/null which $__python 2>&1)
  if [ "$?" != "0" ] ; then
    if [ "$1" = "2" ] ; then
      __python=python
    fi
  fi
  (> /dev/null which $__python 2>&1)
  if [ "$?" = "0" ] ; then
    which $__python 
  else
    echo "NONE"
  fi
}


# Create a new virtual environment for a particular python version
__create_venv() {
  local __python
  __python=$(__get_python_interpreter_path $1)
  if [ "$__python" = "NONE" ] ; then
    info echo "    python$1 not found!"
    exit 1
  else
    __virtualenv=$(which virtualenv)
    __virtualenv_by_version=$(dirname $__python)/virtualenv
    if [ -e $__virtualenv_by_version ] ; then
      debug $__virtualenv_by_version -v -p $__python .venv$1
      rc=$?
    elif [  -e $__virtualenv ] ; then
      debug $__virtualenv -v -p $__python .venv$1
      rc=$?
    else
      info echo "    virtualenv executable not found for $__python!"
      exit 1
    fi
    info echo "    python$1 ($__python) -> .venv$1"
  fi
}


# Run PyLint
__do_pylint() {
  # Detect the test package
  debug echo "Detecting packages..."
  __packages="$(python -c 'import setuptools; print(" ".join(setuptools.find_packages()))')"
  debug echo $__packages
  # Append '/*' to each package name to hand off to pylint
  __pylint_directories=${__packages/ //* }/*
  __pylint_files="$(find . -name '*.py' ! -path '*.venv*' -printf '%p ')"
  debug echo "__pylint_files: $__pylint_files"
  __pylint_cmd="pylint $__pylint_files"
  debug echo "PyLint Command: '$__pylint_cmd'"
  debug echo
  info echo "PyLint: start ($PYLINT_LOG)"
  echo "Python $1" >> $PYLINT_LOG
  echo "" >> $PYLINT_LOG
  __pylint_out=$(2>&1 $__pylint_cmd)
  __pylint_E=$(echo "$__pylint_out" | grep -c "E:")
  __pylint_W=$(echo "$__pylint_out" | grep -c "W:")
  __pylint_C=$(echo "$__pylint_out" | grep -c "C:")
  __pylint_R=$(echo "$__pylint_out" | grep -c "R:")
  __pylint_F=$(echo "$__pylint_out" | grep -c "F:")
  echo "$__pylint_out" >> $PYLINT_LOG
  __summary="PyLint: done  ($__pylint_E errors, $__pylint_W warnings, $__pylint_C conventions, $__pylint_R refactors, $__pylint_F fatals)"
  echo "$__summary" >> $PYLINT_LOG
  echo "" >> $PYLINT_LOG
  echo "" >> $PYLINT_LOG
  __rating_line=$(echo "$__pylint_out" | grep "Your code has been rated at ")
  __rating=$(echo "$__rating_line" | cut -d ' ' -f7 | cut -d '/' -f1)
  info echo "  $__rating_line"
  info echo "$__summary"
  info echo
  rm -f $PYLINT_BADGE
  if [ $(2>/dev/null which anybadge) ] ; then
    anybadge -l pylint -v $__rating -f $PYLINT_BADGE 2=red 4=orange 8=yellow 10=green
  else
    warn echo "anybdage not found, PyLint badge not created."
  fi
}


# Run unit tests
__do_tests() {
  info echo "--Running Python$1 Tests--"
  # This is always printed... otherwise what is the point?
  coverage run $__branch -m unittest discover -s $__test_dir
  log 2 "" coverage report
  rm -f coverage.svg
  coverage-badge -o coverage.svg
  if [ "$__html_cov" = "1" ] ; then
    info echo "Generating html coverage into '$__html_cov_dir'"
    coverage html -d $__html_cov_dir
  fi
  info echo
  info echo "---------Complete---------"
  info echo
  info echo
}


# Go through a simple yaml file and grab all python versions from it
__parse_yaml() {
  # For Windows compatibility, replace any CRLF with LF
  local yaml=($(cat $1 | tr '\r\n' '\n'))
  debug echo "Parsing '$1'"
  local token
  local capturing=0
  local capture_next=0
  __versions=()
  for token in "${yaml[@]}" ; do
    debug echo "Token: '$token'"
    if [ "$token" = "python:" ] ; then
      capturing=1
    elif [ $capturing -eq 1 ] ; then
      if [ $capture_next -eq 1 ] ; then
        __versions+=($(echo "$token" | cut -c 2- | rev | cut -c 2- | rev))
        capture_next=0
      elif [ "$token" = "-" ] ; then
        capture_next=1
      else
        capturing=0
      fi
    fi
  done
}


__parse_rc() {
  # For Windows compatibility, replace any CRLF with LF
  local rc_file=($(cat $1 | tr '\r\n' '\n'))
  local token
  for token in "${rc_file[@]}" ; do
    args+=("$token")
  done
}


#
# Main body
#

__help="\
usage: run-tests.sh [options]
  -h --help                Display this help message.
  -v --verbose             Print everything!  Equivalent to --log 4
  -q --quiet               Produce only some output.  Equivalent to --log 3
  -s --silent              Only report errors.  Equivalent to --log 1
     --log <level>         Set logging level.
                             0 = CRITICAL
                             1 = ERROR
                             2 = WARNING
                             3 = INFO
                             4 = DEBUG
     --recreate-venvs      Force creation of Python Virtual Environments.
     --no-recreate-venvs   Do not force recreation of Python Virtual Environments.
     --pylint              Run PyLint static analysis.
     --no-pylint           Do not run PyLint static analysis.
     --branch              Measure branch coverage in addition to statement coverage.
     --no-branch           Do not measure branch coverage in addition to statement coverage.
     --htmlcov [dir]       Generate an HTML coverage report into 'dir' (default=htmlcov/).
     --test-dir <dir>      Do test discovery from <dir> (default=test/).
     --skip-tests          Skip unit testing, venv creation and PyLint will still run normally.
     --no-skip-tests       Do not skip unit testing."

# Initialize variables
__log_level=3
__recreate_venvs=0
__pylint=1
__branch=""
__html_cov=0
__html_cov_dir=""
__test_dir="test"
__skip_tests=0

# Handle .testrc and arguments
if [ -e ".testrc" ] ; then
  __parse_rc ".testrc"
fi

# Add command line args after the rc args
args+=("$@")


while [ "${args[0]}" != "" ] ; do
  #
  # Help
  #
  if [ "${args[0]}" = "-h" ] || [ "${args[0]}" = "--help" ] ; then
    echo "$__help"
    exit 0
  #
  # Logging
  #
  elif   [ "${args[0]}" = "-v" ] || [ "${args[0]}" = "--verbose" ] ; then
    __log_level=4
  elif [ "${args[0]}" = "-q" ] || [ "${args[0]}" = "--quiet" ] ; then
    __log_level=3
  elif [ "${args[0]}" = "-s" ] || [ "${args[0]}" = "--silent" ] ; then
    __log_level=1
  elif [ "${args[0]}" = "--log" ] ; then
    # Consume the arg following '--log'
    __log_level="${args[1]}"
    # Advance args
    args=(${args[@]:1})
  #
  # Virtual Environment Control
  #
  elif [ "${args[0]}" = "--recreate-venvs" ] ; then
    __recreate_venvs=1
  elif [ "${args[0]}" = "--no-recreate-venvs" ] ; then
    __recreate_venvs=0
  #
  # PyLint Controls
  #
  elif [ "${args[0]}" = "--pylint" ] ; then
    __pylint=1
  elif [ "${args[0]}" = "--no-pylint" ] ; then
    __pylint=0
  #
  # Coverage Controls
  #
  #   Measure branch coverage in addition to statement coverage.
  elif [ "${args[0]}" = "--branch" ] ; then
    __branch="--branch"
  #   Do not measure branch coverage in addition to statement coverage.
  elif [ "${args[0]}" = "--no-branch" ] ; then
    __branch=""
  #   Produce an HTML coverage report.
  elif [ "${args[0]}" = "--htmlcov" ] ; then
    __html_cov=1
    if [ "${args[1]}" = "" ] || [ "${args[1]:0:1}" = "-" ] ; then
      __html_cov_dir="htmlcov/"
    else
      # Consume the arg following '--htmlcov'
      __html_cov_dir="${args[1]}"
      # Advance args
      args=(${args[@]:1})
    fi
  #
  # Test Discovery Control
  #
  #   Test Discovery Start Directory
  elif [ "${args[0]}" = "--test-dir" ] ; then
    # Consume the arg following '--test-dir'
    __test_dir="${args[1]}"
    # Advance args
    args=(${args[@]:1})
  #
  # Skip Tests
  #
  elif [ "${args[0]}" = "--skip-tests" ] ; then
    __skip_tests=1
  elif [ "${args[0]}" = "--no-skip-tests" ] ; then
    __skip_tests=0
  #
  # Unrecognized Option/Argument
  #
  else
    echo "Unrecognized option '${args[0]}'"
    echo "$__help"
    exit 1
  fi
  # Advance args
  args=(${args[@]:1})
done


debug echo "Session Variables:"
debug echo "    __recreate_venvs  '$__recreate_venvs'"
debug echo "    __pylint          '$__pylint'"
debug echo "    __log_level       '$__log_level'"
debug echo "    __html_cov        '$__html_cov'"
debug echo "    __html_cov_dir    '$__html_cov_dir'"
debug echo "    __test_dir        '$__test_dir'"
debug echo "    __skip_tests      '$__skip_tests'"


# PyLint Variables
LOG_DIR=.
PYLINT_LOG=$LOG_DIR/pylint-report
PYLINT_BADGE=$LOG_DIR/pylint.svg


# Enable dot glob so we will see any "hidden" yml files
shopt -s dotglob
__versions=()
# Capture all the yml files in here
__yml_files=($(ls *.yml))
debug echo "__parse_yml $__yml_files"
# Parse each of them
for __file in "${__yml_files[@]}" ; do
  __parse_yaml $__file
done
# Disable dot glob
shopt -u dotglob
# Get the array of version numbers
# Print the versions, replace ' ' with '\n', sort the lines, 
__versions=($(echo "${__versions[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ' | uniq))
debug echo "Versions: '$__versions'"

info echo "Setting up Python virtual environments"

declare -a __pids
declare -a __rcs
for __i in "${!__versions[@]}" ; do
  __version=${__versions[$__i]}
  debug echo "Setting up for Python$__version"
  if [ "$__recreate_venvs" = "1" ] || [ ! -e .venv$__version ] ; then
    __create_venv $__version &
    __pids[$__i]=$!
  else
    info echo "    python$__version (existing virtualenv) -> .venv$__version" &
    __pids[$__i]=$!
  fi
done

for __i in "${!__pids[@]}" ; do
  wait ${__pids[$__i]}
  __rcs[$__i]=$?
done

info echo "done."
info echo

# If we're going to run PyLint, reset the log file
if [ "$__pylint" = "1" ] ; then
  echo "" > $PYLINT_LOG
fi

for __i in "${!__rcs[@]}" ; do
  if [ "${__rcs[$__i]}" = "0" ] ; then
    __version="${__versions[$__i]}"
    . .venv$__version/bin/activate
    if [ "$__pylint" = "1" ] ; then
      __do_pylint $__version
    fi
    if [ "$__skip_tests" = "1" ] ; then
      warn echo "Python$__version Unit Testing Skipped (--skip-tests flag set)"
    else
      # Install the test package, if we're doing the unit tests.
      debug pip install -r test_requirements.txt
      __do_tests $__version
    fi
  fi
done