From 072404ea18540fc75b5c1e46de17fddfaf3ec9e5 Mon Sep 17 00:00:00 2001
From: Thomas Sibley <tsibley@fredhutch.org>
Date: Thu, 28 Apr 2022 15:33:15 -0700
Subject: [PATCH] Hoist coverage setup out of individual cram test files and
 into the CI workflow
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This avoids baking in coverage to the tests and lets it remain a
separate concern (e.g. CI's concern).  In my experience it's better to
have coverage tracing happen outside of tests (i.e. the tests are
unaware) rather than as integrated into the tests themselves.

Partially reverts commit 723d609a18edcfae6280a5c428b1eac8e2b81f02, which
added code for coverage to each cram test file.

I chose to use the github.workspace context var in the CI workflow so I
could use the "env:" block for the job step.  Alternatively, we could
use `realpath` to construct an `env AUGUR=… …` call chaining to the
`cram …` invocation, but that seemed slightly messier to me.
---
 .github/workflows/ci.yaml      | 4 ++++
 tests/builds/zika.t            | 3 +--
 tests/builds/zika_compressed.t | 3 +--
 tests/functional/ancestral.t   | 3 +--
 tests/functional/clades.t      | 5 ++---
 tests/functional/export_v2.t   | 3 +--
 tests/functional/filter.t      | 3 +--
 tests/functional/frequencies.t | 3 +--
 tests/functional/mask.t        | 3 +--
 tests/functional/parse.t       | 3 +--
 tests/functional/refine.t      | 3 +--
 tests/functional/traits.t      | 3 +--
 tests/functional/tree.t        | 3 +--
 13 files changed, 17 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2cb2d24a3..270fd53df 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -44,6 +44,10 @@ jobs:
     - run: conda list
     - run: pytest -c pytest.python3.ini --cov=augur
     - run: cram --shell=/bin/bash tests/functional/*.t tests/builds/*.t
+      env:
+        AUGUR: coverage run -a ${{ github.workspace }}/bin/augur
+        COVERAGE_FILE: ${{ github.workspace }}/.coverage
+        COVERAGE_RCFILE: ${{ github.workspace }}/.coveragerc
     - run: coverage xml
     - run: bash tests/builds/runner.sh
     - if: github.repository == 'nextstrain/augur' && matrix.python-version == '3.10' && matrix.biopython-version == ''
diff --git a/tests/builds/zika.t b/tests/builds/zika.t
index 70aa9bb0d..55f8b2c93 100644
--- a/tests/builds/zika.t
+++ b/tests/builds/zika.t
@@ -7,8 +7,7 @@ Running from the test data directory allows us to use relative paths that won't
   $ TEST_DATA_DIR="$TESTDIR/zika"
   $ mkdir -p "$TMP/out"
   $ pushd "$TEST_DATA_DIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../../bin/augur}"
 
 Parse a FASTA whose defline contains metadata into separate sequence and metadata files.
 
diff --git a/tests/builds/zika_compressed.t b/tests/builds/zika_compressed.t
index 8b0fba7dc..be39ce704 100644
--- a/tests/builds/zika_compressed.t
+++ b/tests/builds/zika_compressed.t
@@ -7,8 +7,7 @@ Running from the test data directory allows us to use relative paths that won't
   $ TEST_DATA_DIR="$TESTDIR/zika"
   $ mkdir -p "$TMP/out"
   $ pushd "$TEST_DATA_DIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../../bin/augur}"
 
 Parse a FASTA whose defline contains metadata into separate sequence and metadata files.
 
diff --git a/tests/functional/ancestral.t b/tests/functional/ancestral.t
index f51d2e6ca..649029879 100644
--- a/tests/functional/ancestral.t
+++ b/tests/functional/ancestral.t
@@ -1,8 +1,7 @@
 Integration tests for augur ancestral.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Infer ancestral sequences for the given tree and alignment.
 The default is to infer ambiguous bases, so there should not be N bases in the inferred output sequences.
diff --git a/tests/functional/clades.t b/tests/functional/clades.t
index 24e3e1d38..619c35d56 100644
--- a/tests/functional/clades.t
+++ b/tests/functional/clades.t
@@ -1,8 +1,7 @@
 Integration tests for augur clades.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Test augur clades with simple Zika input files and hierarchical clades.
 
@@ -14,4 +13,4 @@ Test augur clades with simple Zika input files and hierarchical clades.
 
   $ python3 "$TESTDIR/../../scripts/diff_jsons.py"  clades/clades.json "$TMP/clades.json" \
   >   --exclude-paths "root['generated_by']"
-  {}
+  {}
\ No newline at end of file
diff --git a/tests/functional/export_v2.t b/tests/functional/export_v2.t
index ea4c517c0..39cf13101 100644
--- a/tests/functional/export_v2.t
+++ b/tests/functional/export_v2.t
@@ -1,8 +1,7 @@
 Integration tests for augur export v2.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Minimal export
 
diff --git a/tests/functional/filter.t b/tests/functional/filter.t
index ff25defb7..120e57a93 100644
--- a/tests/functional/filter.t
+++ b/tests/functional/filter.t
@@ -1,8 +1,7 @@
 Integration tests for augur filter.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Filter with exclude query for two regions that comprise all but one strain.
 This filter should leave a single record from Oceania.
diff --git a/tests/functional/frequencies.t b/tests/functional/frequencies.t
index 196c09fe3..937e240d2 100644
--- a/tests/functional/frequencies.t
+++ b/tests/functional/frequencies.t
@@ -1,8 +1,7 @@
 Integration tests for augur frequencies.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Calculate KDE-based tip frequencies from a refined tree.
 Timepoints used to estimate frequencies (i.e., "pivots") get calculated from the range of dates in the given metadata.
diff --git a/tests/functional/mask.t b/tests/functional/mask.t
index f99bd3e89..905f2a78b 100644
--- a/tests/functional/mask.t
+++ b/tests/functional/mask.t
@@ -1,8 +1,7 @@
 Integration tests for augur mask.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Try masking a VCF without any specified mask.
 
diff --git a/tests/functional/parse.t b/tests/functional/parse.t
index 924be9caf..796adc5af 100644
--- a/tests/functional/parse.t
+++ b/tests/functional/parse.t
@@ -1,8 +1,7 @@
 Integration tests for augur parse.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Parse Zika sequences into sequences and metadata.
 
diff --git a/tests/functional/refine.t b/tests/functional/refine.t
index fa15e6707..7ceeb3614 100644
--- a/tests/functional/refine.t
+++ b/tests/functional/refine.t
@@ -1,8 +1,7 @@
 Integration tests for augur refine.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Try building a time tree.
 
diff --git a/tests/functional/traits.t b/tests/functional/traits.t
index 43fbbf9ad..5e2089c10 100644
--- a/tests/functional/traits.t
+++ b/tests/functional/traits.t
@@ -1,8 +1,7 @@
 Integration tests for augur traits.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Infer the ancestral region for a given tree and metadata.
 
diff --git a/tests/functional/tree.t b/tests/functional/tree.t
index f7829ce47..407ed4082 100644
--- a/tests/functional/tree.t
+++ b/tests/functional/tree.t
@@ -1,8 +1,7 @@
 Integration tests for augur tree.
 
   $ pushd "$TESTDIR" > /dev/null
-  $ export COVERAGE_FILE="$TESTDIR/../../.coverage"
-  $ export AUGUR="coverage run -a --rcfile=$TESTDIR/../../.coveragerc ../../bin/augur"
+  $ export AUGUR="${AUGUR:-../../bin/augur}"
 
 Try building a tree with IQ-TREE.