Update test features: add create_fails script, QC process test, and wait on baseline results

cice.setup

@@ -480,6 +480,7 @@ else
     exit -1
   endif
   cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir}
+  cp -f ${ICE_SCRIPTS}/tests/create_fails.csh ${tsdir}
   cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir}
 
   cat >! ${tsdir}/suite.submit << EOF0

configuration/scripts/tests/QC/gen_qc_cases.csh

@@ -183,9 +183,9 @@ endif
 # Generate the non-BFB but non-climate-changing case
 echo "Generating nonbfb case"
 if ($testid != $spval) then
-  set result = `./cice.setup $options -s qc_nonbfb,long --testid qc_test_$testid | grep 'Test case dir\|already exists'`
+  set result = `./cice.setup $options -s qcnonbfb,long --testid qc_test_$testid | grep 'Test case dir\|already exists'`
 else
-  set result = `./cice.setup $options -s qc_nonbfb,long --testid qc_test | grep 'Test case dir\|already exists'`
+  set result = `./cice.setup $options -s qcnonbfb,long --testid qc_test | grep 'Test case dir\|already exists'`
 endif
 set nonbfb_dir = `echo "$result" | awk '{print$NF}'`
 if ($nonbfb_dir == "exists") then

configuration/scripts/tests/create_fails.csh

@@ -0,0 +1,24 @@
+#!/bin/csh
+
+echo " "
+set tmpfile = create_fails.tmp
+set outfile = fails.ts
+
+./results.csh >& /dev/null
+cat results.log | grep ' run\| test' | grep -v PASS | cut -f 2 -d " " | sort -u >! $tmpfile
+
+echo "# Test  Grid  PEs  Sets" >! fails.ts
+foreach line ( "`cat $tmpfile`" )
+  #echo $line
+  set test = `echo $line | cut -d "_" -f 3`
+  set grid = `echo $line | cut -d "_" -f 4`
+  set pes  = `echo $line | cut -d "_" -f 5`
+  set opts = `echo $line | cut -d "_" -f 6- | sed 's/_/,/g'`
+  echo "$test  $grid  $pes  $opts" >> fails.ts
+end
+
+rm $tmpfile
+echo "$0 done"
+echo "Not passed tests written to file...... $outfile"
+echo "To setup a new test suite, try something like"
+echo "  ./cice.setup --suite fails.ts ..."

doc/source/developer_guide/dg_scripts.rst

@@ -163,7 +163,7 @@ it's working properly.
 
 .. _dev_validation:
 
-Code Validation Script
+QC Process Validation
 ----------------------
 
 The code validation (aka QC or quality control) test validates non bit-for-bit model changes.  The directory 
@@ -193,9 +193,9 @@ to the ``cice.setup`` script.  These options include:
 * ``--queue``   : Queue for the batch submission
 * ``--testid``  : test ID, user-defined id for testing
 
-The script creates 4 test cases, with testIDs ``qc_base``, ``qc_bfb``, ``qc_nonbfb``,
+The script creates 4 test cases, with testIDs ``qc_base``, ``qc_bfb``, ``qc_test``,
 and ``qc_fail``.  ``qc_base`` is the base test case with the default QC namelist.
-``qc_bfb`` is identical to ``qc_base``.  ``qc_nonbfb`` is a test that is not bit-for-bit
+``qc_bfb`` is identical to ``qc_base``.  ``qc_test`` is a test that is not bit-for-bit
 when compared to ``qc_base``, but not climate changing.  ``qc_fail`` is a test that is not
 bit-for-bit and also climate changing.
 
@@ -222,7 +222,7 @@ To perform the QC validation, execute the following commands.
 
   # From the CICE base directory
   cp configuration/scripts/tests/QC/gen_qc_cases.csh .
-  cp configuration/scripts/tests/QC/compare_qc_cases.csh
+  cp configuration/scripts/tests/QC/compare_qc_cases.csh .
 
   # Create the required test cases
   ./gen_qc_cases.csh -m <machine> --acct <acct>

doc/source/user_guide/ug_testing.rst

@@ -297,6 +297,13 @@ results.csh script in the testsuite.[testid]::
   cd testsuite.[testid]
   ./results.csh
 
+The script **create_fails.csh** will process the output from results.csh and generate a new 
+test suite file, **fails.ts**, from the failed tests.  
+**fails.ts** can then be edited and passed into ``cice.setup --suite fails.ts ...`` to rerun 
+subsets of failed tests to more efficiently move thru the development, testing, and 
+validation process.  However, a full test suite should be run on the final development
+version of the code.
+
 To report the test results, as is required for Pull Requests to be accepted into 
 the master the CICE Consortium code see :ref:`testreporting`.
 
@@ -411,8 +418,10 @@ The *cice.setup** options ``--setup-only``, ``--setup-build``, and ``--setup-bui
 
 which means by default the test suite builds and submits the jobs.  By defining other values for those environment variables, users can control the suite script.  When using **suite.submit** manually, the string ``true`` (all lowercase) is the only string that will turn on a feature, and both SUITE_RUN and SUITE_SUBMIT cannot be true at the same time.  
 
-By leveraging the **cice.setup** command line arguments ``--setup-only``, ``--setup-build``, and ``--setup-build-run`` as well as the environment variables SUITE_BUILD, SUITE_RUN, and SUITE_SUBMIT, users can run **cice.setup** and **suite.submit** in various combinations to quickly setup, setup and build, submit, resubmit, run interactively, or rebuild and resubmit full testsuites quickly and easily.  See below for an example.
+By leveraging the **cice.setup** command line arguments ``--setup-only``, ``--setup-build``, and ``--setup-build-run`` as well as the environment variables SUITE_BUILD, SUITE_RUN, and SUITE_SUBMIT, users can run **cice.setup** and **suite.submit** in various combinations to quickly setup, setup and build, submit, resubmit, run interactively, or rebuild and resubmit full testsuites quickly and easily.  See :ref:`examplesuites` for an example.
+
 
+.. _examplesuites:
 
 Test Suite Examples
 ~~~~~~~~~~~~~~~~~~~~~~~~