From dbd2b50594e2bc5bea6af85906c9f9a5c427210c Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Sun, 28 Mar 2021 18:10:36 +0000
Subject: [PATCH 01/38] Old repo went stale, trying fresh

---
 tests/auto/.gitignore                       |   2 +
 tests/auto/jobs/bl.py                       | 209 ++++++++++++
 tests/auto/jobs/rt.py                       | 131 ++++++++
 tests/auto/rt_auto.py                       | 353 ++++++++------------
 tests/auto/{rt_auto.sh => start_rt_auto.sh} |  33 +-
 5 files changed, 492 insertions(+), 236 deletions(-)
 create mode 100644 tests/auto/.gitignore
 create mode 100644 tests/auto/jobs/bl.py
 create mode 100644 tests/auto/jobs/rt.py
 rename tests/auto/{rt_auto.sh => start_rt_auto.sh} (56%)

diff --git a/tests/auto/.gitignore b/tests/auto/.gitignore
new file mode 100644
index 0000000000..d1838a36cd
--- /dev/null
+++ b/tests/auto/.gitignore
@@ -0,0 +1,2 @@
+accesstoken
+rt_auto_*.log
diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
new file mode 100644
index 0000000000..d31fa3caef
--- /dev/null
+++ b/tests/auto/jobs/bl.py
@@ -0,0 +1,209 @@
+# Imports
+import datetime
+import logging
+import os
+import sys
+
+
+def run(job_obj):
+    logger = logging.getLogger('BL/RUN')
+    workdir, rtbldir, blstore = set_directories(job_obj)
+    branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
+    run_regression_test(job_obj, pr_repo_loc)
+    post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch)
+
+
+def set_directories(job_obj):
+    logger = logging.getLogger('BL/SET_DIRECTORIES')
+    if job_obj.machine == 'hera':
+        workdir = '/scratch1/NCEPDEV/nems/Brian.Curtis/autort/pr'
+        blstore = '/scratch1/NCEPDEV/nems/Brian.Curtis/RT/NEMSfv3gfs'
+        rtbldir = '/scratch1/NCEPDEV/stmp4/Brian.Curtis/FV3_RT/'\
+                 f'REGRESSION_TEST_{job_obj.compiler.upper()}'
+    elif job_obj.machine == 'jet':
+        workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr'
+        blstore = '/lfs4/HFIP/hfv3gfs/RT/NEMSfv3gfs/'
+        rtbldir = '/lfs4/HFIP/hfv3gfs/emc.nemspara/RT_BASELINE/'\
+                 f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}'
+    elif job_obj.machine == 'gaea':
+        workdir = '/lustre/f2/pdata/ncep/Brian.Curtis/autort/pr'
+        blstore = '/lustre/f2/pdata/esrl/gsd/ufs/ufs-weather-model/RT'
+        rtbldir = '/lustre/f2/scratch/Brian.Curtis/FV3_RT/'\
+                 f'REGRESSION_TEST_{job_obj.compiler.upper()}'
+    elif job_obj.machine == 'orion':
+        workdir = '/work/noaa/nems/emc.nemspara/autort/pr'
+        blstore = '/work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs'
+        rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\
+                 f'REGRESSION_TEST_{job_obj.compiler.upper()}'
+    elif job_obj.machine == 'cheyenne':
+        workdir = '/glade/work/heinzell/fv3/ufs-weather-model/auto-rt'
+        blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT'
+        rtbldir = '/glade/work/heinzell/FV3_RT/'\
+                 f'REGRESSION_TEST_{job_obj.compiler.upper()}'
+    else:
+        print(f'Machine {job_obj.machine} is not supported for this job')
+        raise KeyError
+
+    logger.info(f'machine: {job_obj.machine}')
+    logger.info(f'workdir: {workdir}')
+    logger.info(f'blstore: {blstore}')
+    logger.info(f'rtbldir: {rtbldir}')
+
+    return workdir, rtbldir, blstore
+
+
+def create_bl_dir(job_obj, bldate, blstore):
+    logger = logging.getLogger('BL/CREATE_BL_DIR')
+    bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
+    logger.info(f'Build Dir: {bldir}')
+    if os.path.exists(bldir):
+        print(f'Baseline dir: {bldir} exists. It should not.')
+        raise FileExistsError
+    else:
+        os.makedirs(bldir)
+        if not os.path.exists(bldir):
+            print(f'Someting went wrong creating {bldir}')
+            raise FileNotFoundError
+
+    return bldir
+
+
+def get_bl_date(job_obj):
+    logger = logging.getLogger('BL/GET_BL_DATE')
+    for line in job_obj.preq_dict['preq'].body.splitlines():
+        if 'BL_DATE:' in line:
+            bldate = line
+            bldate = bldate.replace('BL_DATE:', '')
+            bldate = bldate.replace(' ', '')
+            if len(bldate) != 8:
+                print(f'Date: {bldate} is not formatted YYYYMMDD')
+                raise ValueError
+            logger.info(f'bldate: {bldate}')
+            bl_format = '%Y%m%d'
+            try:
+                datetime.datetime.strptime(bldate, bl_format)
+            except ValueError:
+                logger.info(f'Date {bldate} is not formatted YYYYMMDD')
+                raise ValueError
+    return bldate
+
+
+def run_regression_test(job_obj, pr_repo_loc):
+    logger = logging.getLogger('BL/RUN_REGRESSION_TEST')
+    if job_obj.compiler == 'gnu':
+        rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests '
+                       '&& /bin/bash --login ./rt.sh -e -c -l rt_gnu.conf',
+                       pr_repo_loc]]
+    elif job_obj.compiler == 'intel':
+        rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests '
+                       '&& /bin/bash --login ./rt.sh -e -c', pr_repo_loc]]
+    job_obj.run_commands(logger, rt_command)
+
+
+def remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir):
+    logger = logging.getLogger('BL/REMOVE_PR_DATA')
+    rm_command = [
+                 [f'rm -rf {rt_dir}', pr_repo_loc],
+                 [f'rm -rf {repo_dir_str}', pr_repo_loc]
+                 ]
+    job_obj.run_commands(logger, rm_command)
+
+
+def clone_pr_repo(job_obj, workdir):
+    ''' clone the GitHub pull request repo, via command line '''
+    logger = logging.getLogger('BL/CLONE_PR_REPO')
+    repo_name = job_obj.preq_dict['preq'].head.repo.name
+    branch = job_obj.preq_dict['preq'].head.ref
+    git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//')
+    git_url = f'{git_url[0]}//${{ghapitoken}}@{git_url[1]}'
+    logger.debug(f'GIT URL: {git_url}')
+    logger.info('Starting repo clone')
+    repo_dir_str = f'{workdir}/'\
+                   f'{str(job_obj.preq_dict["preq"].id)}/'\
+                   f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}'
+    pr_repo_loc = f'{repo_dir_str}/{repo_name}'
+    job_obj.comment_text_append(f'Repo location: {pr_repo_loc}')
+    create_repo_commands = [
+        [f'mkdir -p "{repo_dir_str}"', os.getcwd()],
+        [f'git clone -b {branch} {git_url}', repo_dir_str],
+        ['git submodule update --init --recursive',
+         f'{repo_dir_str}/{repo_name}'],
+        ['git config user.email "brian.curtis@noaa.gov"',
+         f'{repo_dir_str}/{repo_name}'],
+        ['git config user.name "Brian Curtis"',
+         f'{repo_dir_str}/{repo_name}']
+    ]
+
+    job_obj.run_commands(logger, create_repo_commands)
+
+    logger.info('Finished repo clone')
+    return branch, pr_repo_loc, repo_dir_str
+
+
+def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch):
+    logger = logging.getLogger('BL/MOVE_RT_LOGS')
+    rt_log = f'tests/RegressionTests_{job_obj.machine}'\
+             f'.{job_obj.compiler}.log'
+    filepath = f'{pr_repo_loc}/{rt_log}'
+    rt_dir, logfile_pass = process_logfile(job_obj, filepath)
+    if logfile_pass:
+        bldate = get_bl_date(job_obj)
+        bldir = create_bl_dir(job_obj, bldate, blstore)
+        move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
+        job_obj.run_commands(logger, move_bl_command)
+        update_rt_sh(job_obj, pr_repo_loc, bldate, branch)
+        remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir)
+
+
+def update_rt_sh(job_obj, pr_repo_loc, bldate, branch):
+    logger = logging.getLogger('BL/UPDATE_RT_SH')
+    with open(f'{pr_repo_loc}/tests/rt.sh', 'r') as f:
+        with open(f'{pr_repo_loc}/tests/rt.sh.new', 'w') as w:
+            for line in f:
+                if 'BL_DATE' in line:
+                    w.write(f'BL_DATE={bldate}\n')
+                else:
+                    w.write(line)
+
+    move_rtsh_commands = [
+        [f'git pull --ff-only origin {branch}', pr_repo_loc],
+        [f'mv {pr_repo_loc}/tests/rt.sh.new {pr_repo_loc}/tests/rt.sh',
+         pr_repo_loc],
+
+        [f'git add {pr_repo_loc}/tests/rt.sh', pr_repo_loc],
+        [f'git commit -m "BL JOBS PASSED: {job_obj.machine}'
+         f'.{job_obj.compiler}. Updated rt.sh with new develop date: '
+         f'{bldate}"',
+         pr_repo_loc],
+        ['sleep 10', pr_repo_loc],
+        [f'git push origin {branch}', pr_repo_loc]
+        ]
+    job_obj.run_commands(logger, move_rtsh_commands)
+
+
+def process_logfile(job_obj, logfile):
+    logger = logging.getLogger('BL/PROCESS_LOGFILE')
+    rt_dir = []
+    if os.path.exists(logfile):
+        with open(logfile) as f:
+            for line in f:
+                if 'FAIL' in line and 'Test' in line:
+                    job_obj.comment_text_append(f'{line.rstrip(chr(10))}')
+                elif 'working dir' in line and not rt_dir:
+                    logger.info(f'Found "working dir" in line: {line}')
+                    rt_dir = os.path.split(line.split()[-1])[0]
+                    logger.info(f'It is: {rt_dir}')
+                    job_obj.comment_text_append(f'Please manually delete: '
+                                                f'{rt_dir}')
+                elif 'SUCCESSFUL' in line:
+                    logger.info('RT Successful')
+                    return rt_dir, True
+        logger.critical(f'Log file exists but is not complete')
+        job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}')
+    else:
+        logger.critical(f'Could not find {job_obj.machine}'
+                        f'.{job_obj.compiler} '
+                        f'{job_obj.preq_dict["action"]} log')
+        print(f'Could not find {job_obj.machine}.{job_obj.compiler} '
+              f'{job_obj.preq_dict["action"]} log')
+        raise FileNotFoundError
diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
new file mode 100644
index 0000000000..1797df5ec8
--- /dev/null
+++ b/tests/auto/jobs/rt.py
@@ -0,0 +1,131 @@
+# Imports
+import datetime
+import logging
+import os
+
+
+def run(job_obj):
+    logger = logging.getLogger('RT/RUN')
+    workdir = set_directories(job_obj)
+    branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
+    run_regression_test(job_obj, pr_repo_loc)
+    post_process(job_obj, pr_repo_loc, repo_dir_str, branch)
+
+
+def set_directories(job_obj):
+    logger = logging.getLogger('RT/SET_DIRECTORIES')
+    if job_obj.machine == 'hera':
+        workdir = '/scratch1/NCEPDEV/nems/Brian.Curtis/autort/pr'
+    elif job_obj.machine == 'jet':
+        workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr'
+    elif job_obj.machine == 'gaea':
+        workdir = '/lustre/f2/pdata/ncep/Brian.Curtis/autort/pr'
+    elif job_obj.machine == 'orion':
+        workdir = '/work/noaa/nems/emc.nemspara/autort/pr'
+    elif job_obj.machine == 'cheyenne':
+        workdir = '/glade/work/heinzell/fv3/ufs-weather-model/auto-rt'
+    else:
+        print(f'Machine {job_obj.machine} is not supported for this job')
+        raise KeyError
+
+    logger.info(f'machine: {job_obj.machine}')
+    logger.info(f'workdir: {workdir}')
+
+    return workdir
+
+
+def run_regression_test(job_obj, pr_repo_loc):
+    logger = logging.getLogger('RT/RUN_REGRESSION_TEST')
+    if job_obj.compiler == 'gnu':
+        rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests '
+                       '&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf',
+                       pr_repo_loc]]
+    elif job_obj.compiler == 'intel':
+        rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests '
+                       '&& /bin/bash --login ./rt.sh -e', pr_repo_loc]]
+    job_obj.run_commands(logger, rt_command)
+
+
+def remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir):
+    logger = logging.getLogger('RT/REMOVE_PR_DATA')
+    rm_command = [
+                 [f'rm -rf {rt_dir}', pr_repo_loc],
+                 [f'rm -rf {repo_dir_str}', pr_repo_loc]
+                 ]
+    job_obj.run_commands(logger, rm_command)
+
+
+def clone_pr_repo(job_obj, workdir):
+    ''' clone the GitHub pull request repo, via command line '''
+    logger = logging.getLogger('RT/CLONE_PR_REPO')
+    repo_name = job_obj.preq_dict['preq'].head.repo.name
+    branch = job_obj.preq_dict['preq'].head.ref
+    git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//')
+    git_url = f'{git_url[0]}//${{ghapitoken}}@{git_url[1]}'
+    logger.debug(f'GIT URL: {git_url}')
+    logger.info('Starting repo clone')
+    repo_dir_str = f'{workdir}/'\
+                   f'{str(job_obj.preq_dict["preq"].id)}/'\
+                   f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}'
+    pr_repo_loc = f'{repo_dir_str}/{repo_name}'
+    job_obj.comment_text_append(f'Repo location: {pr_repo_loc}')
+    create_repo_commands = [
+        [f'mkdir -p "{repo_dir_str}"', os.getcwd()],
+        [f'git clone -b {branch} {git_url}', repo_dir_str],
+        ['git submodule update --init --recursive',
+         f'{repo_dir_str}/{repo_name}'],
+        ['git config user.email "brian.curtis@noaa.gov"',
+         f'{repo_dir_str}/{repo_name}'],
+        ['git config user.name "Brian Curtis"',
+         f'{repo_dir_str}/{repo_name}']
+    ]
+
+    job_obj.run_commands(logger, create_repo_commands)
+
+    logger.info('Finished repo clone')
+    return branch, pr_repo_loc, repo_dir_str
+
+
+def post_process(job_obj, pr_repo_loc, repo_dir_str, branch):
+    ''' This is the callback function associated with the "RT" command '''
+    logger = logging.getLogger('RT/MOVE_RT_LOGS')
+    rt_log = f'tests/RegressionTests_{job_obj.machine}'\
+             f'.{job_obj.compiler}.log'
+    filepath = f'{pr_repo_loc}/{rt_log}'
+    rt_dir, logfile_pass = process_logfile(job_obj, filepath)
+    if logfile_pass:
+        move_rt_commands = [
+            [f'git pull --ff-only origin {branch}', pr_repo_loc],
+            [f'git add {rt_log}', pr_repo_loc],
+            [f'git commit -m "RT JOBS PASSED: {job_obj.machine}'
+             f'.{job_obj.compiler}. Log file uploaded. skip-ci"',
+             pr_repo_loc],
+            ['sleep 10', pr_repo_loc],
+            [f'git push origin {branch}', pr_repo_loc]
+        ]
+        job_obj.run_commands(logger, move_rt_commands)
+        remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir)
+
+
+def process_logfile(job_obj, logfile):
+    logger = logging.getLogger('RT/PROCESS_LOGFILE')
+    rt_dir = []
+    if os.path.exists(logfile):
+        with open(logfile) as f:
+            for line in f:
+                if 'FAIL' in line and 'Test' in line:
+                    job_obj.comment_text_append(f'{line.rstrip(chr(10))}')
+                elif 'working dir' in line and not rt_dir:
+                    rt_dir = os.path.split(line.split()[-1])[0]
+                    job_obj.comment_text_append(f'Please manually delete: '
+                                                f'{rt_dir}')
+                elif 'SUCCESSFUL' in line:
+                    return rt_dir, True
+        job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}')
+    else:
+        logger.critical(f'Could not find {job_obj.machine}'
+                        f'.{job_obj.compiler} '
+                        f'{job_obj.preq_dict["action"]} log')
+        print(f'Could not find {job_obj.machine}.{job_obj.compiler} '
+              f'{job_obj.preq_dict["action"]} log')
+        raise FileNotFoundError
diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py
index 5e0859531d..7876897572 100644
--- a/tests/auto/rt_auto.py
+++ b/tests/auto/rt_auto.py
@@ -7,12 +7,13 @@
 prior to start.
 """
 from github import Github as gh
-import argparse
 import datetime
 import subprocess
 import re
 import os
 import logging
+import importlib
+
 
 class GHInterface:
     '''
@@ -26,96 +27,78 @@ class GHInterface:
     client : pyGitHub communication object
       The connection to GitHub to make API requests
     '''
+
     def __init__(self):
         self.logger = logging.getLogger('GHINTERFACE')
+
+        filename = 'accesstoken'
+
+        if os.path.exists(filename):
+            if oct(os.stat(filename).st_mode)[-3:] != 600:
+                with open(filename) as f:
+                    os.environ['ghapitoken'] = f.readline().strip('\n')
+            else:
+                raise Exception('File permission needs to be "600" ')
+        else:
+            raise FileNotFoundError('Cannot find file "accesstoken"')
+
         try:
             self.client = gh(os.getenv('ghapitoken'))
         except Exception as e:
             self.logger.critical(f'Exception is {e}')
             raise(e)
 
-def parse_args_in():
-    ''' Parse all input arguments coming from rt_auto.sh '''
-    logger = logging.getLogger('PARSE_ARGS_IN')
-    # Create Parse
-    parser = argparse.ArgumentParser()
-
-    # Setup Input Arguments
-    choices = ['cheyenne', 'hera', 'orion', 'gaea', 'jet', 'wcoss_dell_p3']
-    parser.add_argument('-m', '--machine', help='Machine name', required=True, choices=choices, type=str)
-    parser.add_argument('-w', '--workdir', help='Working directory', required=True, type=str)
-
-    # Get Arguments
-    args = parser.parse_args()
-
-    return args
-
-def input_data(args):
-    ''' Create dictionaries of data needed for processing UFS pull requests '''
-    logger = logging.getLogger('INPUT_DATA')
-    machine_dict = {
-        'name': args.machine,
-        'workdir': args.workdir
-    }
-    repo_list_dict = [{
-        'name': 'ufs-weather-model',
-        'address': 'ufs-community/ufs-weather-model',
-        'base': 'develop'
-    }]
-    action_list_dict = [{
-        'name': 'RT',
-        'callback_fnc': 'rt_callback'
-    },
-    {
-        'name': 'BL',
-        'callback_fnc': 'bl_callback'
-    }]
-
-    return machine_dict, repo_list_dict, action_list_dict
 
 def set_action_from_label(machine, actions, label):
     ''' Match the label that initiates a job with an action in the dict'''
     # <machine>-<compiler>-<test> i.e. hera-gnu-RT
-    # RT = full regression test suite
     logger = logging.getLogger('MATCH_LABEL_WITH_ACTIONS')
+    logger.info('Setting action from Label')
     split_label = label.name.split('-')
-    if len(split_label) != 3: return False, False #Make sure it has three parts
+    # Make sure it has three parts
+    if len(split_label) != 3:
+        return False, False
+    # Break the parts into their variables
     label_machine = split_label[0]
     label_compiler = split_label[1]
     label_action = split_label[2]
-    if not re.match(label_machine, machine['name']): return False, False #check machine name matches
-    if not str(label_compiler) in ["intel", "gnu"]: return False, False #Compiler must be intel or gnu
-    action_match = next((action for action in actions if re.match(action['name'], label_action)), False)
-    if label_action == 'RT': # SET ACTIONS BASED ON RT COMMAND
-        if label_compiler == "intel":
-            action_match["command"] = f'export RT_COMPILER="intel" && cd tests && /bin/bash --login ./rt.sh -e'
-        elif label_compiler == "gnu":
-            action_match["command"] = f'export RT_COMPILER="gnu" && cd tests && /bin/bash --login ./rt.sh -e -l rt_gnu.conf'
-    elif label_action == 'BL': # SET ACTIONS BASED ON BL COMMAND
-        if label_compiler == "intel":
-            action_match["command"] = f'export RT_COMPILER="intel" && cd tests && /bin/bash --login ./rt.sh -e -c'
-        elif label_compiler == "gnu":
-            action_match["command"] = f'export RT_COMPILER="gnu" && cd tests && /bin/bash --login ./rt.sh -e -c -l rt_gnu.conf'
-
+    # check machine name matches
+    if not re.match(label_machine, machine):
+        return False, False
+    # Compiler must be intel or gnu
+    if not str(label_compiler) in ["intel", "gnu"]:
+        return False, False
+    action_match = next((action for action in actions
+                         if re.match(action, label_action)), False)
+
+    logging.info(f'Compiler: {label_compiler}, Action: {action_match}')
     return label_compiler, action_match
 
 
 def get_preqs_with_actions(repos, machine, ghinterface_obj, actions):
-    ''' Create list of dictionaries of a pull request and its machine label and action '''
+    ''' Create list of dictionaries of a pull request
+        and its machine label and action '''
     logger = logging.getLogger('GET_PREQS_WITH_ACTIONS')
-    gh_preqs = [ghinterface_obj.client.get_repo(repo['address']).get_pulls(state='open', sort='created', base=repo['base']) for repo in repos]
+    logger.info('Getting Pull Requests with Actions')
+    gh_preqs = [ghinterface_obj.client.get_repo(repo['address'])
+                .get_pulls(state='open', sort='created', base=repo['base'])
+                for repo in repos]
     each_pr = [preq for gh_preq in gh_preqs for preq in gh_preq]
-    preq_labels = [{'preq': pr, 'label': label} for pr in each_pr for label in pr.get_labels()]
+    preq_labels = [{'preq': pr, 'label': label} for pr in each_pr
+                   for label in pr.get_labels()]
 
-    return_preq = []
+    jobs = []
+    # return_preq = []
     for pr_label in preq_labels:
-        compiler, match = set_action_from_label(machine, actions, pr_label['label'])
+        compiler, match = set_action_from_label(machine, actions,
+                                                pr_label['label'])
         if match:
-            pr_label['action'] = match.copy()
-            pr_label['compiler'] = compiler
-            return_preq.append(pr_label.copy())
+            pr_label['action'] = match
+            # return_preq.append(pr_label.copy())
+            jobs.append(Job(pr_label.copy(), ghinterface_obj, machine, compiler))
+
+    return jobs
 
-    return return_preq
 
 class Job:
     '''
@@ -134,196 +117,154 @@ class Job:
         provided by the bash script
     '''
 
-    def __init__(self, preq_dict, ghinterface_obj, machine):
+    def __init__(self, preq_dict, ghinterface_obj, machine, compiler):
         self.logger = logging.getLogger('JOB')
         self.preq_dict = preq_dict
+        self.job_mod = importlib.import_module(
+                       f'jobs.{self.preq_dict["action"].lower()}')
         self.ghinterface_obj = ghinterface_obj
         self.machine = machine
+        self.compiler = compiler
+        self.comment_text = ''
+        self.failed_tests = []
+
+    def comment_text_append(self, newtext):
+        self.comment_text += f'{newtext}\n'
 
     def remove_pr_label(self):
-        ''' Removes the pull request label that initiated the job run from PR '''
+        ''' Removes the PR label that initiated the job run from PR '''
         self.logger.info(f'Removing Label: {self.preq_dict["label"]}')
         self.preq_dict['preq'].remove_from_labels(self.preq_dict['label'])
 
     def check_label_before_job_start(self):
         # LETS Check the label still exists before the start of the job in the
         # case of multiple jobs
-        label_to_check = f'{self.machine["name"]}-{self.preq_dict["compiler"]}-{self.preq_dict["action"]["name"]}'
+        label_to_check = f'{self.machine}'\
+                         f'-{self.compiler}'\
+                         f'-{self.preq_dict["action"]}'
         labels = self.preq_dict['preq'].get_labels()
-        label_match = next((label for label in labels if re.match(label.name, label_to_check)), False)
+        label_match = next((label for label in labels
+                            if re.match(label.name, label_to_check)), False)
 
         return label_match
 
-    def run_commands(self, commands_with_cwd):
-        logger = logging.getLogger('JOB/RUN_COMMANDS')
+    def run_commands(self, logger, commands_with_cwd):
         for command, in_cwd in commands_with_cwd:
-            logger.info(f'Running "{command}" in location "{in_cwd}"')
+            logger.info(f'Running `{command}`')
+            logger.info(f'in location "{in_cwd}"')
             try:
-                output = subprocess.Popen(command, shell=True, cwd=in_cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-                out, err = output.communicate()
-                out = [] if not out else out.decode('utf8').split('\n')
-                err = [] if not err else err.decode('utf8').split('\n')
+                output = subprocess.Popen(command, shell=True, cwd=in_cwd,
+                                          stdout=subprocess.PIPE,
+                                          stderr=subprocess.STDOUT)
             except Exception as e:
-                self.job_failed(logger, f'Command {command}', exception=e, out=out, err=err)
-            else:
-                logger.info(f'Finished running: {command}')
-                [logger.debug(f'stdout: {item}') for item in out if not None]
-
-    def remove_pr_data(self):
-        logger = logging.getLogger('JOB/REMOVE_PR_DATA')
-        pr_dir_str = f'{self.machine["workdir"]}/{str(self.preq_dict["preq"].id)}'
-        rm_command = [
-                     [f'rm -rf {self.rt_dir}', self.pr_repo_loc],
-                     [f'rm -rf {pr_dir_str}', self.pr_repo_loc]
-                     ]
-        logger.info(f'Running "{rm_command}"')
-        self.run_commands(rm_command)
-
-    def clone_pr_repo(self):
-        ''' clone the GitHub pull request repo, via command line '''
-        logger = logging.getLogger('JOB/CLONE_PR_REPO')
-        repo_name = self.preq_dict['preq'].head.repo.name
-        self.branch = self.preq_dict['preq'].head.ref
-        git_url = self.preq_dict['preq'].head.repo.html_url.split('//')
-        git_url = f'{git_url[0]}//${{ghapitoken}}@{git_url[1]}'
-        logger.info(f'GIT URL: {git_url}')
-        logger.info('Starting repo clone')
-        repo_dir_str = f'{self.machine["workdir"]}/{str(self.preq_dict["preq"].id)}/{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}'
-        self.pr_repo_loc = repo_dir_str+"/"+repo_name
-        create_repo_commands = [
-            [f'mkdir -p "{repo_dir_str}"', self.machine['workdir']],
-            [f'git clone -b {self.branch} {git_url}', repo_dir_str],
-            [f'git submodule update --init --recursive', f'{repo_dir_str}/{repo_name}']
-        ]
-
-        self.run_commands(create_repo_commands)
-
-        logger.info('Finished repo clone')
-        return self.pr_repo_loc
-
-    def execute_command(self):
-        ''' Run the command associted with the label used to initiate this job '''
-        logger = logging.getLogger('JOB/EXECUTE_COMMAND')
-        compiler = self.preq_dict['compiler']
-        logger.info(f'Compiler being used for command is {compiler}')
-        command = self.preq_dict["action"]["command"]
-
-        try:
-            logger.info(f'Running: "{command}" in "{self.pr_repo_loc}"')
-            output = subprocess.Popen(command, cwd=self.pr_repo_loc, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-            out,err = output.communicate()
-        except Exception as e:
-            out = [] if not out else out.decode('utf8').split('\n')
-            err = [] if not err else err.decode('utf8').split('\n')
-            self.job_failed(logger, f'Command {command}', exception=e, out=out, err=err)
-        else:
-            if output.returncode != 0:
-                self.job_failed(logger, "Script rt.sh", exception=SystemExit, STDOUT=False)
+                self.job_failed(logger, 'subprocess.Popen')
             else:
                 try:
-                    logger.info(f'Attempting to run callback: {self.preq_dict["action"]["callback_fnc"]}')
-                    getattr(self, self.preq_dict['action']['callback_fnc'])()
+                    out, err = output.communicate()
+                    out = [] if not out else out.decode('utf8').split('\n')
+                    logger.info(out)
                 except Exception as e:
-                    self.job_failed(logger, f'Callback function {self.preq_dict["action"]["callback_fnc"]}', exception=e, STDOUT=False)
+                    err = [] if not err else err.decode('utf8').split('\n')
+                    self.job_failed(logger, f'Command {command}', exception=e,
+                                    STDOUT=True, out=out, err=err)
                 else:
-                    logger.info(f'Finished callback {self.preq_dict["action"]["callback_fnc"]}')
-                    [logger.debug(f'stdout: {item}') for item in out if not None]
+                    logger.info(f'Finished running: {command}')
 
     def run(self):
         logger = logging.getLogger('JOB/RUN')
         logger.info(f'Starting Job: {self.preq_dict["label"]}')
+        self.comment_text_append(newtext=f'Machine: {self.machine}')
+        self.comment_text_append(f'Compiler: {self.compiler}')
+        self.comment_text_append(f'Job: {self.preq_dict["action"]}')
         if self.check_label_before_job_start():
             try:
                 logger.info('Calling remove_pr_label')
                 self.remove_pr_label()
-                logger.info('Calling clone_pr_repo')
-                self.clone_pr_repo()
-                logger.info('Calling execute_command')
-                self.execute_command()
-            except Exception as e:
-                self.job_failed(logger, f'run()', exception=e, STDOUT=False)
+                logger.info('Calling Job to Run')
+                self.job_mod.run(self)
+            except Exception:
+                self.job_failed(logger, 'run()')
+                logger.info('Sending comment text')
+                self.send_comment_text()
         else:
             logger.info(f'Cannot find label {self.preq_dict["label"]}')
 
-    def job_failed(self, logger, job_name, exception=Exception, STDOUT=True, out=None, err=None):
-        comment_text = f'{job_name} FAILED \n'\
-                       f'Repo location: {self.pr_repo_loc} \n'\
-                       f'Machine: {self.machine["name"]} \n'\
-                       f'Compiler: {self.preq_dict["compiler"]} \n'
+    def send_comment_text(self):
+        logger = logging.getLogger('JOB/SEND_COMMENT_TEXT')
+        logger.info(f'Comment Text: {self.comment_text}')
+        self.comment_text_append('Please make changes and add '
+                                 'the following label back:')
+        self.comment_text_append(f'{self.machine}'
+                                 f'-{self.compiler}'
+                                 f'-{self.preq_dict["action"]}')
+
+        self.preq_dict['preq'].create_issue_comment(self.comment_text)
+
+    def job_failed(self, logger, job_name, exception=Exception, STDOUT=False,
+                   out=None, err=None):
+        self.comment_text_append(f'{job_name} FAILED. Exception:{exception}')
+        logger.critical(f'{job_name} FAILED. Exception:{exception}')
+
         if STDOUT:
-            comment_text=comment_text+'\n'\
-                                f'STDOUT: {[item for item in out if not None]} \n'\
-                                f'STDERR: {[eitem for eitem in err if not None]} \n'
-        comment_text = comment_text+'Please make changes and add the following label back: '\
-                            f'{self.machine["name"]}-{self.preq_dict["compiler"]}-{self.preq_dict["action"]["name"]}'
-        logger.critical(comment_text)
-        self.preq_dict['preq'].create_issue_comment(comment_text)
-        raise exception
-
-    def process_logfile(self, logfile):
-        self.rt_dir = []
-        if os.path.exists(logfile):
-            with open(logfile) as f:
-                for line in f:
-                    if 'working dir' in line and not self.rt_dir:
-                        self.rt_dir = os.path.split(line.split()[-1])[0]
-                    elif 'SUCCESSFUL' in line:
-                        return True
-            self.job_failed(logger, "Regression Tests", STDOUT=False)
-        else:
-            logger.critical(f'Could not find {self.machine["name"]}.{self.preq_dict["compiler"]} RT log')
-            raise FileNotFoundError(f'Could not find {self.machine["name"]}.{self.preq_dict["compiler"]} RT log')
-
-    # Add Callback Functions After Here
-    def rt_callback(self):
-        ''' This is the callback function associated with the "RT" command '''
-        logger = logging.getLogger('JOB/MOVE_RT_LOGS')
-        rt_log = f'tests/RegressionTests_{self.machine["name"]}.{self.preq_dict["compiler"]}.log'
-        filepath = f'{self.pr_repo_loc}/{rt_log}'
-        logfile_pass = self.process_logfile(filepath)
-        if logfile_pass:
-            move_rt_commands = [
-                [f'git pull --ff-only origin {self.branch}', self.pr_repo_loc],
-                [f'git add {rt_log}', self.pr_repo_loc],
-                [f'git commit -m "PASSED: {self.machine["name"]}.{self.preq_dict["compiler"]}. Log file uploaded. skip-ci"', self.pr_repo_loc],
-                ['sleep 10', self.pr_repo_loc],
-                [f'git push origin {self.branch}', self.pr_repo_loc]
-            ]
-            self.run_commands(move_rt_commands)
-            self.remove_pr_data()
-
-    def bl_callback(self):
-        pass
+            logger.critical(f'STDOUT: {[item for item in out if not None]}')
+            logger.critical(f'STDERR: {[eitem for eitem in err if not None]}')
+
+def setup_env():
+    hostname = os.getenv('HOSTNAME')
+    if bool(re.match(re.compile('hfe.+'), hostname)):
+        machine = 'hera'
+    elif bool(re.match(re.compile('fe.+'), hostname)):
+        machine = 'jet'
+        os.environ['ACCNR'] = 'h-nems'
+    elif bool(re.match(re.compile('gaea.+'), hostname)):
+        machine = 'gaea'
+        os.environ['ACCNR'] = 'nggps_emc'
+    elif bool(re.match(re.compile('Orion-login.+'), hostname)):
+        machine = 'orion'
+    elif bool(re.match(re.compile('.+.cheyenne.ucar.edu'), hostname)):
+        machine = 'cheyenne'
+        os.environ['ACCNR'] = 'P48503002'
+    else:
+        raise KeyError(f'Hostname: {hostname} does not match '\
+                        'for a supported system. Exiting.')
+
+    # Dictionary of GitHub repositories to check
+    repo_dict = [{
+        'name': 'ufs-weather-model',
+        'address': 'BrianCurtis-NOAA/ufs-weather-model',
+        'base': 'develop'
+    }]
+
+    # Approved Actions
+    action_list = ['RT', 'BL']
+
+    return machine, repo_dict, action_list
+
 
 def main():
 
     # handle logging
-    log_path = os.getcwd()
-    log_filename = f'rt_auto_{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}.log'
-    # Please don't run the following on cron with level=logging.DEBUG
-    # as it exposes the GH API Token
-    # Only set it to DEBUG while debugging
-    logging.basicConfig(filename=log_filename, filemode='w', level=logging.INFO)
+    log_filename = f'rt_auto_'\
+                   f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}.log'
+    logging.basicConfig(filename=log_filename, filemode='w',
+                        level=logging.INFO)
     logger = logging.getLogger('MAIN')
     logger.info('Starting Script')
-    # handle input args
-    logger.info('Parsing input args')
-    args = parse_args_in()
 
-    # get input data
-    logger.info('Calling input_data().')
-    machine, repos, actions = input_data(args)
+    # setup environment
+    logger.info('Getting the environment setup')
+    machine, repos, actions = setup_env()
 
     # setup interface with GitHub
     logger.info('Setting up GitHub interface.')
     ghinterface_obj = GHInterface()
 
     # get all pull requests from the GitHub object
-    logger.info('Getting all pull requests, labels and actions applicable to this machine.')
-    preq_dict = get_preqs_with_actions(repos, machine, ghinterface_obj, actions)
-    # add Job objects and run them
-    logger.info('Adding all jobs to an object list and running them.')
-    jobs = [Job(pullreq, ghinterface_obj, machine) for pullreq in preq_dict]
+    # and turn them into Job objects
+    logger.info('Getting all pull requests, '
+                'labels and actions applicable to this machine.')
+    jobs = get_preqs_with_actions(repos, machine,
+                                       ghinterface_obj, actions)
     [job.run() for job in jobs]
 
     logger.info('Script Finished')
diff --git a/tests/auto/rt_auto.sh b/tests/auto/start_rt_auto.sh
similarity index 56%
rename from tests/auto/rt_auto.sh
rename to tests/auto/start_rt_auto.sh
index 17d7e6ff00..74ad09f50a 100644
--- a/tests/auto/rt_auto.sh
+++ b/tests/auto/start_rt_auto.sh
@@ -1,53 +1,26 @@
 #!/bin/bash --login
 set -eux
-if [ -f "accesstoken.sh" ]; then
-  if [ $(stat -L -c "%a" "accesstoken.sh") == "600" ]; then
-    echo "Sourcing accesstoken.sh"
-    source ./accesstoken.sh
-  else
-    echo "accesstoken.sh permissions NEED to be set to 600 before starting"
-    exit 1
-  fi
-else
-  echo "Please create accesstoken.sh (600) with the following content\n"
-  echo "export ghapitoken=<GitHub API Token Here>"
-  exit 1
-fi
 
 if [[ $HOSTNAME == hfe* ]]; then
-  MACHINE_NAME=hera
-  WORKDIR=/scratch1/NCEPDEV/nems/Brian.Curtis/test
   export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH
   export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages
 elif [[ $HOSTNAME == Orion-login-* ]]; then
-  MACHINE_NAME=orion
-  WORKDIR=/work/noaa/nems/bcurtis/test
   export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH
   export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages
 elif [[ $HOSTNAME == fe* ]]; then
-  MACHINE_NAME=jet
-  WORKDIR=/lfs4/HFIP/h-nems/Brian.Curtis/test
-  export ACCNR="h-nems"
   export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH
   export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages
 elif [[ $HOSTNAME == gaea* ]]; then
-  MACHINE_NAME=gaea
-  WORKDIR=/lustre/f2/pdata/ncep/Brian.Curtis/test
-  export LOADEDMODULES=$LOADEDMODULES
-  export ACCNR="nggps_emc" # This applies to Brian.Curtis, may need change later
   export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:$PATH
   export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages
-elif [[ $HOSTNAME == *.cheyenne.ucar.edu ]]; then
-  MACHINE_NAME=cheyenne
-  WORKDIR=/glade/work/heinzell/fv3/ufs-weather-model/auto-rt
-  export ACCNR="P48503002"
+elif [[ $HOSTNAME == *cheyenne* ]]; then
   export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH
   export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages
 else
-  echo "No Python Path for this machine. automated RT not starting"
+  echo "No Python Path for this machine."
   exit 1
 fi
 
-python rt_auto.py -m $MACHINE_NAME -w $WORKDIR
+python rt_auto.py
 
 exit 0

From c7ca5419a1590630b0f714ce7f188487c2867b4c Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Sun, 28 Mar 2021 19:00:30 +0000
Subject: [PATCH 02/38] Check for BL_DATE in PR body BEFORE getting far into
 the Job

---
 tests/auto/jobs/bl.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index d31fa3caef..172bc50eb3 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -7,6 +7,7 @@
 
 def run(job_obj):
     logger = logging.getLogger('BL/RUN')
+    bldate = get_bl_date(job_obj)
     workdir, rtbldir, blstore = set_directories(job_obj)
     branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
     run_regression_test(job_obj, pr_repo_loc)
@@ -78,14 +79,18 @@ def get_bl_date(job_obj):
             if len(bldate) != 8:
                 print(f'Date: {bldate} is not formatted YYYYMMDD')
                 raise ValueError
-            logger.info(f'bldate: {bldate}')
+            logger.info(f'BL_DATE: {bldate}')
             bl_format = '%Y%m%d'
             try:
                 datetime.datetime.strptime(bldate, bl_format)
             except ValueError:
                 logger.info(f'Date {bldate} is not formatted YYYYMMDD')
                 raise ValueError
-    return bldate
+            return bldate
+        else:
+            logger.critical('"BL_DATE:YYYYMMDD" needs to be in the PR body.'\
+                            'On its own line. Stopping')
+            raise ValueError
 
 
 def run_regression_test(job_obj, pr_repo_loc):

From de8a812a7450c8b09494e88c103b0b2fb9444aa1 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Sun, 28 Mar 2021 19:14:22 +0000
Subject: [PATCH 03/38] Raised error at wrong point in get_bl_date()

---
 tests/auto/jobs/bl.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 172bc50eb3..4564df3f1a 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -87,10 +87,9 @@ def get_bl_date(job_obj):
                 logger.info(f'Date {bldate} is not formatted YYYYMMDD')
                 raise ValueError
             return bldate
-        else:
-            logger.critical('"BL_DATE:YYYYMMDD" needs to be in the PR body.'\
-                            'On its own line. Stopping')
-            raise ValueError
+    logger.critical('"BL_DATE:YYYYMMDD" needs to be in the PR body.'\
+                    'On its own line. Stopping')
+    raise ValueError
 
 
 def run_regression_test(job_obj, pr_repo_loc):

From b68615f5fdfc259700fd86cab9e50953834e17ef Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Sun, 28 Mar 2021 19:49:31 +0000
Subject: [PATCH 04/38] BL JOBS PASSED: hera.gnu. Updated rt.sh with new
 develop date: 20210324

---
 tests/rt.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 tests/rt.sh

diff --git a/tests/rt.sh b/tests/rt.sh
old mode 100755
new mode 100644

From c91b9d6e562e220d9c700e82b5ed1294c98baa64 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Sun, 28 Mar 2021 19:57:16 +0000
Subject: [PATCH 05/38] change rt.sh to work for code

---
 tests/rt.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/rt.sh b/tests/rt.sh
index c5e54b9bef..2042e343e0 100755
--- a/tests/rt.sh
+++ b/tests/rt.sh
@@ -415,10 +415,11 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
   TEST_35D=true
 fi
 
+BL_DATE=20210324
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]]; then
-  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20210324/${RT_COMPILER^^}}
+  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else
-  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20210324}
+  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}}
 fi
 
 INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20210324}

From 4134be246585e8585b4f50e220c107f1a0bff559 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Sun, 28 Mar 2021 22:32:50 +0000
Subject: [PATCH 06/38] Make logging better in update_rt_sh

---
 tests/auto/jobs/bl.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 4564df3f1a..8225d16142 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -161,13 +161,22 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch):
 
 def update_rt_sh(job_obj, pr_repo_loc, bldate, branch):
     logger = logging.getLogger('BL/UPDATE_RT_SH')
+    BLDATEFOUND = False
     with open(f'{pr_repo_loc}/tests/rt.sh', 'r') as f:
         with open(f'{pr_repo_loc}/tests/rt.sh.new', 'w') as w:
             for line in f:
                 if 'BL_DATE' in line:
+                    logger.info('Found BL_DATE in line')
+                    logger.info(f'Writing "BL_DATE-{bldate}" into file')
                     w.write(f'BL_DATE={bldate}\n')
+                    BLDATEFOUND = True
                 else:
                     w.write(line)
+    if not BLDATEFOUND:
+        job_obj.comment_text_append('BL_DATE not found in rt.sh.'
+                                    'Please manually edit rt.sh '
+                                    'with BL_DATE={bldate})
+    logger.info('Finished update_rt_sh')
 
     move_rtsh_commands = [
         [f'git pull --ff-only origin {branch}', pr_repo_loc],

From 14471952ac1540088c084dcea9522911ce784339 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Sun, 28 Mar 2021 23:50:13 +0000
Subject: [PATCH 07/38] Lets not go through with the BL job if the directory
 already exists as it should not before starting the job

---
 tests/auto/jobs/bl.py | 38 ++++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 8225d16142..e166b3d5bf 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -9,9 +9,11 @@ def run(job_obj):
     logger = logging.getLogger('BL/RUN')
     bldate = get_bl_date(job_obj)
     workdir, rtbldir, blstore = set_directories(job_obj)
-    branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
-    run_regression_test(job_obj, pr_repo_loc)
-    post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch)
+    bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
+    if not check_for_bl_dir(bldir):
+        branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
+        run_regression_test(job_obj, pr_repo_loc)
+        post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, branch)
 
 
 def set_directories(job_obj):
@@ -42,7 +44,7 @@ def set_directories(job_obj):
         rtbldir = '/glade/work/heinzell/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     else:
-        print(f'Machine {job_obj.machine} is not supported for this job')
+        logger.critical(f'Machine {job_obj.machine} is not supported for this job')
         raise KeyError
 
     logger.info(f'machine: {job_obj.machine}')
@@ -53,21 +55,23 @@ def set_directories(job_obj):
     return workdir, rtbldir, blstore
 
 
-def create_bl_dir(job_obj, bldate, blstore):
-    logger = logging.getLogger('BL/CREATE_BL_DIR')
-    bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
-    logger.info(f'Build Dir: {bldir}')
+def check_for_bl_dir(bldir):
+    logger = logging.getLogger('BL/CHECK_FOR_BL_DIR')
+    logger.info('Checking if baseline directory exists')
     if os.path.exists(bldir):
-        print(f'Baseline dir: {bldir} exists. It should not.')
+        logger.critical(f'Baseline dir: {bldir} exists. It should not, yet.')
         raise FileExistsError
-    else:
+    return False
+
+
+def create_bl_dir(bldir):
+    logger = logging.getLogger('BL/CREATE_BL_DIR')
+    if not check_for_bl_dir(bldir):
         os.makedirs(bldir)
         if not os.path.exists(bldir):
-            print(f'Someting went wrong creating {bldir}')
+            logger.critical(f'Someting went wrong creating {bldir}')
             raise FileNotFoundError
 
-    return bldir
-
 
 def get_bl_date(job_obj):
     logger = logging.getLogger('BL/GET_BL_DATE')
@@ -144,7 +148,7 @@ def clone_pr_repo(job_obj, workdir):
     return branch, pr_repo_loc, repo_dir_str
 
 
-def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch):
+def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, branch):
     logger = logging.getLogger('BL/MOVE_RT_LOGS')
     rt_log = f'tests/RegressionTests_{job_obj.machine}'\
              f'.{job_obj.compiler}.log'
@@ -152,7 +156,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch):
     rt_dir, logfile_pass = process_logfile(job_obj, filepath)
     if logfile_pass:
         bldate = get_bl_date(job_obj)
-        bldir = create_bl_dir(job_obj, bldate, blstore)
+        create_bl_dir(bldir)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
         job_obj.run_commands(logger, move_bl_command)
         update_rt_sh(job_obj, pr_repo_loc, bldate, branch)
@@ -175,7 +179,7 @@ def update_rt_sh(job_obj, pr_repo_loc, bldate, branch):
     if not BLDATEFOUND:
         job_obj.comment_text_append('BL_DATE not found in rt.sh.'
                                     'Please manually edit rt.sh '
-                                    'with BL_DATE={bldate})
+                                    'with BL_DATE={bldate}')
     logger.info('Finished update_rt_sh')
 
     move_rtsh_commands = [
@@ -217,6 +221,4 @@ def process_logfile(job_obj, logfile):
         logger.critical(f'Could not find {job_obj.machine}'
                         f'.{job_obj.compiler} '
                         f'{job_obj.preq_dict["action"]} log')
-        print(f'Could not find {job_obj.machine}.{job_obj.compiler} '
-              f'{job_obj.preq_dict["action"]} log')
         raise FileNotFoundError

From 29f1944d504116d0ff7eac0edd12a490bbf17c46 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 29 Mar 2021 00:26:28 +0000
Subject: [PATCH 08/38] BL JOBS PASSED: hera.gnu. Updated rt.sh with new
 develop date: 20210331

---
 tests/rt.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/rt.sh b/tests/rt.sh
index 2042e343e0..d86ce0bcf2 100644
--- a/tests/rt.sh
+++ b/tests/rt.sh
@@ -415,11 +415,11 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20210324
+BL_DATE=20210331
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]]; then
-  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
+BL_DATE=20210331
 else
-  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}}
+BL_DATE=20210331
 fi
 
 INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20210324}

From 58da308e4852fea6cdd0c072b154c3c5a93b3f3a Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 29 Mar 2021 00:29:54 +0000
Subject: [PATCH 09/38] Search for BL_DATE= not BL_DATE as the latter is not
 unique

---
 tests/auto/jobs/bl.py | 2 +-
 tests/rt.sh           | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 tests/rt.sh

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index e166b3d5bf..05cbd65ec9 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -169,7 +169,7 @@ def update_rt_sh(job_obj, pr_repo_loc, bldate, branch):
     with open(f'{pr_repo_loc}/tests/rt.sh', 'r') as f:
         with open(f'{pr_repo_loc}/tests/rt.sh.new', 'w') as w:
             for line in f:
-                if 'BL_DATE' in line:
+                if 'BL_DATE=' in line:
                     logger.info('Found BL_DATE in line')
                     logger.info(f'Writing "BL_DATE-{bldate}" into file')
                     w.write(f'BL_DATE={bldate}\n')
diff --git a/tests/rt.sh b/tests/rt.sh
old mode 100644
new mode 100755
index d86ce0bcf2..86cb337b82
--- a/tests/rt.sh
+++ b/tests/rt.sh
@@ -417,9 +417,9 @@ fi
 
 BL_DATE=20210331
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]]; then
-BL_DATE=20210331
+  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else
-BL_DATE=20210331
+  RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}}
 fi
 
 INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20210324}

From a256db40a854f1b8090ee3a196ce7aa22234f1f0 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 29 Mar 2021 00:41:19 +0000
Subject: [PATCH 10/38] need to test date that doesn't match code

---
 tests/rt.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/rt.sh b/tests/rt.sh
index 86cb337b82..2042e343e0 100755
--- a/tests/rt.sh
+++ b/tests/rt.sh
@@ -415,7 +415,7 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20210331
+BL_DATE=20210324
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else

From 7798cac991fe6a7890d8526f4971a246c83005e7 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 29 Mar 2021 01:15:45 +0000
Subject: [PATCH 11/38] BL JOBS PASSED: hera.gnu. Updated rt.sh with new
 develop date: 20210331

---
 tests/rt.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100755 => 100644 tests/rt.sh

diff --git a/tests/rt.sh b/tests/rt.sh
old mode 100755
new mode 100644
index 2042e343e0..86cb337b82
--- a/tests/rt.sh
+++ b/tests/rt.sh
@@ -415,7 +415,7 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20210324
+BL_DATE=20210331
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else

From 1168e418bce2bf6e165851d3cc30937aa2e5f37e Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 29 Mar 2021 01:17:26 +0000
Subject: [PATCH 12/38] Adjusting to upstream date for test with RT

---
 tests/rt.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/rt.sh b/tests/rt.sh
index 86cb337b82..2042e343e0 100644
--- a/tests/rt.sh
+++ b/tests/rt.sh
@@ -415,7 +415,7 @@ if [[ $TESTS_FILE =~ '35d' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20210331
+BL_DATE=20210324
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else

From 2bce7e40af533fe4a51a00398c60d2e8998944ab Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 29 Mar 2021 01:54:49 +0000
Subject: [PATCH 13/38] RT JOBS PASSED: hera.gnu. Log file uploaded. skip-ci

---
 tests/RegressionTests_hera.gnu.log | 148 ++++++++++++++---------------
 1 file changed, 74 insertions(+), 74 deletions(-)

diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
index 235fcdd476..36da6d3188 100644
--- a/tests/RegressionTests_hera.gnu.log
+++ b/tests/RegressionTests_hera.gnu.log
@@ -1,19 +1,19 @@
-Thu Mar 25 14:46:21 UTC 2021
+Mon Mar 29 01:21:51 UTC 2021
 Start Regression test
 
-Compile 001 elapsed time 203 seconds. APP=ATM SUITES=FV3_GFS_2017_gfdlmp
+Compile 001 elapsed time 206 seconds. APP=ATM SUITES=FV3_GFS_2017_gfdlmp
 Compile 002 elapsed time 205 seconds. APP=ATM SUITES=FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v16_flake,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16_RRTMGP
 Compile 003 elapsed time 228 seconds. APP=ATM SUITES=FV3_GSD_v0,FV3_GFS_v16_thompson,FV3_RRFS_v1beta 32BIT=Y
-Compile 004 elapsed time 203 seconds. APP=ATM SUITES=HAFS_v0_hwrf_thompson,HAFS_v0_hwrf,FV3_GFS_v16b_ugwpv1,FV3_GFS_v16_ras
-Compile 005 elapsed time 132 seconds. APP=ATM 32BIT=Y DEBUG=Y
-Compile 006 elapsed time 100 seconds. APP=ATM SUITES=FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16_RRTMGP DEBUG=Y
+Compile 004 elapsed time 205 seconds. APP=ATM SUITES=HAFS_v0_hwrf_thompson,HAFS_v0_hwrf,FV3_GFS_v16b_ugwpv1,FV3_GFS_v16_ras
+Compile 005 elapsed time 133 seconds. APP=ATM 32BIT=Y DEBUG=Y
+Compile 006 elapsed time 99 seconds. APP=ATM SUITES=FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16_RRTMGP DEBUG=Y
 Compile 007 elapsed time 230 seconds. APP=ATM SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y
-Compile 008 elapsed time 102 seconds. APP=ATM SUITES=HAFS_v0_hwrf_thompson,HAFS_v0_hwrf,FV3_GFS_v16b_ugwpv1,FV3_GFS_v16_ras DEBUG=Y
-Compile 009 elapsed time 231 seconds. APP=S2S SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled
-Compile 010 elapsed time 213 seconds. APP=DATM_NEMS
+Compile 008 elapsed time 99 seconds. APP=ATM SUITES=HAFS_v0_hwrf_thompson,HAFS_v0_hwrf,FV3_GFS_v16b_ugwpv1,FV3_GFS_v16_ras DEBUG=Y
+Compile 009 elapsed time 230 seconds. APP=S2S SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled
+Compile 010 elapsed time 211 seconds. APP=DATM_NEMS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfdlmp
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfdlmp
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfdlmp
 Checking test 001 fv3_gfdlmp results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -58,13 +58,13 @@ Checking test 001 fv3_gfdlmp results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 230.658929
+  0: The total amount of wall time                        = 227.245900
 
 Test 001 fv3_gfdlmp PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v15p2
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v15p2
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v15p2
 Checking test 002 fv3_gfs_v15p2 results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -129,13 +129,13 @@ Checking test 002 fv3_gfs_v15p2 results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 633.270456
+  0: The total amount of wall time                        = 607.707584
 
 Test 002 fv3_gfs_v15p2 PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16
 Checking test 003 fv3_gfs_v16 results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -212,13 +212,13 @@ Checking test 003 fv3_gfs_v16 results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 653.003005
+  0: The total amount of wall time                        = 635.202902
 
 Test 003 fv3_gfs_v16 PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_restart
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_restart
 Checking test 004 fv3_gfs_v16_restart results ....
  Comparing phyf048.tile1.nc .........OK
  Comparing phyf048.tile2.nc .........OK
@@ -265,13 +265,13 @@ Checking test 004 fv3_gfs_v16_restart results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 337.439096
+  0: The total amount of wall time                        = 329.909174
 
 Test 004 fv3_gfs_v16_restart PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_stochy
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_stochy
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_stochy
 Checking test 005 fv3_gfs_v16_stochy results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -336,13 +336,13 @@ Checking test 005 fv3_gfs_v16_stochy results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 352.897264
+  0: The total amount of wall time                        = 354.802633
 
 Test 005 fv3_gfs_v16_stochy PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_flake
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_flake
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_flake
 Checking test 006 fv3_gfs_v16_flake results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -407,13 +407,13 @@ Checking test 006 fv3_gfs_v16_flake results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 612.923154
+  0: The total amount of wall time                        = 592.571301
 
 Test 006 fv3_gfs_v16_flake PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v15p2_RRTMGP
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v15p2_RRTMGP
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v15p2_RRTMGP
 Checking test 007 fv3_gfs_v15p2_RRTMGP results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -478,13 +478,13 @@ Checking test 007 fv3_gfs_v15p2_RRTMGP results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 693.671152
+  0: The total amount of wall time                        = 694.806268
 
 Test 007 fv3_gfs_v15p2_RRTMGP PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_RRTMGP
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_RRTMGP
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_RRTMGP
 Checking test 008 fv3_gfs_v16_RRTMGP results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -549,13 +549,13 @@ Checking test 008 fv3_gfs_v16_RRTMGP results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 701.608400
+  0: The total amount of wall time                        = 675.733062
 
 Test 008 fv3_gfs_v16_RRTMGP PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gsd
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gsd
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gsd
 Checking test 009 fv3_gsd results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -644,13 +644,13 @@ Checking test 009 fv3_gsd results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 820.313640
+  0: The total amount of wall time                        = 828.761641
 
 Test 009 fv3_gsd PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_thompson
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_thompson
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_thompson
 Checking test 010 fv3_thompson results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -715,13 +715,13 @@ Checking test 010 fv3_thompson results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 397.498999
+  0: The total amount of wall time                        = 383.373478
 
 Test 010 fv3_thompson PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_thompson_no_aero
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_thompson_no_aero
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_thompson_no_aero
 Checking test 011 fv3_thompson_no_aero results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -786,13 +786,13 @@ Checking test 011 fv3_thompson_no_aero results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 365.386068
+  0: The total amount of wall time                        = 400.329658
 
 Test 011 fv3_thompson_no_aero PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_rrfs_v1beta
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_rrfs_v1beta
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_rrfs_v1beta
 Checking test 012 fv3_rrfs_v1beta results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -857,13 +857,13 @@ Checking test 012 fv3_rrfs_v1beta results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 421.769696
+  0: The total amount of wall time                        = 440.519148
 
 Test 012 fv3_rrfs_v1beta PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/HAFS_v0_HWRF_thompson
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_HAFS_v0_hwrf_thompson
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_HAFS_v0_hwrf_thompson
 Checking test 013 fv3_HAFS_v0_hwrf_thompson results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -928,13 +928,13 @@ Checking test 013 fv3_HAFS_v0_hwrf_thompson results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 618.573402
+  0: The total amount of wall time                        = 621.937992
 
 Test 013 fv3_HAFS_v0_hwrf_thompson PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/ESG_HAFS_v0_HWRF_thompson
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_esg_HAFS_v0_hwrf_thompson
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_esg_HAFS_v0_hwrf_thompson
 Checking test 014 fv3_esg_HAFS_v0_hwrf_thompson results ....
  Comparing atmos_4xdaily.nc .........OK
  Comparing phyf000.nc .........OK
@@ -949,13 +949,13 @@ Checking test 014 fv3_esg_HAFS_v0_hwrf_thompson results ....
  Comparing RESTART/sfc_data.nc .........OK
  Comparing RESTART/phy_data.nc .........OK
 
- 0: The total amount of wall time                        = 445.981440
+ 0: The total amount of wall time                        = 444.561950
 
 Test 014 fv3_esg_HAFS_v0_hwrf_thompson PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfsv16_ugwpv1
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfsv16_ugwpv1
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfsv16_ugwpv1
 Checking test 015 fv3_gfsv16_ugwpv1 results ....
  Comparing phyf000.tile1.nc .........OK
  Comparing phyf000.tile2.nc .........OK
@@ -1014,13 +1014,13 @@ Checking test 015 fv3_gfsv16_ugwpv1 results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 893.286609
+  0: The total amount of wall time                        = 898.725646
 
 Test 015 fv3_gfsv16_ugwpv1 PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfsv16_ugwpv1_warmstart
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfsv16_ugwpv1_warmstart
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfsv16_ugwpv1_warmstart
 Checking test 016 fv3_gfsv16_ugwpv1_warmstart results ....
  Comparing phyf000.tile1.nc .........OK
  Comparing phyf000.tile2.nc .........OK
@@ -1079,13 +1079,13 @@ Checking test 016 fv3_gfsv16_ugwpv1_warmstart results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 904.914218
+  0: The total amount of wall time                        = 909.760157
 
 Test 016 fv3_gfsv16_ugwpv1_warmstart PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_ras
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_ras
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_ras
 Checking test 017 fv3_gfs_v16_ras results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1150,13 +1150,13 @@ Checking test 017 fv3_gfs_v16_ras results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 571.976108
+  0: The total amount of wall time                        = 577.451401
 
 Test 017 fv3_gfs_v16_ras PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_control_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_control_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_control_debug
 Checking test 018 fv3_control_debug results ....
  Comparing phyf000.tile1.nc .........OK
  Comparing phyf000.tile2.nc .........OK
@@ -1183,13 +1183,13 @@ Checking test 018 fv3_control_debug results ....
  Comparing dynf006.tile5.nc .........OK
  Comparing dynf006.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 87.764311
+  0: The total amount of wall time                        = 84.860555
 
 Test 018 fv3_control_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_regional_control_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_regional_control_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_regional_control_debug
 Checking test 019 fv3_regional_control_debug results ....
  Comparing atmos_4xdaily.nc .........OK
  Comparing fv3_history2d.nc .........OK
@@ -1197,13 +1197,13 @@ Checking test 019 fv3_regional_control_debug results ....
  Comparing RESTART/fv_core.res.tile1_new.nc .........OK
  Comparing RESTART/fv_tracer.res.tile1_new.nc .........OK
 
- 0: The total amount of wall time                        = 165.637467
+ 0: The total amount of wall time                        = 168.043017
 
 Test 019 fv3_regional_control_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_rrfs_v1beta_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_rrfs_v1beta_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_rrfs_v1beta_debug
 Checking test 020 fv3_rrfs_v1beta_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1268,13 +1268,13 @@ Checking test 020 fv3_rrfs_v1beta_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 117.136661
+  0: The total amount of wall time                        = 118.328498
 
 Test 020 fv3_rrfs_v1beta_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gsd_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gsd_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gsd_debug
 Checking test 021 fv3_gsd_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1339,13 +1339,13 @@ Checking test 021 fv3_gsd_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 123.939896
+  0: The total amount of wall time                        = 120.191853
 
 Test 021 fv3_gsd_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_thompson_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_thompson_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_thompson_debug
 Checking test 022 fv3_thompson_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1410,13 +1410,13 @@ Checking test 022 fv3_thompson_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 200.658729
+  0: The total amount of wall time                        = 200.893572
 
 Test 022 fv3_thompson_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_thompson_no_aero_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_thompson_no_aero_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_thompson_no_aero_debug
 Checking test 023 fv3_thompson_no_aero_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1481,13 +1481,13 @@ Checking test 023 fv3_thompson_no_aero_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 192.644521
+  0: The total amount of wall time                        = 191.549437
 
 Test 023 fv3_thompson_no_aero_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v15p2_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v15p2_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v15p2_debug
 Checking test 024 fv3_gfs_v15p2_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1552,13 +1552,13 @@ Checking test 024 fv3_gfs_v15p2_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 280.811160
+  0: The total amount of wall time                        = 277.220240
 
 Test 024 fv3_gfs_v15p2_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_debug
 Checking test 025 fv3_gfs_v16_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1623,13 +1623,13 @@ Checking test 025 fv3_gfs_v16_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 170.549779
+  0: The total amount of wall time                        = 167.585897
 
 Test 025 fv3_gfs_v16_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v15p2_RRTMGP_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v15p2_RRTMGP_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v15p2_RRTMGP_debug
 Checking test 026 fv3_gfs_v15p2_RRTMGP_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1694,13 +1694,13 @@ Checking test 026 fv3_gfs_v15p2_RRTMGP_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 341.451969
+  0: The total amount of wall time                        = 325.519801
 
 Test 026 fv3_gfs_v15p2_RRTMGP_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_RRTMGP_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_RRTMGP_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_RRTMGP_debug
 Checking test 027 fv3_gfs_v16_RRTMGP_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1765,13 +1765,13 @@ Checking test 027 fv3_gfs_v16_RRTMGP_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 339.837849
+  0: The total amount of wall time                        = 338.969370
 
 Test 027 fv3_gfs_v16_RRTMGP_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_multigases
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_multigases
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_multigases
 Checking test 028 fv3_multigases results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1842,13 +1842,13 @@ Checking test 028 fv3_multigases results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 444.581635
+  0: The total amount of wall time                        = 441.383308
 
 Test 028 fv3_multigases PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/HAFS_v0_HWRF_thompson_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_HAFS_v0_hwrf_thompson_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_HAFS_v0_hwrf_thompson_debug
 Checking test 029 fv3_HAFS_v0_hwrf_thompson_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1913,13 +1913,13 @@ Checking test 029 fv3_HAFS_v0_hwrf_thompson_debug results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 134.692318
+  0: The total amount of wall time                        = 131.072335
 
 Test 029 fv3_HAFS_v0_hwrf_thompson_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/ESG_HAFS_v0_HWRF_thompson_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_esg_HAFS_v0_hwrf_thompson_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_esg_HAFS_v0_hwrf_thompson_debug
 Checking test 030 fv3_esg_HAFS_v0_hwrf_thompson_debug results ....
  Comparing atmos_4xdaily.nc .........OK
  Comparing phyf000.nc .........OK
@@ -1934,13 +1934,13 @@ Checking test 030 fv3_esg_HAFS_v0_hwrf_thompson_debug results ....
  Comparing RESTART/sfc_data.nc .........OK
  Comparing RESTART/phy_data.nc .........OK
 
- 0: The total amount of wall time                        = 227.558155
+ 0: The total amount of wall time                        = 221.920625
 
 Test 030 fv3_esg_HAFS_v0_hwrf_thompson_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfsv16_ugwpv1_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfsv16_ugwpv1_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfsv16_ugwpv1_debug
 Checking test 031 fv3_gfsv16_ugwpv1_debug results ....
  Comparing phyf000.tile1.nc .........OK
  Comparing phyf000.tile2.nc .........OK
@@ -1999,13 +1999,13 @@ Checking test 031 fv3_gfsv16_ugwpv1_debug results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 392.457886
+  0: The total amount of wall time                        = 389.640352
 
 Test 031 fv3_gfsv16_ugwpv1_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_ras_debug
-working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_ras_debug
+working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_ras_debug
 Checking test 032 fv3_gfs_v16_ras_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -2070,11 +2070,11 @@ Checking test 032 fv3_gfs_v16_ras_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 224.014594
+  0: The total amount of wall time                        = 215.609388
 
 Test 032 fv3_gfs_v16_ras_debug PASS
 
 
 REGRESSION TEST WAS SUCCESSFUL
-Thu Mar 25 16:09:45 UTC 2021
-Elapsed time: 01h:23m:25s. Have a nice day!
+Mon Mar 29 01:54:48 UTC 2021
+Elapsed time: 00h:32m:58s. Have a nice day!

From 7981107b29a5be124c53e436f10e53075472cf78 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 29 Mar 2021 02:02:40 +0000
Subject: [PATCH 14/38] Reverting regression test log, and fixing hera
 directories for emc.nemspara

---
 tests/RegressionTests_hera.gnu.log | 148 ++++++++++++++---------------
 tests/auto/jobs/bl.py              |   6 +-
 2 files changed, 77 insertions(+), 77 deletions(-)

diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
index 36da6d3188..235fcdd476 100644
--- a/tests/RegressionTests_hera.gnu.log
+++ b/tests/RegressionTests_hera.gnu.log
@@ -1,19 +1,19 @@
-Mon Mar 29 01:21:51 UTC 2021
+Thu Mar 25 14:46:21 UTC 2021
 Start Regression test
 
-Compile 001 elapsed time 206 seconds. APP=ATM SUITES=FV3_GFS_2017_gfdlmp
+Compile 001 elapsed time 203 seconds. APP=ATM SUITES=FV3_GFS_2017_gfdlmp
 Compile 002 elapsed time 205 seconds. APP=ATM SUITES=FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v16_flake,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16_RRTMGP
 Compile 003 elapsed time 228 seconds. APP=ATM SUITES=FV3_GSD_v0,FV3_GFS_v16_thompson,FV3_RRFS_v1beta 32BIT=Y
-Compile 004 elapsed time 205 seconds. APP=ATM SUITES=HAFS_v0_hwrf_thompson,HAFS_v0_hwrf,FV3_GFS_v16b_ugwpv1,FV3_GFS_v16_ras
-Compile 005 elapsed time 133 seconds. APP=ATM 32BIT=Y DEBUG=Y
-Compile 006 elapsed time 99 seconds. APP=ATM SUITES=FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16_RRTMGP DEBUG=Y
+Compile 004 elapsed time 203 seconds. APP=ATM SUITES=HAFS_v0_hwrf_thompson,HAFS_v0_hwrf,FV3_GFS_v16b_ugwpv1,FV3_GFS_v16_ras
+Compile 005 elapsed time 132 seconds. APP=ATM 32BIT=Y DEBUG=Y
+Compile 006 elapsed time 100 seconds. APP=ATM SUITES=FV3_GFS_v15p2,FV3_GFS_v16,FV3_GFS_v15p2_RRTMGP,FV3_GFS_v16_RRTMGP DEBUG=Y
 Compile 007 elapsed time 230 seconds. APP=ATM SUITES=FV3_GFS_2017_fv3wam 32BIT=Y MULTI_GASES=Y
-Compile 008 elapsed time 99 seconds. APP=ATM SUITES=HAFS_v0_hwrf_thompson,HAFS_v0_hwrf,FV3_GFS_v16b_ugwpv1,FV3_GFS_v16_ras DEBUG=Y
-Compile 009 elapsed time 230 seconds. APP=S2S SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled
-Compile 010 elapsed time 211 seconds. APP=DATM_NEMS
+Compile 008 elapsed time 102 seconds. APP=ATM SUITES=HAFS_v0_hwrf_thompson,HAFS_v0_hwrf,FV3_GFS_v16b_ugwpv1,FV3_GFS_v16_ras DEBUG=Y
+Compile 009 elapsed time 231 seconds. APP=S2S SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled
+Compile 010 elapsed time 213 seconds. APP=DATM_NEMS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfdlmp
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfdlmp
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfdlmp
 Checking test 001 fv3_gfdlmp results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -58,13 +58,13 @@ Checking test 001 fv3_gfdlmp results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 227.245900
+  0: The total amount of wall time                        = 230.658929
 
 Test 001 fv3_gfdlmp PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v15p2
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v15p2
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v15p2
 Checking test 002 fv3_gfs_v15p2 results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -129,13 +129,13 @@ Checking test 002 fv3_gfs_v15p2 results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 607.707584
+  0: The total amount of wall time                        = 633.270456
 
 Test 002 fv3_gfs_v15p2 PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16
 Checking test 003 fv3_gfs_v16 results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -212,13 +212,13 @@ Checking test 003 fv3_gfs_v16 results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 635.202902
+  0: The total amount of wall time                        = 653.003005
 
 Test 003 fv3_gfs_v16 PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_restart
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_restart
 Checking test 004 fv3_gfs_v16_restart results ....
  Comparing phyf048.tile1.nc .........OK
  Comparing phyf048.tile2.nc .........OK
@@ -265,13 +265,13 @@ Checking test 004 fv3_gfs_v16_restart results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 329.909174
+  0: The total amount of wall time                        = 337.439096
 
 Test 004 fv3_gfs_v16_restart PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_stochy
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_stochy
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_stochy
 Checking test 005 fv3_gfs_v16_stochy results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -336,13 +336,13 @@ Checking test 005 fv3_gfs_v16_stochy results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 354.802633
+  0: The total amount of wall time                        = 352.897264
 
 Test 005 fv3_gfs_v16_stochy PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_flake
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_flake
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_flake
 Checking test 006 fv3_gfs_v16_flake results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -407,13 +407,13 @@ Checking test 006 fv3_gfs_v16_flake results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 592.571301
+  0: The total amount of wall time                        = 612.923154
 
 Test 006 fv3_gfs_v16_flake PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v15p2_RRTMGP
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v15p2_RRTMGP
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v15p2_RRTMGP
 Checking test 007 fv3_gfs_v15p2_RRTMGP results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -478,13 +478,13 @@ Checking test 007 fv3_gfs_v15p2_RRTMGP results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 694.806268
+  0: The total amount of wall time                        = 693.671152
 
 Test 007 fv3_gfs_v15p2_RRTMGP PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_RRTMGP
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_RRTMGP
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_RRTMGP
 Checking test 008 fv3_gfs_v16_RRTMGP results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -549,13 +549,13 @@ Checking test 008 fv3_gfs_v16_RRTMGP results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 675.733062
+  0: The total amount of wall time                        = 701.608400
 
 Test 008 fv3_gfs_v16_RRTMGP PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gsd
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gsd
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gsd
 Checking test 009 fv3_gsd results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -644,13 +644,13 @@ Checking test 009 fv3_gsd results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 828.761641
+  0: The total amount of wall time                        = 820.313640
 
 Test 009 fv3_gsd PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_thompson
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_thompson
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_thompson
 Checking test 010 fv3_thompson results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -715,13 +715,13 @@ Checking test 010 fv3_thompson results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 383.373478
+  0: The total amount of wall time                        = 397.498999
 
 Test 010 fv3_thompson PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_thompson_no_aero
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_thompson_no_aero
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_thompson_no_aero
 Checking test 011 fv3_thompson_no_aero results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -786,13 +786,13 @@ Checking test 011 fv3_thompson_no_aero results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 400.329658
+  0: The total amount of wall time                        = 365.386068
 
 Test 011 fv3_thompson_no_aero PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_rrfs_v1beta
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_rrfs_v1beta
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_rrfs_v1beta
 Checking test 012 fv3_rrfs_v1beta results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -857,13 +857,13 @@ Checking test 012 fv3_rrfs_v1beta results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 440.519148
+  0: The total amount of wall time                        = 421.769696
 
 Test 012 fv3_rrfs_v1beta PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/HAFS_v0_HWRF_thompson
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_HAFS_v0_hwrf_thompson
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_HAFS_v0_hwrf_thompson
 Checking test 013 fv3_HAFS_v0_hwrf_thompson results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -928,13 +928,13 @@ Checking test 013 fv3_HAFS_v0_hwrf_thompson results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 621.937992
+  0: The total amount of wall time                        = 618.573402
 
 Test 013 fv3_HAFS_v0_hwrf_thompson PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/ESG_HAFS_v0_HWRF_thompson
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_esg_HAFS_v0_hwrf_thompson
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_esg_HAFS_v0_hwrf_thompson
 Checking test 014 fv3_esg_HAFS_v0_hwrf_thompson results ....
  Comparing atmos_4xdaily.nc .........OK
  Comparing phyf000.nc .........OK
@@ -949,13 +949,13 @@ Checking test 014 fv3_esg_HAFS_v0_hwrf_thompson results ....
  Comparing RESTART/sfc_data.nc .........OK
  Comparing RESTART/phy_data.nc .........OK
 
- 0: The total amount of wall time                        = 444.561950
+ 0: The total amount of wall time                        = 445.981440
 
 Test 014 fv3_esg_HAFS_v0_hwrf_thompson PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfsv16_ugwpv1
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfsv16_ugwpv1
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfsv16_ugwpv1
 Checking test 015 fv3_gfsv16_ugwpv1 results ....
  Comparing phyf000.tile1.nc .........OK
  Comparing phyf000.tile2.nc .........OK
@@ -1014,13 +1014,13 @@ Checking test 015 fv3_gfsv16_ugwpv1 results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 898.725646
+  0: The total amount of wall time                        = 893.286609
 
 Test 015 fv3_gfsv16_ugwpv1 PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfsv16_ugwpv1_warmstart
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfsv16_ugwpv1_warmstart
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfsv16_ugwpv1_warmstart
 Checking test 016 fv3_gfsv16_ugwpv1_warmstart results ....
  Comparing phyf000.tile1.nc .........OK
  Comparing phyf000.tile2.nc .........OK
@@ -1079,13 +1079,13 @@ Checking test 016 fv3_gfsv16_ugwpv1_warmstart results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 909.760157
+  0: The total amount of wall time                        = 904.914218
 
 Test 016 fv3_gfsv16_ugwpv1_warmstart PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_ras
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_ras
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_ras
 Checking test 017 fv3_gfs_v16_ras results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1150,13 +1150,13 @@ Checking test 017 fv3_gfs_v16_ras results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 577.451401
+  0: The total amount of wall time                        = 571.976108
 
 Test 017 fv3_gfs_v16_ras PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_control_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_control_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_control_debug
 Checking test 018 fv3_control_debug results ....
  Comparing phyf000.tile1.nc .........OK
  Comparing phyf000.tile2.nc .........OK
@@ -1183,13 +1183,13 @@ Checking test 018 fv3_control_debug results ....
  Comparing dynf006.tile5.nc .........OK
  Comparing dynf006.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 84.860555
+  0: The total amount of wall time                        = 87.764311
 
 Test 018 fv3_control_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_regional_control_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_regional_control_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_regional_control_debug
 Checking test 019 fv3_regional_control_debug results ....
  Comparing atmos_4xdaily.nc .........OK
  Comparing fv3_history2d.nc .........OK
@@ -1197,13 +1197,13 @@ Checking test 019 fv3_regional_control_debug results ....
  Comparing RESTART/fv_core.res.tile1_new.nc .........OK
  Comparing RESTART/fv_tracer.res.tile1_new.nc .........OK
 
- 0: The total amount of wall time                        = 168.043017
+ 0: The total amount of wall time                        = 165.637467
 
 Test 019 fv3_regional_control_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_rrfs_v1beta_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_rrfs_v1beta_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_rrfs_v1beta_debug
 Checking test 020 fv3_rrfs_v1beta_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1268,13 +1268,13 @@ Checking test 020 fv3_rrfs_v1beta_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 118.328498
+  0: The total amount of wall time                        = 117.136661
 
 Test 020 fv3_rrfs_v1beta_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gsd_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gsd_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gsd_debug
 Checking test 021 fv3_gsd_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1339,13 +1339,13 @@ Checking test 021 fv3_gsd_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 120.191853
+  0: The total amount of wall time                        = 123.939896
 
 Test 021 fv3_gsd_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_thompson_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_thompson_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_thompson_debug
 Checking test 022 fv3_thompson_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1410,13 +1410,13 @@ Checking test 022 fv3_thompson_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 200.893572
+  0: The total amount of wall time                        = 200.658729
 
 Test 022 fv3_thompson_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_thompson_no_aero_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_thompson_no_aero_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_thompson_no_aero_debug
 Checking test 023 fv3_thompson_no_aero_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1481,13 +1481,13 @@ Checking test 023 fv3_thompson_no_aero_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 191.549437
+  0: The total amount of wall time                        = 192.644521
 
 Test 023 fv3_thompson_no_aero_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v15p2_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v15p2_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v15p2_debug
 Checking test 024 fv3_gfs_v15p2_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1552,13 +1552,13 @@ Checking test 024 fv3_gfs_v15p2_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 277.220240
+  0: The total amount of wall time                        = 280.811160
 
 Test 024 fv3_gfs_v15p2_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_debug
 Checking test 025 fv3_gfs_v16_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1623,13 +1623,13 @@ Checking test 025 fv3_gfs_v16_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 167.585897
+  0: The total amount of wall time                        = 170.549779
 
 Test 025 fv3_gfs_v16_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v15p2_RRTMGP_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v15p2_RRTMGP_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v15p2_RRTMGP_debug
 Checking test 026 fv3_gfs_v15p2_RRTMGP_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1694,13 +1694,13 @@ Checking test 026 fv3_gfs_v15p2_RRTMGP_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 325.519801
+  0: The total amount of wall time                        = 341.451969
 
 Test 026 fv3_gfs_v15p2_RRTMGP_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_RRTMGP_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_RRTMGP_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_RRTMGP_debug
 Checking test 027 fv3_gfs_v16_RRTMGP_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1765,13 +1765,13 @@ Checking test 027 fv3_gfs_v16_RRTMGP_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 338.969370
+  0: The total amount of wall time                        = 339.837849
 
 Test 027 fv3_gfs_v16_RRTMGP_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_multigases
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_multigases
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_multigases
 Checking test 028 fv3_multigases results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1842,13 +1842,13 @@ Checking test 028 fv3_multigases results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 441.383308
+  0: The total amount of wall time                        = 444.581635
 
 Test 028 fv3_multigases PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/HAFS_v0_HWRF_thompson_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_HAFS_v0_hwrf_thompson_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_HAFS_v0_hwrf_thompson_debug
 Checking test 029 fv3_HAFS_v0_hwrf_thompson_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -1913,13 +1913,13 @@ Checking test 029 fv3_HAFS_v0_hwrf_thompson_debug results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 131.072335
+  0: The total amount of wall time                        = 134.692318
 
 Test 029 fv3_HAFS_v0_hwrf_thompson_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/ESG_HAFS_v0_HWRF_thompson_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_esg_HAFS_v0_hwrf_thompson_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_esg_HAFS_v0_hwrf_thompson_debug
 Checking test 030 fv3_esg_HAFS_v0_hwrf_thompson_debug results ....
  Comparing atmos_4xdaily.nc .........OK
  Comparing phyf000.nc .........OK
@@ -1934,13 +1934,13 @@ Checking test 030 fv3_esg_HAFS_v0_hwrf_thompson_debug results ....
  Comparing RESTART/sfc_data.nc .........OK
  Comparing RESTART/phy_data.nc .........OK
 
- 0: The total amount of wall time                        = 221.920625
+ 0: The total amount of wall time                        = 227.558155
 
 Test 030 fv3_esg_HAFS_v0_hwrf_thompson_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfsv16_ugwpv1_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfsv16_ugwpv1_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfsv16_ugwpv1_debug
 Checking test 031 fv3_gfsv16_ugwpv1_debug results ....
  Comparing phyf000.tile1.nc .........OK
  Comparing phyf000.tile2.nc .........OK
@@ -1999,13 +1999,13 @@ Checking test 031 fv3_gfsv16_ugwpv1_debug results ....
  Comparing RESTART/phy_data.tile5.nc .........OK
  Comparing RESTART/phy_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 389.640352
+  0: The total amount of wall time                        = 392.457886
 
 Test 031 fv3_gfsv16_ugwpv1_debug PASS
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20210324/GNU/fv3_gfs_v16_ras_debug
-working dir  = /scratch1/NCEPDEV/stmp2/Brian.Curtis/FV3_RT/rt_210460/fv3_gfs_v16_ras_debug
+working dir  = /scratch1/NCEPDEV/stmp2/emc.nemspara/FV3_RT/rt_84193/fv3_gfs_v16_ras_debug
 Checking test 032 fv3_gfs_v16_ras_debug results ....
  Comparing atmos_4xdaily.tile1.nc .........OK
  Comparing atmos_4xdaily.tile2.nc .........OK
@@ -2070,11 +2070,11 @@ Checking test 032 fv3_gfs_v16_ras_debug results ....
  Comparing RESTART/sfc_data.tile5.nc .........OK
  Comparing RESTART/sfc_data.tile6.nc .........OK
 
-  0: The total amount of wall time                        = 215.609388
+  0: The total amount of wall time                        = 224.014594
 
 Test 032 fv3_gfs_v16_ras_debug PASS
 
 
 REGRESSION TEST WAS SUCCESSFUL
-Mon Mar 29 01:54:48 UTC 2021
-Elapsed time: 00h:32m:58s. Have a nice day!
+Thu Mar 25 16:09:45 UTC 2021
+Elapsed time: 01h:23m:25s. Have a nice day!
diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 05cbd65ec9..6510224d53 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -19,9 +19,9 @@ def run(job_obj):
 def set_directories(job_obj):
     logger = logging.getLogger('BL/SET_DIRECTORIES')
     if job_obj.machine == 'hera':
-        workdir = '/scratch1/NCEPDEV/nems/Brian.Curtis/autort/pr'
-        blstore = '/scratch1/NCEPDEV/nems/Brian.Curtis/RT/NEMSfv3gfs'
-        rtbldir = '/scratch1/NCEPDEV/stmp4/Brian.Curtis/FV3_RT/'\
+        workdir = '/scratch1/NCEPDEV/nems/emc.nemspara/autort/pr'
+        blstore = '/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs'
+        rtbldir = '/scratch1/NCEPDEV/stmp4/emc.nemspara/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'jet':
         workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr'

From 5df473d5111c4625cb97a894ef1c7f437d16f852 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 29 Mar 2021 02:27:35 +0000
Subject: [PATCH 15/38] Don't need to skip-ci any more

---
 tests/auto/jobs/rt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
index 1797df5ec8..c2c96c738a 100644
--- a/tests/auto/jobs/rt.py
+++ b/tests/auto/jobs/rt.py
@@ -98,7 +98,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch):
             [f'git pull --ff-only origin {branch}', pr_repo_loc],
             [f'git add {rt_log}', pr_repo_loc],
             [f'git commit -m "RT JOBS PASSED: {job_obj.machine}'
-             f'.{job_obj.compiler}. Log file uploaded. skip-ci"',
+             f'.{job_obj.compiler}. Log file uploaded."',
              pr_repo_loc],
             ['sleep 10', pr_repo_loc],
             [f'git push origin {branch}', pr_repo_loc]

From 20634a5ea414c0b8ac72728330d7cca205a1202a Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Tue, 30 Mar 2021 15:34:19 +0000
Subject: [PATCH 16/38] Lets look in ufs-community and not BrianCurtis-NOAA for
 repo PR's

---
 tests/auto/rt_auto.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py
index 7876897572..de642d0afa 100644
--- a/tests/auto/rt_auto.py
+++ b/tests/auto/rt_auto.py
@@ -231,7 +231,7 @@ def setup_env():
     # Dictionary of GitHub repositories to check
     repo_dict = [{
         'name': 'ufs-weather-model',
-        'address': 'BrianCurtis-NOAA/ufs-weather-model',
+        'address': 'ufs-community/ufs-weather-model',
         'base': 'develop'
     }]
 

From 6297c157079fe394d9dd622fdf1b89783da082ee Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Tue, 30 Mar 2021 15:41:07 +0000
Subject: [PATCH 17/38] Adjustments for GAEA and HERA

---
 tests/auto/jobs/bl.py | 6 +++---
 tests/auto/jobs/rt.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 6510224d53..366a3347fb 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -29,9 +29,9 @@ def set_directories(job_obj):
         rtbldir = '/lfs4/HFIP/hfv3gfs/emc.nemspara/RT_BASELINE/'\
                  f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'gaea':
-        workdir = '/lustre/f2/pdata/ncep/Brian.Curtis/autort/pr'
-        blstore = '/lustre/f2/pdata/esrl/gsd/ufs/ufs-weather-model/RT'
-        rtbldir = '/lustre/f2/scratch/Brian.Curtis/FV3_RT/'\
+        workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr'
+        blstore = '/lustre/f2/pdata/ncep_shared/emc.nemspara/RT'
+        rtbldir = '/lustre/f2/scratch/emc.nemspara/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'orion':
         workdir = '/work/noaa/nems/emc.nemspara/autort/pr'
diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
index c2c96c738a..568bba39ee 100644
--- a/tests/auto/jobs/rt.py
+++ b/tests/auto/jobs/rt.py
@@ -15,11 +15,11 @@ def run(job_obj):
 def set_directories(job_obj):
     logger = logging.getLogger('RT/SET_DIRECTORIES')
     if job_obj.machine == 'hera':
-        workdir = '/scratch1/NCEPDEV/nems/Brian.Curtis/autort/pr'
+        workdir = '/scratch1/NCEPDEV/nems/emc.nemspara/autort/pr'
     elif job_obj.machine == 'jet':
         workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr'
     elif job_obj.machine == 'gaea':
-        workdir = '/lustre/f2/pdata/ncep/Brian.Curtis/autort/pr'
+        workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr'
     elif job_obj.machine == 'orion':
         workdir = '/work/noaa/nems/emc.nemspara/autort/pr'
     elif job_obj.machine == 'cheyenne':

From 3192ab0b80fd35813e144bbd5333b93efb68f113 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Tue, 30 Mar 2021 18:35:00 +0000
Subject: [PATCH 18/38] Allow empty commit messages

---
 tests/auto/jobs/bl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 366a3347fb..35fb65bfd4 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -188,7 +188,7 @@ def update_rt_sh(job_obj, pr_repo_loc, bldate, branch):
          pr_repo_loc],
 
         [f'git add {pr_repo_loc}/tests/rt.sh', pr_repo_loc],
-        [f'git commit -m "BL JOBS PASSED: {job_obj.machine}'
+        [f'git commit --allow-empty -m "BL JOBS PASSED: {job_obj.machine}'
          f'.{job_obj.compiler}. Updated rt.sh with new develop date: '
          f'{bldate}"',
          pr_repo_loc],

From 3a1daf772ffe74b33f7101118f86b6d92ed30603 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Tue, 30 Mar 2021 19:52:44 +0000
Subject: [PATCH 19/38] Run RT job after successful BL Job

---
 tests/auto/jobs/bl.py | 109 ++++++++++++++++++++----------------------
 1 file changed, 53 insertions(+), 56 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 35fb65bfd4..82ed983e83 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -3,17 +3,17 @@
 import logging
 import os
 import sys
-
+from . import rt
 
 def run(job_obj):
     logger = logging.getLogger('BL/RUN')
-    bldate = get_bl_date(job_obj)
+    #bldate = get_bl_date(job_obj)
     workdir, rtbldir, blstore = set_directories(job_obj)
-    bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
-    if not check_for_bl_dir(bldir):
-        branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
-        run_regression_test(job_obj, pr_repo_loc)
-        post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, branch)
+    #bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
+    #if not check_for_bl_dir(bldir):
+    branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
+    run_regression_test(job_obj, pr_repo_loc)
+    post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch)
 
 
 def set_directories(job_obj):
@@ -73,27 +73,27 @@ def create_bl_dir(bldir):
             raise FileNotFoundError
 
 
-def get_bl_date(job_obj):
-    logger = logging.getLogger('BL/GET_BL_DATE')
-    for line in job_obj.preq_dict['preq'].body.splitlines():
-        if 'BL_DATE:' in line:
-            bldate = line
-            bldate = bldate.replace('BL_DATE:', '')
-            bldate = bldate.replace(' ', '')
-            if len(bldate) != 8:
-                print(f'Date: {bldate} is not formatted YYYYMMDD')
-                raise ValueError
-            logger.info(f'BL_DATE: {bldate}')
-            bl_format = '%Y%m%d'
-            try:
-                datetime.datetime.strptime(bldate, bl_format)
-            except ValueError:
-                logger.info(f'Date {bldate} is not formatted YYYYMMDD')
-                raise ValueError
-            return bldate
-    logger.critical('"BL_DATE:YYYYMMDD" needs to be in the PR body.'\
-                    'On its own line. Stopping')
-    raise ValueError
+#def get_bl_date(job_obj):
+#    logger = logging.getLogger('BL/GET_BL_DATE')
+#    for line in job_obj.preq_dict['preq'].body.splitlines():
+#        if 'BL_DATE:' in line:
+#            bldate = line
+#            bldate = bldate.replace('BL_DATE:', '')
+#            bldate = bldate.replace(' ', '')
+#            if len(bldate) != 8:
+#                print(f'Date: {bldate} is not formatted YYYYMMDD')
+#                raise ValueError
+#            logger.info(f'BL_DATE: {bldate}')
+#            bl_format = '%Y%m%d'
+#            try:
+#                datetime.datetime.strptime(bldate, bl_format)
+#            except ValueError:
+#                logger.info(f'Date {bldate} is not formatted YYYYMMDD')
+#                raise ValueError
+#            return bldate
+#    logger.critical('"BL_DATE:YYYYMMDD" needs to be in the PR body.'\
+#                    'On its own line. Stopping')
+#    raise ValueError
 
 
 def run_regression_test(job_obj, pr_repo_loc):
@@ -148,55 +148,52 @@ def clone_pr_repo(job_obj, workdir):
     return branch, pr_repo_loc, repo_dir_str
 
 
-def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir, branch):
+def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch):
     logger = logging.getLogger('BL/MOVE_RT_LOGS')
     rt_log = f'tests/RegressionTests_{job_obj.machine}'\
              f'.{job_obj.compiler}.log'
     filepath = f'{pr_repo_loc}/{rt_log}'
     rt_dir, logfile_pass = process_logfile(job_obj, filepath)
     if logfile_pass:
-        bldate = get_bl_date(job_obj)
+        bldate = get_bl_date(job_obj, pr_repo_loc)
+    	bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
         create_bl_dir(bldir)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
         job_obj.run_commands(logger, move_bl_command)
-        update_rt_sh(job_obj, pr_repo_loc, bldate, branch)
+        # update_rt_sh(job_obj, pr_repo_loc, bldate, branch)
+	logger.info('Starting RT Job')
+	rt.run(job_obj)
+        logger.info('Finished with RT Job')
         remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir)
 
 
-def update_rt_sh(job_obj, pr_repo_loc, bldate, branch):
+def get_bl_date(job_obj, pr_repo_loc):
     logger = logging.getLogger('BL/UPDATE_RT_SH')
     BLDATEFOUND = False
     with open(f'{pr_repo_loc}/tests/rt.sh', 'r') as f:
-        with open(f'{pr_repo_loc}/tests/rt.sh.new', 'w') as w:
-            for line in f:
-                if 'BL_DATE=' in line:
-                    logger.info('Found BL_DATE in line')
-                    logger.info(f'Writing "BL_DATE-{bldate}" into file')
-                    w.write(f'BL_DATE={bldate}\n')
-                    BLDATEFOUND = True
-                else:
-                    w.write(line)
+        for line in f:
+            if 'BL_DATE=' in line:
+                logger.info('Found BL_DATE in line')
+                BLDATEFOUND = True
+		bldate = line
+            	bldate = bldate.replace('BL_DATE=', '')
+            	bldate = bldate.replace(' ', '')
+            	bl_format = '%Y%m%d'
+                try:
+                    datetime.datetime.strptime(bldate, bl_format)
+                except ValueError:
+                    logger.info(f'Date {bldate} is not formatted YYYYMMDD')
+                    raise ValueError
     if not BLDATEFOUND:
         job_obj.comment_text_append('BL_DATE not found in rt.sh.'
                                     'Please manually edit rt.sh '
                                     'with BL_DATE={bldate}')
-    logger.info('Finished update_rt_sh')
-
-    move_rtsh_commands = [
-        [f'git pull --ff-only origin {branch}', pr_repo_loc],
-        [f'mv {pr_repo_loc}/tests/rt.sh.new {pr_repo_loc}/tests/rt.sh',
-         pr_repo_loc],
-
-        [f'git add {pr_repo_loc}/tests/rt.sh', pr_repo_loc],
-        [f'git commit --allow-empty -m "BL JOBS PASSED: {job_obj.machine}'
-         f'.{job_obj.compiler}. Updated rt.sh with new develop date: '
-         f'{bldate}"',
-         pr_repo_loc],
-        ['sleep 10', pr_repo_loc],
-        [f'git push origin {branch}', pr_repo_loc]
-        ]
+        job_obj.job_failed(logger, 'get_bl_date()')
+    logger.info('Finished get_bl_date')
     job_obj.run_commands(logger, move_rtsh_commands)
 
+    return bldate	
+
 
 def process_logfile(job_obj, logfile):
     logger = logging.getLogger('BL/PROCESS_LOGFILE')

From 7c6330a2016c3aafa73905b39300a22312336401 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Tue, 30 Mar 2021 21:16:33 +0000
Subject: [PATCH 20/38] Space issues from bad editor

---
 tests/auto/jobs/bl.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 82ed983e83..be24d1cc6b 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -156,13 +156,13 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch):
     rt_dir, logfile_pass = process_logfile(job_obj, filepath)
     if logfile_pass:
         bldate = get_bl_date(job_obj, pr_repo_loc)
-    	bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
+        bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
         create_bl_dir(bldir)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
         job_obj.run_commands(logger, move_bl_command)
         # update_rt_sh(job_obj, pr_repo_loc, bldate, branch)
-	logger.info('Starting RT Job')
-	rt.run(job_obj)
+        logger.info('Starting RT Job')
+        rt.run(job_obj)
         logger.info('Finished with RT Job')
         remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir)
 
@@ -175,10 +175,10 @@ def get_bl_date(job_obj, pr_repo_loc):
             if 'BL_DATE=' in line:
                 logger.info('Found BL_DATE in line')
                 BLDATEFOUND = True
-		bldate = line
-            	bldate = bldate.replace('BL_DATE=', '')
-            	bldate = bldate.replace(' ', '')
-            	bl_format = '%Y%m%d'
+                bldate = line
+                bldate = bldate.replace('BL_DATE=', '')
+                bldate = bldate.replace(' ', '')
+                bl_format = '%Y%m%d'
                 try:
                     datetime.datetime.strptime(bldate, bl_format)
                 except ValueError:
@@ -192,7 +192,7 @@ def get_bl_date(job_obj, pr_repo_loc):
     logger.info('Finished get_bl_date')
     job_obj.run_commands(logger, move_rtsh_commands)
 
-    return bldate	
+    return bldate
 
 
 def process_logfile(job_obj, logfile):

From 4e68e6f8715a03768c4c1dced7b6c74c7dfbd7da Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Wed, 31 Mar 2021 12:50:50 +0000
Subject: [PATCH 21/38] Some fixes

---
 tests/auto/rt_auto.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py
index de642d0afa..88e063ae4e 100644
--- a/tests/auto/rt_auto.py
+++ b/tests/auto/rt_auto.py
@@ -202,7 +202,6 @@ def send_comment_text(self):
 
     def job_failed(self, logger, job_name, exception=Exception, STDOUT=False,
                    out=None, err=None):
-        self.comment_text_append(f'{job_name} FAILED. Exception:{exception}')
         logger.critical(f'{job_name} FAILED. Exception:{exception}')
 
         if STDOUT:

From db102d01f1f35544bee69f03624c6d2933d34e42 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Wed, 31 Mar 2021 20:25:06 +0000
Subject: [PATCH 22/38] Fixes from Gaea tests

---
 tests/auto/jobs/bl.py | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index be24d1cc6b..2915953197 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -7,13 +7,12 @@
 
 def run(job_obj):
     logger = logging.getLogger('BL/RUN')
-    #bldate = get_bl_date(job_obj)
     workdir, rtbldir, blstore = set_directories(job_obj)
-    #bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
-    #if not check_for_bl_dir(bldir):
-    branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
+    pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
+    bldate = get_bl_date(job_obj, pr_repo_loc)
+    bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
     run_regression_test(job_obj, pr_repo_loc)
-    post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch)
+    post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir)
 
 
 def set_directories(job_obj):
@@ -30,7 +29,7 @@ def set_directories(job_obj):
                  f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'gaea':
         workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr'
-        blstore = '/lustre/f2/pdata/ncep_shared/emc.nemspara/RT'
+        blstore = '/lustre/f2/pdata/ncep_shared/emc.nemspara/RT/NEMSfv3gfs'
         rtbldir = '/lustre/f2/scratch/emc.nemspara/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'orion':
@@ -145,22 +144,19 @@ def clone_pr_repo(job_obj, workdir):
     job_obj.run_commands(logger, create_repo_commands)
 
     logger.info('Finished repo clone')
-    return branch, pr_repo_loc, repo_dir_str
+    return pr_repo_loc, repo_dir_str
 
 
-def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, blstore, branch):
+def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir):
     logger = logging.getLogger('BL/MOVE_RT_LOGS')
     rt_log = f'tests/RegressionTests_{job_obj.machine}'\
              f'.{job_obj.compiler}.log'
     filepath = f'{pr_repo_loc}/{rt_log}'
     rt_dir, logfile_pass = process_logfile(job_obj, filepath)
     if logfile_pass:
-        bldate = get_bl_date(job_obj, pr_repo_loc)
-        bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
         create_bl_dir(bldir)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
         job_obj.run_commands(logger, move_bl_command)
-        # update_rt_sh(job_obj, pr_repo_loc, bldate, branch)
         logger.info('Starting RT Job')
         rt.run(job_obj)
         logger.info('Finished with RT Job')
@@ -176,11 +172,14 @@ def get_bl_date(job_obj, pr_repo_loc):
                 logger.info('Found BL_DATE in line')
                 BLDATEFOUND = True
                 bldate = line
+                bldate = bldate.rstrip('\n')
                 bldate = bldate.replace('BL_DATE=', '')
-                bldate = bldate.replace(' ', '')
+                bldate = bldate.strip(' ')
+                logger.info(f'bldate is "{bldate}"')
+                logger.info(f'Type bldate: {type(bldate)}')
                 bl_format = '%Y%m%d'
                 try:
-                    datetime.datetime.strptime(bldate, bl_format)
+                    datetime.datetime.strptime(bldate, '%Y%m%d')
                 except ValueError:
                     logger.info(f'Date {bldate} is not formatted YYYYMMDD')
                     raise ValueError
@@ -190,7 +189,6 @@ def get_bl_date(job_obj, pr_repo_loc):
                                     'with BL_DATE={bldate}')
         job_obj.job_failed(logger, 'get_bl_date()')
     logger.info('Finished get_bl_date')
-    job_obj.run_commands(logger, move_rtsh_commands)
 
     return bldate
 

From ff19c2617e2f21df9168dddc14d501502c198c17 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Wed, 7 Apr 2021 13:33:09 +0000
Subject: [PATCH 23/38] Update to h-nems accouunt and storage locations on Jet

---
 tests/auto/jobs/bl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 2915953197..a536347052 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -24,8 +24,8 @@ def set_directories(job_obj):
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'jet':
         workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr'
-        blstore = '/lfs4/HFIP/hfv3gfs/RT/NEMSfv3gfs/'
-        rtbldir = '/lfs4/HFIP/hfv3gfs/emc.nemspara/RT_BASELINE/'\
+        blstore = '/lfs4/HFIP/h-nems/emc.nemspara/RT/NEMSfv3gfs/'
+        rtbldir = '/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\
                  f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'gaea':
         workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr'

From 67eb9f369b8900cb73b288a8d29b2d81d49c7c92 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Thu, 8 Apr 2021 16:17:04 +0000
Subject: [PATCH 24/38] FAIL is not shown where baselines are there but jobs
 does not complete, thus changing search string to 'Test' and 'failed'

---
 tests/auto/jobs/bl.py | 4 +++-
 tests/auto/jobs/rt.py | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index a536347052..7d7e74237a 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -196,10 +196,12 @@ def get_bl_date(job_obj, pr_repo_loc):
 def process_logfile(job_obj, logfile):
     logger = logging.getLogger('BL/PROCESS_LOGFILE')
     rt_dir = []
+    fail_string_list = ['Test', 'failed']
     if os.path.exists(logfile):
         with open(logfile) as f:
             for line in f:
-                if 'FAIL' in line and 'Test' in line:
+                if all(x in line for x in fail_string_list):
+                # if 'FAIL' in line and 'Test' in line:
                     job_obj.comment_text_append(f'{line.rstrip(chr(10))}')
                 elif 'working dir' in line and not rt_dir:
                     logger.info(f'Found "working dir" in line: {line}')
diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
index 568bba39ee..59c7813790 100644
--- a/tests/auto/jobs/rt.py
+++ b/tests/auto/jobs/rt.py
@@ -110,10 +110,12 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch):
 def process_logfile(job_obj, logfile):
     logger = logging.getLogger('RT/PROCESS_LOGFILE')
     rt_dir = []
+    fail_string_list = ['Test', 'failed']
     if os.path.exists(logfile):
         with open(logfile) as f:
             for line in f:
-                if 'FAIL' in line and 'Test' in line:
+                if all(x in line for x in fail_string_list):
+                # if 'FAIL' in line and 'Test' in line:
                     job_obj.comment_text_append(f'{line.rstrip(chr(10))}')
                 elif 'working dir' in line and not rt_dir:
                     rt_dir = os.path.split(line.split()[-1])[0]

From e5a1b213bc25ac20a8d3019d85facd129816874d Mon Sep 17 00:00:00 2001
From: Brian Curtis <64433609+BrianCurtis-NOAA@users.noreply.github.com>
Date: Fri, 9 Apr 2021 16:17:51 -0400
Subject: [PATCH 25/38] test addition

on-behalf-of @ufs-community <brian.curtis@noaa.gov>
---
 tests/auto/jobs/rt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
index 59c7813790..85e7271279 100644
--- a/tests/auto/jobs/rt.py
+++ b/tests/auto/jobs/rt.py
@@ -115,7 +115,7 @@ def process_logfile(job_obj, logfile):
         with open(logfile) as f:
             for line in f:
                 if all(x in line for x in fail_string_list):
-                # if 'FAIL' in line and 'Test' in line:
+                # if 'FAIL' in line and 'Test' in line: # Test Addition
                     job_obj.comment_text_append(f'{line.rstrip(chr(10))}')
                 elif 'working dir' in line and not rt_dir:
                     rt_dir = os.path.split(line.split()[-1])[0]

From a6008f82ac02d6e73bd964378f14ade307483bc1 Mon Sep 17 00:00:00 2001
From: Brian Curtis <64433609+BrianCurtis-NOAA@users.noreply.github.com>
Date: Fri, 9 Apr 2021 16:30:22 -0400
Subject: [PATCH 26/38] test subtraction

on-behalf-of @ufs-community
---
 tests/auto/jobs/rt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
index 85e7271279..59c7813790 100644
--- a/tests/auto/jobs/rt.py
+++ b/tests/auto/jobs/rt.py
@@ -115,7 +115,7 @@ def process_logfile(job_obj, logfile):
         with open(logfile) as f:
             for line in f:
                 if all(x in line for x in fail_string_list):
-                # if 'FAIL' in line and 'Test' in line: # Test Addition
+                # if 'FAIL' in line and 'Test' in line:
                     job_obj.comment_text_append(f'{line.rstrip(chr(10))}')
                 elif 'working dir' in line and not rt_dir:
                     rt_dir = os.path.split(line.split()[-1])[0]

From ddea93c2c5977802bae07935e8b20f926e600bd2 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 19 Apr 2021 09:25:23 -0600
Subject: [PATCH 27/38] Cheyenne Fixes

---
 tests/auto/jobs/bl.py       | 4 ++--
 tests/auto/jobs/rt.py       | 2 +-
 tests/auto/rt_auto.py       | 2 +-
 tests/auto/start_rt_auto.sh | 3 ++-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 7d7e74237a..dd6a6603c1 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -38,9 +38,9 @@ def set_directories(job_obj):
         rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'cheyenne':
-        workdir = '/glade/work/heinzell/fv3/ufs-weather-model/auto-rt'
+        workdir = '/glade/work/briancurtis/git/BrianCurtis-NOAA/ufs-weather-model/tests/auto/pr'
         blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT'
-        rtbldir = '/glade/work/heinzell/FV3_RT/'\
+        rtbldir = '/glade/scratch/briancurtis/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     else:
         logger.critical(f'Machine {job_obj.machine} is not supported for this job')
diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
index 59c7813790..01d406f679 100644
--- a/tests/auto/jobs/rt.py
+++ b/tests/auto/jobs/rt.py
@@ -23,7 +23,7 @@ def set_directories(job_obj):
     elif job_obj.machine == 'orion':
         workdir = '/work/noaa/nems/emc.nemspara/autort/pr'
     elif job_obj.machine == 'cheyenne':
-        workdir = '/glade/work/heinzell/fv3/ufs-weather-model/auto-rt'
+        workdir = '/glade/work/briancurtis/git/BrianCurtis-NOAA/ufs-weather-model/tests/auto/pr'
     else:
         print(f'Machine {job_obj.machine} is not supported for this job')
         raise KeyError
diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py
index 88e063ae4e..c003a40d7c 100644
--- a/tests/auto/rt_auto.py
+++ b/tests/auto/rt_auto.py
@@ -220,7 +220,7 @@ def setup_env():
         os.environ['ACCNR'] = 'nggps_emc'
     elif bool(re.match(re.compile('Orion-login.+'), hostname)):
         machine = 'orion'
-    elif bool(re.match(re.compile('.+.cheyenne.ucar.edu'), hostname)):
+    elif bool(re.match(re.compile('chadmin.+'), hostname)):
         machine = 'cheyenne'
         os.environ['ACCNR'] = 'P48503002'
     else:
diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh
index 74ad09f50a..0aae3cafba 100644
--- a/tests/auto/start_rt_auto.sh
+++ b/tests/auto/start_rt_auto.sh
@@ -13,7 +13,8 @@ elif [[ $HOSTNAME == fe* ]]; then
 elif [[ $HOSTNAME == gaea* ]]; then
   export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:$PATH
   export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages
-elif [[ $HOSTNAME == *cheyenne* ]]; then
+elif [[ $HOSTNAME == *chadmin* ]]; then
+  export MACHINE_ID=cheyenne
   export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH
   export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages
 else

From 1e591a6070c29194d4480254fe62b4beb0aee76a Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Tue, 20 Apr 2021 19:10:20 +0000
Subject: [PATCH 28/38] Need to fail nicely when PR owners block commits by
 maintainers

---
 tests/auto/jobs/rt.py | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
index 568bba39ee..a8a183ca9c 100644
--- a/tests/auto/jobs/rt.py
+++ b/tests/auto/jobs/rt.py
@@ -94,17 +94,25 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch):
     filepath = f'{pr_repo_loc}/{rt_log}'
     rt_dir, logfile_pass = process_logfile(job_obj, filepath)
     if logfile_pass:
-        move_rt_commands = [
-            [f'git pull --ff-only origin {branch}', pr_repo_loc],
-            [f'git add {rt_log}', pr_repo_loc],
-            [f'git commit -m "RT JOBS PASSED: {job_obj.machine}'
-             f'.{job_obj.compiler}. Log file uploaded."',
-             pr_repo_loc],
-            ['sleep 10', pr_repo_loc],
-            [f'git push origin {branch}', pr_repo_loc]
-        ]
-        job_obj.run_commands(logger, move_rt_commands)
-        remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir)
+        if job_obj.preq_dict['preq'].maintainer_can_modify:
+            move_rt_commands = [
+                [f'git pull --ff-only origin {branch}', pr_repo_loc],
+                [f'git add {rt_log}', pr_repo_loc],
+                [f'git commit -m "RT JOBS PASSED: {job_obj.machine}'
+                 f'.{job_obj.compiler}. Log file uploaded.\n\n'
+                  'on-behalf-of @ufs-community"',
+                 pr_repo_loc],
+                ['sleep 10', pr_repo_loc],
+                [f'git push origin {branch}', pr_repo_loc]
+            ]
+            job_obj.run_commands(logger, move_rt_commands)
+            remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir)
+        else:
+            job_obj.comment_text_append(f'Cannot upload {job_obj.machine}.'\
+                                        f'{job_obj.compiler} RT Log'\
+                                        'It is blocked by PR owner')
+            job_obj.comment_text_append(f'Please obtain logs from {pr_repo_loc}')
+            job_obj.preq_dict['preq'].create_issue_comment(job_obj.comment_text)
 
 
 def process_logfile(job_obj, logfile):

From d1e13a94b60259e324215d431ff740be98b11bbc Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Mon, 26 Apr 2021 07:27:05 -0600
Subject: [PATCH 29/38] Typo in RTBLDIR

---
 tests/auto/jobs/bl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index dd6a6603c1..e53208b634 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -39,8 +39,8 @@ def set_directories(job_obj):
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'cheyenne':
         workdir = '/glade/work/briancurtis/git/BrianCurtis-NOAA/ufs-weather-model/tests/auto/pr'
-        blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT'
-        rtbldir = '/glade/scratch/briancurtis/FV3_RT/'\
+        blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT/NEMSfv3gfs'
+        rtbldir = '/glade/work/briancurtis/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     else:
         logger.critical(f'Machine {job_obj.machine} is not supported for this job')

From 0e9a294b56e7db6332f4fb664c993cd82b033eb0 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Thu, 27 May 2021 09:39:31 -0600
Subject: [PATCH 30/38] New autort location on cheyenne. lets note when
 baselines are completed in the PR comment if something fails later on

---
 tests/auto/jobs/bl.py | 5 +++--
 tests/auto/jobs/rt.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index e53208b634..fdafd9f999 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -38,9 +38,9 @@ def set_directories(job_obj):
         rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     elif job_obj.machine == 'cheyenne':
-        workdir = '/glade/work/briancurtis/git/BrianCurtis-NOAA/ufs-weather-model/tests/auto/pr'
+        workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr'
         blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT/NEMSfv3gfs'
-        rtbldir = '/glade/work/briancurtis/FV3_RT/'\
+        rtbldir = '/glade/work/dtcufsrt/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     else:
         logger.critical(f'Machine {job_obj.machine} is not supported for this job')
@@ -157,6 +157,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir):
         create_bl_dir(bldir)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
         job_obj.run_commands(logger, move_bl_command)
+	job_obj.comment_text_append(f'Baseline creation and move successful')
         logger.info('Starting RT Job')
         rt.run(job_obj)
         logger.info('Finished with RT Job')
diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py
index 5c1389073a..c69bcda669 100644
--- a/tests/auto/jobs/rt.py
+++ b/tests/auto/jobs/rt.py
@@ -23,7 +23,7 @@ def set_directories(job_obj):
     elif job_obj.machine == 'orion':
         workdir = '/work/noaa/nems/emc.nemspara/autort/pr'
     elif job_obj.machine == 'cheyenne':
-        workdir = '/glade/work/briancurtis/git/BrianCurtis-NOAA/ufs-weather-model/tests/auto/pr'
+        workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr'
     else:
         print(f'Machine {job_obj.machine} is not supported for this job')
         raise KeyError

From 8e85cfc2d351110b367e3869ee473ff1b6a9d239 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Thu, 27 May 2021 09:45:49 -0600
Subject: [PATCH 31/38] /glade/scratch/ not /glade/work

---
 tests/auto/jobs/bl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index fdafd9f999..ba28c53c1d 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -40,7 +40,7 @@ def set_directories(job_obj):
     elif job_obj.machine == 'cheyenne':
         workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr'
         blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT/NEMSfv3gfs'
-        rtbldir = '/glade/work/dtcufsrt/FV3_RT/'\
+        rtbldir = '/glade/scratch/dtcufsrt/FV3_RT/'\
                  f'REGRESSION_TEST_{job_obj.compiler.upper()}'
     else:
         logger.critical(f'Machine {job_obj.machine} is not supported for this job')

From 7f343ffbdb6dbf29eda8bc188a8401faf67db66a Mon Sep 17 00:00:00 2001
From: Brian Curtis <64433609+BrianCurtis-NOAA@users.noreply.github.com>
Date: Wed, 2 Jun 2021 09:44:48 -0400
Subject: [PATCH 32/38] remove f-string where no variables are

remove f-string in comment text append
---
 tests/auto/jobs/bl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index ba28c53c1d..0f0022cd66 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -157,7 +157,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir):
         create_bl_dir(bldir)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
         job_obj.run_commands(logger, move_bl_command)
-	job_obj.comment_text_append(f'Baseline creation and move successful')
+	job_obj.comment_text_append('Baseline creation and move successful')
         logger.info('Starting RT Job')
         rt.run(job_obj)
         logger.info('Finished with RT Job')

From c01b725b7af555fdd7e0d0ac626c287df14d7a33 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Thu, 3 Jun 2021 17:05:02 +0000
Subject: [PATCH 33/38] line had tab indentation instead of space

---
 tests/auto/jobs/bl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 0f0022cd66..8c39aefb8c 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -157,7 +157,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir):
         create_bl_dir(bldir)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
         job_obj.run_commands(logger, move_bl_command)
-	job_obj.comment_text_append('Baseline creation and move successful')
+        job_obj.comment_text_append('Baseline creation and move successful')
         logger.info('Starting RT Job')
         rt.run(job_obj)
         logger.info('Finished with RT Job')

From 59ca5da2fe7328388e55c84f83828166936a094c Mon Sep 17 00:00:00 2001
From: Brian Curtis <Brian.Curtis@noaa.gov>
Date: Wed, 11 Aug 2021 09:54:05 -0400
Subject: [PATCH 34/38] Lets check to see if the baseline dir exists before
 running BL job

---
 tests/auto/jobs/bl.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 8c39aefb8c..8c433b7e35 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -11,6 +11,7 @@ def run(job_obj):
     pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
     bldate = get_bl_date(job_obj, pr_repo_loc)
     bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
+    bldirbool = check_for_bl_dir(bldir)
     run_regression_test(job_obj, pr_repo_loc)
     post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir)
 

From f2533ae6886db3d8e07d3a9df70daba2ded99ac1 Mon Sep 17 00:00:00 2001
From: Brian Curtis <Brian.Curtis@noaa.gov>
Date: Wed, 11 Aug 2021 10:07:17 -0400
Subject: [PATCH 35/38] Add a comment to the PR text on BLDIR existing

---
 tests/auto/jobs/bl.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 8c433b7e35..fcf8d7e85c 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -60,6 +60,8 @@ def check_for_bl_dir(bldir):
     logger.info('Checking if baseline directory exists')
     if os.path.exists(bldir):
         logger.critical(f'Baseline dir: {bldir} exists. It should not, yet.')
+        job_obj.comment_text_append(f'{bldir}\n Exists already. '
+                                    'It should not yet. Please delete.')
         raise FileExistsError
     return False
 

From 0cf6bf3cfda44c9756715ac18d82a4fdaa2aa9c0 Mon Sep 17 00:00:00 2001
From: Brian Curtis <Brian.Curtis@noaa.gov>
Date: Wed, 11 Aug 2021 10:11:12 -0400
Subject: [PATCH 36/38] job_obj needs to be added to call variables

---
 tests/auto/jobs/bl.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index fcf8d7e85c..14d62c1d1c 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -11,7 +11,7 @@ def run(job_obj):
     pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir)
     bldate = get_bl_date(job_obj, pr_repo_loc)
     bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}'
-    bldirbool = check_for_bl_dir(bldir)
+    bldirbool = check_for_bl_dir(bldir, job_obj)
     run_regression_test(job_obj, pr_repo_loc)
     post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir)
 
@@ -55,7 +55,7 @@ def set_directories(job_obj):
     return workdir, rtbldir, blstore
 
 
-def check_for_bl_dir(bldir):
+def check_for_bl_dir(bldir, job_obj):
     logger = logging.getLogger('BL/CHECK_FOR_BL_DIR')
     logger.info('Checking if baseline directory exists')
     if os.path.exists(bldir):
@@ -66,9 +66,9 @@ def check_for_bl_dir(bldir):
     return False
 
 
-def create_bl_dir(bldir):
+def create_bl_dir(bldir, job_obj):
     logger = logging.getLogger('BL/CREATE_BL_DIR')
-    if not check_for_bl_dir(bldir):
+    if not check_for_bl_dir(bldir, job_obj):
         os.makedirs(bldir)
         if not os.path.exists(bldir):
             logger.critical(f'Someting went wrong creating {bldir}')
@@ -157,7 +157,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir):
     filepath = f'{pr_repo_loc}/{rt_log}'
     rt_dir, logfile_pass = process_logfile(job_obj, filepath)
     if logfile_pass:
-        create_bl_dir(bldir)
+        create_bl_dir(bldir, job_obj)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
         job_obj.run_commands(logger, move_bl_command)
         job_obj.comment_text_append('Baseline creation and move successful')

From ad5b017f0e5958c338c288f89345ebc6487b61a5 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Wed, 11 Aug 2021 09:24:23 -0500
Subject: [PATCH 37/38] Special case for orion to use adjust_permissions.sh

---
 tests/auto/jobs/bl.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py
index 8c39aefb8c..869f76a026 100644
--- a/tests/auto/jobs/bl.py
+++ b/tests/auto/jobs/bl.py
@@ -156,6 +156,8 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir):
     if logfile_pass:
         create_bl_dir(bldir)
         move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]]
+        if job_obj.machine == 'orion':
+            move_bl_command.append([f'/bin/bash --login adjust_permissions.sh orion develop-{bldate}', blstore])
         job_obj.run_commands(logger, move_bl_command)
         job_obj.comment_text_append('Baseline creation and move successful')
         logger.info('Starting RT Job')

From 0d6539fa720df1deabe9dcbb13f18c361a6104e2 Mon Sep 17 00:00:00 2001
From: Brian Curtis <brian.curtis@noaa.gov>
Date: Wed, 11 Aug 2021 18:36:33 +0000
Subject: [PATCH 38/38] Hera ECFLOW node adjustments

---
 tests/auto/rt_auto.py       | 2 ++
 tests/auto/start_rt_auto.sh | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py
index c003a40d7c..bed530b3d0 100644
--- a/tests/auto/rt_auto.py
+++ b/tests/auto/rt_auto.py
@@ -212,6 +212,8 @@ def setup_env():
     hostname = os.getenv('HOSTNAME')
     if bool(re.match(re.compile('hfe.+'), hostname)):
         machine = 'hera'
+    elif bool(re.match(re.compile('hecflow.+'), hostname)):
+        machine = 'hera'
     elif bool(re.match(re.compile('fe.+'), hostname)):
         machine = 'jet'
         os.environ['ACCNR'] = 'h-nems'
diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh
index 0aae3cafba..af85cdccbb 100644
--- a/tests/auto/start_rt_auto.sh
+++ b/tests/auto/start_rt_auto.sh
@@ -4,6 +4,9 @@ set -eux
 if [[ $HOSTNAME == hfe* ]]; then
   export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH
   export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages
+elif [[ $HOSTNAME == hecflow* ]]; then
+  export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH
+  export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages
 elif [[ $HOSTNAME == Orion-login-* ]]; then
   export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH
   export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages