diff --git a/tests/auto/jobs/bl.py b/tests/auto/jobs/bl.py index 2915953197..1b758211a4 100644 --- a/tests/auto/jobs/bl.py +++ b/tests/auto/jobs/bl.py @@ -11,6 +11,7 @@ def run(job_obj): pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir) bldate = get_bl_date(job_obj, pr_repo_loc) bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}' + bldirbool = check_for_bl_dir(bldir, job_obj) run_regression_test(job_obj, pr_repo_loc) post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir) @@ -24,8 +25,8 @@ def set_directories(job_obj): f'REGRESSION_TEST_{job_obj.compiler.upper()}' elif job_obj.machine == 'jet': workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' - blstore = '/lfs4/HFIP/hfv3gfs/RT/NEMSfv3gfs/' - rtbldir = '/lfs4/HFIP/hfv3gfs/emc.nemspara/RT_BASELINE/'\ + blstore = '/lfs4/HFIP/h-nems/emc.nemspara/RT/NEMSfv3gfs/' + rtbldir = '/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\ f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}' elif job_obj.machine == 'gaea': workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr' @@ -38,9 +39,9 @@ def set_directories(job_obj): rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' elif job_obj.machine == 'cheyenne': - workdir = '/glade/work/heinzell/fv3/ufs-weather-model/auto-rt' - blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT' - rtbldir = '/glade/work/heinzell/FV3_RT/'\ + workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr' + blstore = '/glade/p/ral/jntp/GMTB/ufs-weather-model/RT/NEMSfv3gfs' + rtbldir = '/glade/scratch/dtcufsrt/FV3_RT/'\ f'REGRESSION_TEST_{job_obj.compiler.upper()}' else: logger.critical(f'Machine {job_obj.machine} is not supported for this job') @@ -54,18 +55,20 @@ def set_directories(job_obj): return workdir, rtbldir, blstore -def check_for_bl_dir(bldir): +def check_for_bl_dir(bldir, job_obj): logger = logging.getLogger('BL/CHECK_FOR_BL_DIR') logger.info('Checking if baseline directory exists') if os.path.exists(bldir): logger.critical(f'Baseline dir: {bldir} exists. It should not, yet.') + job_obj.comment_text_append(f'{bldir}\n Exists already. ' + 'It should not yet. Please delete.') raise FileExistsError return False -def create_bl_dir(bldir): +def create_bl_dir(bldir, job_obj): logger = logging.getLogger('BL/CREATE_BL_DIR') - if not check_for_bl_dir(bldir): + if not check_for_bl_dir(bldir, job_obj): os.makedirs(bldir) if not os.path.exists(bldir): logger.critical(f'Someting went wrong creating {bldir}') @@ -154,9 +157,12 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir): filepath = f'{pr_repo_loc}/{rt_log}' rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: - create_bl_dir(bldir) + create_bl_dir(bldir, job_obj) move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]] + if job_obj.machine == 'orion': + move_bl_command.append([f'/bin/bash --login adjust_permissions.sh orion develop-{bldate}', blstore]) job_obj.run_commands(logger, move_bl_command) + job_obj.comment_text_append('Baseline creation and move successful') logger.info('Starting RT Job') rt.run(job_obj) logger.info('Finished with RT Job') @@ -196,10 +202,12 @@ def get_bl_date(job_obj, pr_repo_loc): def process_logfile(job_obj, logfile): logger = logging.getLogger('BL/PROCESS_LOGFILE') rt_dir = [] + fail_string_list = ['Test', 'failed'] if os.path.exists(logfile): with open(logfile) as f: for line in f: - if 'FAIL' in line and 'Test' in line: + if all(x in line for x in fail_string_list): + # if 'FAIL' in line and 'Test' in line: job_obj.comment_text_append(f'{line.rstrip(chr(10))}') elif 'working dir' in line and not rt_dir: logger.info(f'Found "working dir" in line: {line}') diff --git a/tests/auto/jobs/rt.py b/tests/auto/jobs/rt.py index 568bba39ee..c69bcda669 100644 --- a/tests/auto/jobs/rt.py +++ b/tests/auto/jobs/rt.py @@ -23,7 +23,7 @@ def set_directories(job_obj): elif job_obj.machine == 'orion': workdir = '/work/noaa/nems/emc.nemspara/autort/pr' elif job_obj.machine == 'cheyenne': - workdir = '/glade/work/heinzell/fv3/ufs-weather-model/auto-rt' + workdir = '/glade/scratch/dtcufsrt/autort/tests/auto/pr' else: print(f'Machine {job_obj.machine} is not supported for this job') raise KeyError @@ -94,26 +94,36 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): filepath = f'{pr_repo_loc}/{rt_log}' rt_dir, logfile_pass = process_logfile(job_obj, filepath) if logfile_pass: - move_rt_commands = [ - [f'git pull --ff-only origin {branch}', pr_repo_loc], - [f'git add {rt_log}', pr_repo_loc], - [f'git commit -m "RT JOBS PASSED: {job_obj.machine}' - f'.{job_obj.compiler}. Log file uploaded."', - pr_repo_loc], - ['sleep 10', pr_repo_loc], - [f'git push origin {branch}', pr_repo_loc] - ] - job_obj.run_commands(logger, move_rt_commands) - remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir) + if job_obj.preq_dict['preq'].maintainer_can_modify: + move_rt_commands = [ + [f'git pull --ff-only origin {branch}', pr_repo_loc], + [f'git add {rt_log}', pr_repo_loc], + [f'git commit -m "RT JOBS PASSED: {job_obj.machine}' + f'.{job_obj.compiler}. Log file uploaded.\n\n' + 'on-behalf-of @ufs-community"', + pr_repo_loc], + ['sleep 10', pr_repo_loc], + [f'git push origin {branch}', pr_repo_loc] + ] + job_obj.run_commands(logger, move_rt_commands) + remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir) + else: + job_obj.comment_text_append(f'Cannot upload {job_obj.machine}.'\ + f'{job_obj.compiler} RT Log'\ + 'It is blocked by PR owner') + job_obj.comment_text_append(f'Please obtain logs from {pr_repo_loc}') + job_obj.preq_dict['preq'].create_issue_comment(job_obj.comment_text) def process_logfile(job_obj, logfile): logger = logging.getLogger('RT/PROCESS_LOGFILE') rt_dir = [] + fail_string_list = ['Test', 'failed'] if os.path.exists(logfile): with open(logfile) as f: for line in f: - if 'FAIL' in line and 'Test' in line: + if all(x in line for x in fail_string_list): + # if 'FAIL' in line and 'Test' in line: job_obj.comment_text_append(f'{line.rstrip(chr(10))}') elif 'working dir' in line and not rt_dir: rt_dir = os.path.split(line.split()[-1])[0] diff --git a/tests/auto/rt_auto.py b/tests/auto/rt_auto.py index 88e063ae4e..bed530b3d0 100644 --- a/tests/auto/rt_auto.py +++ b/tests/auto/rt_auto.py @@ -212,6 +212,8 @@ def setup_env(): hostname = os.getenv('HOSTNAME') if bool(re.match(re.compile('hfe.+'), hostname)): machine = 'hera' + elif bool(re.match(re.compile('hecflow.+'), hostname)): + machine = 'hera' elif bool(re.match(re.compile('fe.+'), hostname)): machine = 'jet' os.environ['ACCNR'] = 'h-nems' @@ -220,7 +222,7 @@ def setup_env(): os.environ['ACCNR'] = 'nggps_emc' elif bool(re.match(re.compile('Orion-login.+'), hostname)): machine = 'orion' - elif bool(re.match(re.compile('.+.cheyenne.ucar.edu'), hostname)): + elif bool(re.match(re.compile('chadmin.+'), hostname)): machine = 'cheyenne' os.environ['ACCNR'] = 'P48503002' else: diff --git a/tests/auto/start_rt_auto.sh b/tests/auto/start_rt_auto.sh index 74ad09f50a..af85cdccbb 100644 --- a/tests/auto/start_rt_auto.sh +++ b/tests/auto/start_rt_auto.sh @@ -4,6 +4,9 @@ set -eux if [[ $HOSTNAME == hfe* ]]; then export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages +elif [[ $HOSTNAME == hecflow* ]]; then + export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH + export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages elif [[ $HOSTNAME == Orion-login-* ]]; then export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages @@ -13,7 +16,8 @@ elif [[ $HOSTNAME == fe* ]]; then elif [[ $HOSTNAME == gaea* ]]; then export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:$PATH export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages -elif [[ $HOSTNAME == *cheyenne* ]]; then +elif [[ $HOSTNAME == *chadmin* ]]; then + export MACHINE_ID=cheyenne export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages else