From 354666d974ca5c7bbd5ff5b752e09f588c638467 Mon Sep 17 00:00:00 2001 From: ClimaBot Date: Mon, 23 Jan 2023 14:38:04 -0700 Subject: [PATCH 1/4] bl-test, updating baseline test. --- tests/auto-jenkins/jobs/rt.py | 161 +++++++++ tests/auto-jenkins/rt_auto_jenkins.py | 365 ++++++++++++++++++++ tests/auto-jenkins/start_rt_auto_jenkins.sh | 33 ++ 3 files changed, 559 insertions(+) create mode 100644 tests/auto-jenkins/jobs/rt.py create mode 100755 tests/auto-jenkins/rt_auto_jenkins.py create mode 100755 tests/auto-jenkins/start_rt_auto_jenkins.sh diff --git a/tests/auto-jenkins/jobs/rt.py b/tests/auto-jenkins/jobs/rt.py new file mode 100644 index 0000000000..568b390fda --- /dev/null +++ b/tests/auto-jenkins/jobs/rt.py @@ -0,0 +1,161 @@ +# Imports +import datetime +import logging +import os +import re + +#Logging filter to santize log output and ensure Github Access tokens are not leaked. +def loggingfilter(record): + if re.search('ghp_(.*)\@', str(record.msg)): + record.msg = re.sub(r'ghp_(.*)\@', r'ghp_****@', str(record.msg)) + elif re.search('github_pat_(.*)\@', str(record.msg)): + record.msg = re.sub(r'github_pat_(.*)\@', r'github_pat_****@', str(record.msg)) + + return True + +def run(job_obj): + logger = logging.getLogger('RT/RUN') + workdir = set_directories(job_obj) + branch, pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir) + run_regression_test(job_obj, pr_repo_loc) + post_process(job_obj, pr_repo_loc, repo_dir_str, branch) + + +def set_directories(job_obj): + logger = logging.getLogger('RT/SET_DIRECTORIES') + if job_obj.machine == 'hera': + workdir = '/scratch1/NCEPDEV/nems/role.epic/autort/pr' + elif job_obj.machine == 'jet': + workdir = '/lfs4/HFIP/hfv3gfs/role.epic/autort/pr' + elif job_obj.machine == 'gaea': + workdir = '/lustre/f2/pdata/ncep/role.epic/autort/pr' + elif job_obj.machine == 'orion': + workdir = '/work/noaa/epic-ps/role-epic-ps/autort/tests/auto/pr' + elif job_obj.machine == 'cheyenne': + workdir = '/glade/scratch/epicufsrt/autort/jenkins/autort/pr' + else: + print(f'Machine {job_obj.machine} is not supported for this job') + raise KeyError + + logger.info(f'machine: {job_obj.machine}') + logger.info(f'workdir: {workdir}') + + return workdir + + +def run_regression_test(job_obj, pr_repo_loc): + logger = logging.getLogger('RT/RUN_REGRESSION_TEST') + if job_obj.compiler == 'gnu' and job_obj.machine != 'hera': + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' + '&& /bin/bash --login ./rt.sh -e -l rt_gnu.conf', + pr_repo_loc]] + elif job_obj.compiler == 'gnu' and job_obj.machine == 'hera': + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' + '&& /bin/bash --login ./rt.sh -r -l rt_gnu.conf', + pr_repo_loc]] + elif job_obj.compiler == 'intel' and job_obj.machine != 'hera': + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' + '&& /bin/bash --login ./rt.sh -e', pr_repo_loc]] + elif job_obj.compiler == 'intel' and job_obj.machine == 'hera': + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' + '&& /bin/bash --login ./rt.sh -r', pr_repo_loc]] + job_obj.run_commands(logger, rt_command) + + +def remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir): + logger = logging.getLogger('RT/REMOVE_PR_DATA') + rm_command = [ + [f'rm -rf {rt_dir}', pr_repo_loc], + [f'rm -rf {repo_dir_str}', pr_repo_loc] + ] + job_obj.run_commands(logger, rm_command) + + +def clone_pr_repo(job_obj, workdir): + ''' clone the GitHub pull request repo, via command line ''' + filename = ('jenkinsaccesstoken') + f = open(filename) + os.environ['ghapitoken'] = f.readline().strip('\n') + + logger = logging.getLogger('RT/CLONE_PR_REPO') + logger.addFilter(loggingfilter) + repo_name = job_obj.preq_dict['preq'].head.repo.name + branch = job_obj.preq_dict['preq'].head.ref + git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//') + git_token = os.environ['ghapitoken'] + git_https_url = f'{git_url[0]}//{git_token}@{git_url[1]}' + git_ssh_url = job_obj.preq_dict['preq'].head.repo.ssh_url + logger.debug(f'GIT SSH_URL: {git_ssh_url}') + logger.info('Starting repo clone') + repo_dir_str = f'{workdir}/'\ + f'{str(job_obj.preq_dict["preq"].id)}/'\ + f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}' + pr_repo_loc = f'{repo_dir_str}/{repo_name}' + job_obj.comment_text_append(f'[RT] Repo location: {pr_repo_loc}') + create_repo_commands = [ + [f'mkdir -p "{repo_dir_str}"', os.getcwd()], + [f'git clone -b {branch} {git_ssh_url}', repo_dir_str], + ['git submodule update --init --recursive', + f'{repo_dir_str}/{repo_name}'], + [f'git remote add httpsorigin {git_https_url}', + f'{repo_dir_str}/{repo_name}'], + ['git config user.email "ecc.platform@noaa.gov"', + f'{repo_dir_str}/{repo_name}'], + ['git config user.name "epic-cicd-jenkins"', + f'{repo_dir_str}/{repo_name}'] + ] + + job_obj.run_commands(logger, create_repo_commands) + + logger.info('Finished repo clone') + return branch, pr_repo_loc, repo_dir_str + + +def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): + ''' This is the callback function associated with the "RT" command ''' + logger = logging.getLogger('RT/MOVE_RT_LOGS') + rt_log = f'tests/RegressionTests_{job_obj.machine}'\ + f'.{job_obj.compiler}.log' + filepath = f'{pr_repo_loc}/{rt_log}' + rt_dir, logfile_pass = process_logfile(job_obj, filepath) + if logfile_pass: + #if job_obj.preq_dict['preq'].maintainer_can_modify: + move_rt_commands = [ + [f'git pull --ff-only origin {branch}', pr_repo_loc], + [f'git add {rt_log}', pr_repo_loc], + [f'git commit -m "[AutoRT] {job_obj.machine}' + f'.{job_obj.compiler} Job Completed.\n\n\n' + 'on-behalf-of @ufs-community "', + pr_repo_loc], + ['sleep 10', pr_repo_loc], + [f'git push httpsorigin {branch}', pr_repo_loc] + ] + job_obj.run_commands(logger, move_rt_commands) + else: + job_obj.comment_text_append(f'[RT] Log file shows failures.') + job_obj.comment_text_append(f'[RT] Please obtain logs from {pr_repo_loc}') + job_obj.preq_dict['preq'].create_issue_comment(job_obj.comment_text) + + +def process_logfile(job_obj, logfile): + logger = logging.getLogger('RT/PROCESS_LOGFILE') + rt_dir = [] + fail_string_list = ['Test', 'failed'] + if os.path.exists(logfile): + with open(logfile) as f: + for line in f: + if all(x in line for x in fail_string_list): + # if 'FAIL' in line and 'Test' in line: + job_obj.comment_text_append(f'[RT] Error: {line.rstrip(chr(10))}') + elif 'working dir' in line and not rt_dir: + rt_dir = os.path.split(line.split()[-1])[0] + elif 'SUCCESSFUL' in line: + return rt_dir, True + job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}') + else: + logger.critical(f'Could not find {job_obj.machine}' + f'.{job_obj.compiler} ' + f'{job_obj.preq_dict["action"]} log') + print(f'Could not find {job_obj.machine}.{job_obj.compiler} ' + f'{job_obj.preq_dict["action"]} log') + raise FileNotFoundError diff --git a/tests/auto-jenkins/rt_auto_jenkins.py b/tests/auto-jenkins/rt_auto_jenkins.py new file mode 100755 index 0000000000..e393e5ff40 --- /dev/null +++ b/tests/auto-jenkins/rt_auto_jenkins.py @@ -0,0 +1,365 @@ +"""Automation of UFS Regression Testing + +This script automates the process of UFS regression testing for code managers +at NOAA-EMC + +This script should be started through rt_auto.sh so that env vars are set up +prior to start. +""" +from github import Github as gh +import datetime +import subprocess +import re +import os +import sys +from glob import glob +import logging +import importlib +from shutil import rmtree + +class GHInterface: + ''' + This class stores information for communicating with GitHub + ... + + Attributes + ---------- + GHACCESSTOKEN : str + API token to autheticate with GitHub + client : pyGitHub communication object + The connection to GitHub to make API requests + ''' + + def __init__(self): + self.logger = logging.getLogger('GHINTERFACE') + + filename = 'jenkinsaccesstoken' + + if os.path.exists(filename): + if oct(os.stat(filename).st_mode)[-3:] != 600: + with open(filename) as f: + os.environ['ghapitoken'] = f.readline().strip('\n') + else: + raise Exception('File permission needs to be "600" ') + else: + raise FileNotFoundError('Cannot find file "accesstoken"') + + try: + self.client = gh(os.getenv('ghapitoken')) + except Exception as e: + self.logger.critical(f'Exception is {e}') + raise(e) + + +def set_action_from_label(machine, actions, label): + ''' Match the label that initiates a job with an action in the dict''' + # -- i.e. hera-gnu-RT + logger = logging.getLogger('MATCH_LABEL_WITH_ACTIONS') + logger.info('Setting action from Label') + split_label = label.name.split('-') + # Make sure it has three parts + if len(split_label) != 3: + return False, False + # Break the parts into their variables + label_machine = split_label[0] + label_compiler = split_label[1] + label_action = split_label[2] + # check machine name matches + if not re.match(label_machine, machine): + return False, False + # Compiler must be intel or gnu + if not str(label_compiler) in ["intel", "gnu"]: + return False, False + action_match = next((action for action in actions + if re.match(action, label_action)), False) + + logging.info(f'Compiler: {label_compiler}, Action: {action_match}') + return label_compiler, action_match + +def delete_pr_dirs(each_pr, machine): + if machine == 'hera': + workdir = '/scratch1/NCEPDEV/nems/role.epic/autort/pr' + elif machine == 'jet': + workdir = '/lfs4/HFIP/hfv3gfs/role.epic/autort/pr' + elif machine == 'gaea': + workdir = '/lustre/f2/pdata/ncep/role.epic/autort/pr' + elif machine == 'orion': + workdir = '/work/noaa/epic-ps/role-epic-ps/autort/pr' + elif machine == 'cheyenne': + workdir = '/glade/scratch/epicufsrt/autort/jenkins/autort/pr' + else: + logging.error(f'Machine {machine} is not supported for this job') + raise KeyError + ids = [str(pr.id) for pr in each_pr] + logging.debug(f'ids are: {ids}') + dirs = [x.split('/')[-2] for x in glob(f'{workdir}/*/')] + logging.debug(f'dirs: {dirs}') + for dir in dirs: + if dir != 'pr': + logging.debug(f'Checking dir {dir}') + if not dir in ids: + logging.debug(f'ID NOT A MATCH, DELETING {dir}') + delete_rt_dirs(dir, machine, workdir) + if os.path.isdir(f'{workdir}/{dir}'): + logging.debug(f'Executing rmtree in "{workdir}/{dir}"') + rmtree(f'{workdir}/{dir}') + else: + logging.debug(f'{workdir}/{dir} does not exist, not attempting to remove') + else: + logging.debug(f'ID A MATCH, NOT DELETING {dir}') + # job_obj.preq_dict["preq"].id + + +def delete_rt_dirs(in_dir, machine, workdir): + if machine == 'hera': + rt_dir ='/scratch1/NCEPDEV/stmp4/role.epic/FV3_RT' + elif machine == 'jet': + rt_dir ='/lfs4/HFIP/hfv3gfs/role.epic/RT_BASELINE/'\ + f'emc.nemspara/FV3_RT' + elif machine == 'gaea': + rt_dir = '/lustre/f2/scratch/role.epic/FV3_RT' + elif machine == 'orion': + rt_dir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT' + elif machine == 'cheyenne': + rt_dir = '/glade/scratch/epicufsrt/FV3_RT' + else: + logging.error(f'Machine {machine} is not supported for this job') + raise KeyError + globdir = f'{workdir}/{in_dir}/**/compile_*.log' + logging.debug(f'globdir: {globdir}') + logfiles = glob(globdir, recursive=True) + if not logfiles: + return + logging.debug(f'logfiles: {logfiles}') + matches = [] + for logfile in logfiles: + with open(logfile, "r") as fp: + lines = [line.split('/') for line in fp if 'rt_' in line] + lines = list(set([item for sublist in lines for item in sublist])) + lines = [s for s in lines if 'rt_' in s and '\n' not in s] + if lines: + matches.append(lines) + logging.debug(f'lines: {lines}') + matches = list(set([item for sublist in matches for item in sublist])) + logging.debug(f'matches: {matches}') + for match in matches: + if os.path.isdir(f'{rt_dir}/{match}'): + logging.debug(f'Executing rmtree in "{rt_dir}/{match}"') + rmtree(f'{rt_dir}/{match}') + else: + logging.debug(f'{rt_dir}/{match} does not exist, not attempting to remove') + + +def get_preqs_with_actions(repos, machine, ghinterface_obj, actions): + ''' Create list of dictionaries of a pull request + and its machine label and action ''' + logger = logging.getLogger('GET_PREQS_WITH_ACTIONS') + logger.info('Getting Pull Requests with Actions') + gh_preqs = [ghinterface_obj.client.get_repo(repo['address']) + .get_pulls(state='open', sort='created', base=repo['base']) + for repo in repos] + each_pr = [preq for gh_preq in gh_preqs for preq in gh_preq] + delete_pr_dirs(each_pr, machine) + preq_labels = [{'preq': pr, 'label': label} for pr in each_pr + for label in pr.get_labels()] + + jobs = [] + # return_preq = [] + for pr_label in preq_labels: + compiler, match = set_action_from_label(machine, actions, + pr_label['label']) + if match: + pr_label['action'] = match + # return_preq.append(pr_label.copy()) + jobs.append(Job(pr_label.copy(), ghinterface_obj, machine, compiler)) + + return jobs + + +class Job: + ''' + This class stores all information needed to run jobs on this machine. + This class provides all methods needed to run all jobs. + ... + + Attributes + ---------- + preq_dict: dict + Dictionary of all data that comes from the GitHub pull request + ghinterface_obj: object + An interface to GitHub setup through class GHInterface + machine: dict + Information about the machine the jobs will be running on + provided by the bash script + ''' + + def __init__(self, preq_dict, ghinterface_obj, machine, compiler): + self.logger = logging.getLogger('JOB') + self.preq_dict = preq_dict + self.job_mod = importlib.import_module( + f'jobs.{self.preq_dict["action"].lower()}') + self.ghinterface_obj = ghinterface_obj + self.machine = machine + self.compiler = compiler + self.comment_text = '***Automated RT Failure Notification***\n' + self.failed_tests = [] + + def comment_text_append(self, newtext): + self.comment_text += f'{newtext}\n' + + def remove_pr_label(self): + ''' Removes the PR label that initiated the job run from PR ''' + self.logger.info(f'Removing Label: {self.preq_dict["label"]}') + self.preq_dict['preq'].remove_from_labels(self.preq_dict['label']) + + def check_label_before_job_start(self): + # LETS Check the label still exists before the start of the job in the + # case of multiple jobs + label_to_check = f'{self.machine}'\ + f'-{self.compiler}'\ + f'-{self.preq_dict["action"]}' + labels = self.preq_dict['preq'].get_labels() + label_match = next((label for label in labels + if re.match(label.name, label_to_check)), False) + + return label_match + + def run_commands(self, logger, commands_with_cwd): + for command, in_cwd in commands_with_cwd: + logger.info(f'Running `{command}`') + logger.info(f'in location "{in_cwd}"') + try: + output = subprocess.Popen(command, shell=True, cwd=in_cwd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + except Exception as e: + self.job_failed(logger, 'subprocess.Popen') + else: + try: + out, err = output.communicate() + out = [] if not out else out.decode('utf8').split('\n') + logger.info(out) + except Exception as e: + err = [] if not err else err.decode('utf8').split('\n') + self.job_failed(logger, f'Command {command}', exception=e, + STDOUT=True, out=out, err=err) + else: + logger.info(f'Finished running: {command}') + + def run(self): + logger = logging.getLogger('JOB/RUN') + logger.info(f'Starting Job: {self.preq_dict["label"]}') + self.comment_text_append(newtext=f'Machine: {self.machine}') + self.comment_text_append(f'Compiler: {self.compiler}') + self.comment_text_append(f'Job: {self.preq_dict["action"]}') + if self.check_label_before_job_start(): + try: + logger.info('Calling remove_pr_label') + self.remove_pr_label() + logger.info('Calling Job to Run') + self.job_mod.run(self) + except Exception: + self.job_failed(logger, 'run()') + logger.info('Sending comment text') + self.send_comment_text() + else: + logger.info(f'Cannot find label {self.preq_dict["label"]}') + + def send_comment_text(self): + logger = logging.getLogger('JOB/SEND_COMMENT_TEXT') + logger.info(f'Comment Text: {self.comment_text}') + self.comment_text_append('Please make changes and add ' + 'the following label back: ' + f'{self.machine}' + f'-{self.compiler}' + f'-{self.preq_dict["action"]}') + + self.preq_dict['preq'].create_issue_comment(self.comment_text) + + def job_failed(self, logger, job_name, exception=Exception, STDOUT=False, + out=None, err=None): + logger.critical(f'{job_name} FAILED. Exception:{exception}') + + if STDOUT: + logger.critical(f'STDOUT: {[item for item in out if not None]}') + logger.critical(f'STDERR: {[eitem for eitem in err if not None]}') + +def setup_env(): + hostname = os.getenv('HOSTNAME') + if bool(re.match(re.compile('hfe.+'), hostname)): + machine = 'hera' + elif bool(re.match(re.compile('hecflow.+'), hostname)): + machine = 'hera' + elif bool(re.match(re.compile('fe.+'), hostname)): + machine = 'jet' + os.environ['ACCNR'] = 'hfv3gfs' + elif bool(re.match(re.compile('tfe.+'), hostname)): + machine = 'jet' + os.environ['ACCNR'] = 'hfv3gfs' + elif bool(re.match(re.compile('gaea.+'), hostname)): + machine = 'gaea' + os.environ['ACCNR'] = 'nggps_emc' + elif bool(re.match(re.compile('Orion-login.+'), hostname)): + machine = 'orion' + os.environ['ACCNR'] = 'epic-ps' + elif bool(re.match(re.compile('cheyenne.+'), hostname)): + machine = 'cheyenne' + os.environ['ACCNR'] = 'SCSG0002' + elif bool(re.match(re.compile('chadmin.+'), hostname)): + machine = 'cheyenne' + os.environ['ACCNR'] = 'SCSG0002' + else: + raise KeyError(f'Hostname: {hostname} does not match '\ + 'for a supported system. Exiting.') + + github_org = sys.argv[1] + base = sys.argv[2] + model = '/ufs-weather-model' + address = github_org + model + # Dictionary of GitHub repositories to check + repo_dict = [{ + 'name': 'ufs-weather-model', + 'address': address, + 'base': base + }] + + # Approved Actions + action_list = ['RT', 'BL'] + + return machine, repo_dict, action_list + + +def main(): + + # handle logging + log_filename = f'rt_auto_'\ + f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}.log' + #logging.basicConfig(filename=log_filename, filemode='w', + # level=logging.INFO) + logging.basicConfig(filename=log_filename, filemode='w', + level=logging.DEBUG) + logger = logging.getLogger('MAIN') + logger.info('Starting Script') + + # setup environment + logger.info('Getting the environment setup') + machine, repos, actions = setup_env() + + # setup interface with GitHub + logger.info('Setting up GitHub interface.') + ghinterface_obj = GHInterface() + + # get all pull requests from the GitHub object + # and turn them into Job objects + logger.info('Getting all pull requests, ' + 'labels and actions applicable to this machine.') + jobs = get_preqs_with_actions(repos, machine, + ghinterface_obj, actions) + [job.run() for job in jobs] + + + logger.info('Script Finished') + + +if __name__ == '__main__': + main() diff --git a/tests/auto-jenkins/start_rt_auto_jenkins.sh b/tests/auto-jenkins/start_rt_auto_jenkins.sh new file mode 100755 index 0000000000..203b334767 --- /dev/null +++ b/tests/auto-jenkins/start_rt_auto_jenkins.sh @@ -0,0 +1,33 @@ +#!/bin/bash --login +set -eux + +if [[ $HOSTNAME == hfe* ]]; then + export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH + export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages +elif [[ $HOSTNAME == hecflow* ]]; then + export PATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/bin:$PATH + export PYTHONPATH=/scratch1/NCEPDEV/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages +elif [[ $HOSTNAME == Orion-login-* ]]; then + export PATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/bin:$PATH + export PYTHONPATH=/work/noaa/nems/emc.nemspara/soft/miniconda3/lib/python3.8/site-packages +elif [[ $HOSTNAME == fe* ]]; then + export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH + export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages +elif [[ $HOSTNAME == tfe* ]]; then + export PATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/bin:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/bin:$PATH + export PYTHONPATH=/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/lfs4/HFIP/hfv3gfs/software/miniconda3/4.8.3/lib/python3.8/site-packages +elif [[ $HOSTNAME == gaea* ]]; then + export PATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/envs/ufs-weather-model/bin:$PATH + export PYTHONPATH=/lustre/f2/pdata/esrl/gsd/contrib/miniconda3/4.8.3/lib/python3.8/site-packages +elif [[ $HOSTNAME == *chadmin* ]] || [[ $HOSTNAME == *cheyenne* ]]; then + export MACHINE_ID=cheyenne + export PATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/bin:/glade/p/ral/jntp/tools/miniconda3/4.8.3/bin:$PATH + export PYTHONPATH=/glade/p/ral/jntp/tools/miniconda3/4.8.3/envs/ufs-weather-model/lib/python3.8/site-packages:/glade/p/ral/jntp/tools/miniconda3/4.8.3/lib/python3.8/site-packages +else + echo "No Python Path for this machine." + exit 1 +fi + +python rt_auto_jenkins.py $1 $2 + +exit 0 From b5c6fc58a3a5e90d36d9b0bb45b2fbd1de011313 Mon Sep 17 00:00:00 2001 From: ClimaBot Date: Mon, 23 Jan 2023 15:07:00 -0700 Subject: [PATCH 2/4] Updating BLDATE parm. --- tests/rt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rt.sh b/tests/rt.sh index 733282337f..182be3638f 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -445,7 +445,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then TEST_35D=true fi -BL_DATE=20230120 +BL_DATE=20230123 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}} From 47829e66a50ffc38ebcbe82d51bbd811f0b0303d Mon Sep 17 00:00:00 2001 From: ClimaBot Date: Tue, 24 Jan 2023 12:01:19 -0700 Subject: [PATCH 3/4] giving it a ludicrious date. --- tests/rt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rt.sh b/tests/rt.sh index 182be3638f..d3523fa651 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -445,7 +445,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then TEST_35D=true fi -BL_DATE=20230123 +BL_DATE=21991231 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}} From 704af3a8deffbcceff6cbe7504bd57185350f713 Mon Sep 17 00:00:00 2001 From: ClimaBot Date: Tue, 24 Jan 2023 15:01:42 -0700 Subject: [PATCH 4/4] bl-test, reverting date, updating rt.py email, and paths for Cheyenne with bl.py. --- tests/auto-jenkins/jobs/bl.py | 233 ++++++++++++++++++++++++++++++++++ tests/auto-jenkins/jobs/rt.py | 2 +- tests/rt.sh | 2 +- 3 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 tests/auto-jenkins/jobs/bl.py diff --git a/tests/auto-jenkins/jobs/bl.py b/tests/auto-jenkins/jobs/bl.py new file mode 100644 index 0000000000..76610487f8 --- /dev/null +++ b/tests/auto-jenkins/jobs/bl.py @@ -0,0 +1,233 @@ +# Imports +import datetime +import logging +import os +import sys +from . import rt + +def run(job_obj): + logger = logging.getLogger('BL/RUN') + workdir, rtbldir, blstore = set_directories(job_obj) + pr_repo_loc, repo_dir_str = clone_pr_repo(job_obj, workdir) + bldate = get_bl_date(job_obj, pr_repo_loc) + bldir = f'{blstore}/develop-{bldate}/{job_obj.compiler.upper()}' + bldirbool = check_for_bl_dir(bldir, job_obj) + run_regression_test(job_obj, pr_repo_loc) + post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir) + + +def set_directories(job_obj): + logger = logging.getLogger('BL/SET_DIRECTORIES') + if job_obj.machine == 'hera': + workdir = '/scratch1/NCEPDEV/nems/emc.nemspara/autort/pr' + blstore = '/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs' + rtbldir = '/scratch1/NCEPDEV/stmp4/emc.nemspara/FV3_RT/'\ + f'REGRESSION_TEST_{job_obj.compiler.upper()}' + elif job_obj.machine == 'jet': + workdir = '/lfs4/HFIP/h-nems/emc.nemspara/autort/pr' + blstore = '/lfs4/HFIP/h-nems/emc.nemspara/RT/NEMSfv3gfs/' + rtbldir = '/lfs4/HFIP/h-nems/emc.nemspara/RT_BASELINE/'\ + f'emc.nemspara/FV3_RT/REGRESSION_TEST_{job_obj.compiler.upper()}' + elif job_obj.machine == 'gaea': + workdir = '/lustre/f2/pdata/ncep/emc.nemspara/autort/pr' + blstore = '/lustre/f2/pdata/ncep_shared/emc.nemspara/RT/NEMSfv3gfs' + rtbldir = '/lustre/f2/scratch/emc.nemspara/FV3_RT/'\ + f'REGRESSION_TEST_{job_obj.compiler.upper()}' + elif job_obj.machine == 'orion': + workdir = '/work/noaa/nems/emc.nemspara/autort/pr' + blstore = '/work/noaa/nems/emc.nemspara/RT/NEMSfv3gfs' + rtbldir = '/work/noaa/stmp/bcurtis/stmp/bcurtis/FV3_RT/'\ + f'REGRESSION_TEST_{job_obj.compiler.upper()}' + elif job_obj.machine == 'cheyenne': + workdir = '/glade/scratch/epicufsrt/autort/jenkins/autort/pr' + blstore = '/glade/scratch/epicufsrt/GMTB/ufs-weather-model/RT/NEMSfv3gfs' + rtbldir = '/glade/scratch/epicufsrt/FV3_RT/'\ + f'REGRESSION_TEST_{job_obj.compiler.upper()}' + else: + logger.critical(f'Machine {job_obj.machine} is not supported for this job') + raise KeyError + + logger.info(f'machine: {job_obj.machine}') + logger.info(f'workdir: {workdir}') + logger.info(f'blstore: {blstore}') + logger.info(f'rtbldir: {rtbldir}') + + return workdir, rtbldir, blstore + + +def check_for_bl_dir(bldir, job_obj): + logger = logging.getLogger('BL/CHECK_FOR_BL_DIR') + logger.info('Checking if baseline directory exists') + if os.path.exists(bldir): + logger.critical(f'Baseline dir: {bldir} exists. It should not, yet.') + job_obj.comment_text_append(f'{bldir}\n Exists already. ' + 'It should not yet. Please delete.') + raise FileExistsError + return False + + +def create_bl_dir(bldir, job_obj): + logger = logging.getLogger('BL/CREATE_BL_DIR') + if not check_for_bl_dir(bldir, job_obj): + os.makedirs(bldir) + if not os.path.exists(bldir): + logger.critical(f'Someting went wrong creating {bldir}') + raise FileNotFoundError + + +#def get_bl_date(job_obj): +# logger = logging.getLogger('BL/GET_BL_DATE') +# for line in job_obj.preq_dict['preq'].body.splitlines(): +# if 'BL_DATE:' in line: +# bldate = line +# bldate = bldate.replace('BL_DATE:', '') +# bldate = bldate.replace(' ', '') +# if len(bldate) != 8: +# print(f'Date: {bldate} is not formatted YYYYMMDD') +# raise ValueError +# logger.info(f'BL_DATE: {bldate}') +# bl_format = '%Y%m%d' +# try: +# datetime.datetime.strptime(bldate, bl_format) +# except ValueError: +# logger.info(f'Date {bldate} is not formatted YYYYMMDD') +# raise ValueError +# return bldate +# logger.critical('"BL_DATE:YYYYMMDD" needs to be in the PR body.'\ +# 'On its own line. Stopping') +# raise ValueError + +def run_regression_test(job_obj, pr_repo_loc): + logger = logging.getLogger('RT/RUN_REGRESSION_TEST') + if job_obj.compiler == 'gnu' and job_obj.machine != 'hera': + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' + '&& /bin/bash --login ./rt.sh -e -c -l rt_gnu.conf', + pr_repo_loc]] + elif job_obj.compiler == 'gnu' and job_obj.machine == 'hera': + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' + '&& /bin/bash --login ./rt.sh -r -c -l rt_gnu.conf', + pr_repo_loc]] + elif job_obj.compiler == 'intel' and job_obj.machine != 'hera': + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' + '&& /bin/bash --login ./rt.sh -e -c', pr_repo_loc]] + elif job_obj.compiler == 'intel' and job_obj.machine == 'hera': + rt_command = [[f'export RT_COMPILER="{job_obj.compiler}" && cd tests ' + '&& /bin/bash --login ./rt.sh -r -c', pr_repo_loc]] + job_obj.run_commands(logger, rt_command) + + +def remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir): + logger = logging.getLogger('BL/REMOVE_PR_DATA') + rm_command = [ + [f'rm -rf {rt_dir}', pr_repo_loc], + [f'rm -rf {repo_dir_str}', pr_repo_loc] + ] + job_obj.run_commands(logger, rm_command) + + +def clone_pr_repo(job_obj, workdir): + ''' clone the GitHub pull request repo, via command line ''' + logger = logging.getLogger('BL/CLONE_PR_REPO') + repo_name = job_obj.preq_dict['preq'].head.repo.name + branch = job_obj.preq_dict['preq'].head.ref + git_url = job_obj.preq_dict['preq'].head.repo.html_url.split('//') + git_url = f'{git_url[0]}//${{ghapitoken}}@{git_url[1]}' + logger.debug(f'GIT URL: {git_url}') + logger.info('Starting repo clone') + repo_dir_str = f'{workdir}/'\ + f'{str(job_obj.preq_dict["preq"].id)}/'\ + f'{datetime.datetime.now().strftime("%Y%m%d%H%M%S")}' + pr_repo_loc = f'{repo_dir_str}/{repo_name}' + job_obj.comment_text_append(f'Repo location: {pr_repo_loc}') + create_repo_commands = [ + [f'mkdir -p "{repo_dir_str}"', os.getcwd()], + [f'git clone -b {branch} {git_url}', repo_dir_str], + ['git submodule update --init --recursive', + f'{repo_dir_str}/{repo_name}'], + ['git config user.email "ecc.platform@noaa.gov"', + f'{repo_dir_str}/{repo_name}'], + ['git config user.name "epic-cicd-jenkins"', + f'{repo_dir_str}/{repo_name}'] + ] + + job_obj.run_commands(logger, create_repo_commands) + + logger.info('Finished repo clone') + return pr_repo_loc, repo_dir_str + + +def post_process(job_obj, pr_repo_loc, repo_dir_str, rtbldir, bldir): + logger = logging.getLogger('BL/MOVE_RT_LOGS') + rt_log = f'tests/RegressionTests_{job_obj.machine}'\ + f'.{job_obj.compiler}.log' + filepath = f'{pr_repo_loc}/{rt_log}' + rt_dir, logfile_pass = process_logfile(job_obj, filepath) + if logfile_pass: + create_bl_dir(bldir, job_obj) + move_bl_command = [[f'mv {rtbldir}/* {bldir}/', pr_repo_loc]] + if job_obj.machine == 'orion': + move_bl_command.append([f'/bin/bash --login adjust_permissions.sh orion develop-{bldate}', blstore]) + job_obj.run_commands(logger, move_bl_command) + job_obj.comment_text_append('Baseline creation and move successful') + logger.info('Starting RT Job') + rt.run(job_obj) + logger.info('Finished with RT Job') + remove_pr_data(job_obj, pr_repo_loc, repo_dir_str, rt_dir) + + +def get_bl_date(job_obj, pr_repo_loc): + logger = logging.getLogger('BL/UPDATE_RT_SH') + BLDATEFOUND = False + with open(f'{pr_repo_loc}/tests/rt.sh', 'r') as f: + for line in f: + if 'BL_DATE=' in line: + logger.info('Found BL_DATE in line') + BLDATEFOUND = True + bldate = line + bldate = bldate.rstrip('\n') + bldate = bldate.replace('BL_DATE=', '') + bldate = bldate.strip(' ') + logger.info(f'bldate is "{bldate}"') + logger.info(f'Type bldate: {type(bldate)}') + bl_format = '%Y%m%d' + try: + datetime.datetime.strptime(bldate, '%Y%m%d') + except ValueError: + logger.info(f'Date {bldate} is not formatted YYYYMMDD') + raise ValueError + if not BLDATEFOUND: + job_obj.comment_text_append('BL_DATE not found in rt.sh.' + 'Please manually edit rt.sh ' + 'with BL_DATE={bldate}') + job_obj.job_failed(logger, 'get_bl_date()') + logger.info('Finished get_bl_date') + + return bldate + + +def process_logfile(job_obj, logfile): + logger = logging.getLogger('BL/PROCESS_LOGFILE') + rt_dir = [] + fail_string_list = ['Test', 'failed'] + if os.path.exists(logfile): + with open(logfile) as f: + for line in f: + if all(x in line for x in fail_string_list): + # if 'FAIL' in line and 'Test' in line: + job_obj.comment_text_append(f'{line.rstrip(chr(10))}') + elif 'working dir' in line and not rt_dir: + logger.info(f'Found "working dir" in line: {line}') + rt_dir = os.path.split(line.split()[-1])[0] + logger.info(f'It is: {rt_dir}') + job_obj.comment_text_append(f'Please manually delete: ' + f'{rt_dir}') + elif 'SUCCESSFUL' in line: + logger.info('RT Successful') + return rt_dir, True + logger.critical(f'Log file exists but is not complete') + job_obj.job_failed(logger, f'{job_obj.preq_dict["action"]}') + else: + logger.critical(f'Could not find {job_obj.machine}' + f'.{job_obj.compiler} ' + f'{job_obj.preq_dict["action"]} log') + raise FileNotFoundError diff --git a/tests/auto-jenkins/jobs/rt.py b/tests/auto-jenkins/jobs/rt.py index 568b390fda..294c85720c 100644 --- a/tests/auto-jenkins/jobs/rt.py +++ b/tests/auto-jenkins/jobs/rt.py @@ -125,7 +125,7 @@ def post_process(job_obj, pr_repo_loc, repo_dir_str, branch): [f'git add {rt_log}', pr_repo_loc], [f'git commit -m "[AutoRT] {job_obj.machine}' f'.{job_obj.compiler} Job Completed.\n\n\n' - 'on-behalf-of @ufs-community "', + 'on-behalf-of @ufs-community "', pr_repo_loc], ['sleep 10', pr_repo_loc], [f'git push httpsorigin {branch}', pr_repo_loc] diff --git a/tests/rt.sh b/tests/rt.sh index d3523fa651..733282337f 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -445,7 +445,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then TEST_35D=true fi -BL_DATE=21991231 +BL_DATE=20230120 RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}