From e2cd1a355ef6b20e2e8873f6cee56cd0dc551e23 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 11:14:48 -0500 Subject: [PATCH 1/8] Start to generalize script to also work for hofx --- parm/atm/hofx/hofx_nomodel.yaml | 16 ++++++ ...atm_var_analysis.py => run_fv3jedi_exe.py} | 56 ++++++++++--------- 2 files changed, 46 insertions(+), 26 deletions(-) create mode 100644 parm/atm/hofx/hofx_nomodel.yaml rename ush/{run_single_atm_var_analysis.py => run_fv3jedi_exe.py} (66%) mode change 100644 => 100755 diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml new file mode 100644 index 000000000..74584108c --- /dev/null +++ b/parm/atm/hofx/hofx_nomodel.yaml @@ -0,0 +1,16 @@ +window begin: '$(ATM_WINDOW_BEGIN)' +window length: $(ATM_WINDOW_LENGTH) +geometry: $(GEOM_BKG) +state: + datapath: $(BKG_DIR) + filetype: fms restart + datetime: $(BKG_ISOTIME) + filename_core: $(BKG_YYYYMMDDpHHMMSS).fv_core.res.nc + filename_trcr: $(BKG_YYYYMMDDpHHMMSS).fv_tracer.res.nc + filename_sfcd: $(BKG_YYYYMMDDpHHMMSS).sfc_data.nc + filename_sfcw: $(BKG_YYYYMMDDpHHMMSS).fv_srf_wnd.res.nc + filename_cplr: $(BKG_YYYYMMDDpHHMMSS).coupler.res + state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, + slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, + u_srf,v_srf,f10m] +observations: $<< $(OBS_LIST) diff --git a/ush/run_single_atm_var_analysis.py b/ush/run_fv3jedi_exe.py old mode 100644 new mode 100755 similarity index 66% rename from ush/run_single_atm_var_analysis.py rename to ush/run_fv3jedi_exe.py index 78ce6d197..cce1113f3 --- a/ush/run_single_atm_var_analysis.py +++ b/ush/run_fv3jedi_exe.py @@ -9,7 +9,7 @@ import yaml -def run_atm_var_analysis(yamlconfig): +def run_fv3jedi_exe(yamlconfig): logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') # open YAML file to get config try: @@ -30,9 +30,12 @@ def run_atm_var_analysis(yamlconfig): sys.path.append(ufsda_path) import ufsda # compute config for YAML for executable - analysis_subconfig = all_config_dict['analysis options'] - valid_time = analysis_subconfig['valid_time'] - h = re.findall('PT(\\d+)H', analysis_subconfig['atm_window_length'])[0] + app_mode = all_config_dict['GDASApp mode'] + if app_mode not in ['hofx', 'variational']: + raise KeyError(f"{app_mode} not supported") + executable_subconfig = all_config_dict['executable options'] + valid_time = executable_subconfig['valid_time'] + h = re.findall('PT(\\d+)H', executable_subconfig['atm_window_length'])[0] prev_cycle = valid_time - dt.timedelta(hours=int(h)) window_begin = valid_time - dt.timedelta(hours=int(h)/2) cyc = valid_time.strftime("%H") @@ -40,12 +43,12 @@ def run_atm_var_analysis(yamlconfig): gcyc = prev_cycle.strftime("%H") gdate = prev_cycle.strftime("%Y%m%d%H") var_config = { - 'BERROR_YAML': analysis_subconfig['berror_yaml'], - 'OBS_YAML_DIR': analysis_subconfig['obs_yaml_dir'], - 'OBS_LIST': analysis_subconfig['obs_list'], - 'atm': analysis_subconfig['atm'], - 'layout_x': str(analysis_subconfig['layout_x']), - 'layout_y': str(analysis_subconfig['layout_y']), + 'BERROR_YAML': executable_subconfig.get('berror_yaml', './'), + 'OBS_YAML_DIR': executable_subconfig['obs_yaml_dir'], + 'OBS_LIST': executable_subconfig['obs_list'], + 'atm': executable_subconfig.get('atm', False), + 'layout_x': str(executable_subconfig['layout_x']), + 'layout_y': str(executable_subconfig['layout_y']), 'BKG_DIR': os.path.join(workdir, 'bkg'), 'fv3jedi_fix_dir': os.path.join(workdir, 'Data', 'fv3files'), 'fv3jedi_fieldset_dir': os.path.join(workdir, 'Data', 'fieldsets'), @@ -53,44 +56,45 @@ def run_atm_var_analysis(yamlconfig): 'fv3jedi_staticb_dir': os.path.join(workdir, 'berror'), 'BIAS_DIR': os.path.join(workdir, 'obs'), 'CRTM_COEFF_DIR': os.path.join(workdir, 'crtm'), - 'BIAS_PREFIX': f"{analysis_subconfig['dump']}.t{gcyc}z", + 'BIAS_PREFIX': f"{executable_subconfig['dump']}.t{gcyc}z", 'BIAS_DATE': f"{gdate}", 'DIAG_DIR': os.path.join(workdir, 'diags'), 'OBS_DIR': os.path.join(workdir, 'obs'), - 'OBS_PREFIX': f"{analysis_subconfig['dump']}.t{cyc}z", + 'OBS_PREFIX': f"{executable_subconfig['dump']}.t{cyc}z", 'OBS_DATE': f"{cdate}", 'valid_time': f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'prev_valid_time': f"{prev_cycle.strftime('%Y-%m-%dT%H:%M:%SZ')}", - 'atm_window_length': analysis_subconfig['atm_window_length'], - 'CASE': analysis_subconfig['case'], - 'CASE_ENKF': analysis_subconfig['case_enkf'], - 'DOHYBVAR': analysis_subconfig['dohybvar'], - 'LEVS': str(analysis_subconfig['levs']), + 'atm_window_length': executable_subconfig['atm_window_length'], + 'CASE': executable_subconfig['case'], + 'CASE_ENKF': executable_subconfig.get('case_enkf', executable_subconfig['case']), + 'DOHYBVAR': executable_subconfig.get('dohybvar', False), + 'LEVS': str(executable_subconfig['levs']), } - template = analysis_subconfig['var_yaml'] - output_file = os.path.join(workdir, 'fv3jedi_var.yaml') + template = executable_subconfig['yaml_template'] + output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml") # set some environment variables os.environ['PARMgfs'] = os.path.join(all_config_dict['GDASApp home'], 'parm') # generate YAML for executable based on input config logging.info(f'Using YAML template {template}') ufsda.yamltools.genYAML(var_config, template=template, output=output_file) - logging.info(f'Wrote Variational DA YAML file to {output_file}') + logging.info(f'Wrote YAML file to {output_file}') # use R2D2 to stage backgrounds, obs, bias correction files, etc. ufsda.stage.gdas_single_cycle(var_config) # link additional fix files needed (CRTM, fieldsets, etc.) - gdasfix = analysis_subconfig['gdas_fix_root'] + gdasfix = executable_subconfig['gdas_fix_root'] ufsda.stage.gdas_fix(gdasfix, workdir, var_config) # link executable - varexe = os.path.join(all_config_dict['GDASApp home'], 'build', 'bin', 'fv3jedi_var.x') - ufsda.disk_utils.symlink(varexe, os.path.join(workdir, 'fv3jedi_var.x')) + baseexe = os.path.basename(executable_subconfig['exe_path']) + ufsda.disk_utils.symlink(executable_subconfig['exe_path'], os.path.join(workdir, baseexe)) # create output directories ufsda.disk_utils.mkdir(os.path.join(workdir, 'diags')) - ufsda.disk_utils.mkdir(os.path.join(workdir, 'anl')) + if app_mode in ['variational']: + ufsda.disk_utils.mkdir(os.path.join(workdir, 'anl')) # generate job submission script job_script = ufsda.misc_utils.create_batch_job(all_config_dict['job options'], workdir, - os.path.join(workdir, 'fv3jedi_var.x'), + os.path.join(workdir, baseexe), output_file) # submit job to queue ufsda.misc_utils.submit_batch_job(all_config_dict['job options'], workdir, job_script) @@ -100,4 +104,4 @@ def run_atm_var_analysis(yamlconfig): parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', type=str, help='Input YAML Configuration', required=True) args = parser.parse_args() - run_atm_var_analysis(args.config) + run_fv3jedi_exe(args.config) From e38bf8763631f073d9499d05fa338f9ec88d414c Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 11:22:57 -0500 Subject: [PATCH 2/8] Working hofx exe --- ush/ufsda/yamltools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/ufsda/yamltools.py b/ush/ufsda/yamltools.py index d009ceb27..5cc21d1fa 100644 --- a/ush/ufsda/yamltools.py +++ b/ush/ufsda/yamltools.py @@ -48,7 +48,9 @@ def parse_config(input_config_dict, template=None, clean=True): if config_out.get('atm', True): config_out = atmanl_case(config_out) config_out.pop('atm', None) # pop out boolean variable that will cause issues later - # do a first round of includes first + # do a first round of substitutions first + config_out = replace_vars(config_out) + # now do a first round of includes config_out = include_yaml(config_out) # pull common key values out to top layer config_out = pop_out_common(config_out) From 38cee9bc186c3f6ce55f32c114e19e6b3aa14d07 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 11:59:48 -0500 Subject: [PATCH 3/8] change string format of slurm job script creation --- ush/ufsda/misc_utils.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/ush/ufsda/misc_utils.py b/ush/ufsda/misc_utils.py index e7b665926..905fb94c3 100644 --- a/ush/ufsda/misc_utils.py +++ b/ush/ufsda/misc_utils.py @@ -32,18 +32,22 @@ def create_batch_job(job_config, working_dir, exe_path, yaml_path): with open(batch_script, 'w') as f: f.write('#!/bin/bash\n') if scheduler[job_config['machine']] == 'slurm': - f.write('#SBATCH -J GDASApp\n') - f.write('#SBATCH -o GDASApp.o%J\n') - f.write(f"#SBATCH -A {job_config['account']}\n") - f.write(f"#SBATCH -q {job_config['queue']}\n") - f.write(f"#SBATCH -p {job_config['partition']}\n") - f.write(f"#SBATCH --ntasks={job_config['ntasks']}\n") - f.write(f"#SBATCH --cpus-per-task={job_config['cpus-per-task']}\n") - f.write(f"#SBATCH --exclusive\n") - f.write(f"#SBATCH -t {job_config['walltime']}\n") - f.write(f"module use {job_config['modulepath']}\n") - f.write(f"module load GDAS/{job_config['machine']}\n") - f.write(f"cd {working_dir}\n") + sbatch = f"""#SBATCH -J GDASApp +#SBATCH -o GDASApp.o%J +#SBATCH -A {job_config['account']} +#SBATCH -q {job_config['queue']} +#SBATCH -p {job_config['partition']} +#SBATCH --ntasks={job_config['ntasks']} +#SBATCH --cpus-per-task={job_config['cpus-per-task']} +#SBATCH --exclusive +#SBATCH -t {job_config['walltime']}""" + f.write(sbatch) + commands = f""" +module use {job_config['modulepath']} +module load GDAS/{job_config['machine']} +cd {working_dir} +""" + f.write(commands) if scheduler[job_config['machine']] == 'slurm': f.write(f"srun -n $SLURM_NTASKS {exe_path} {yaml_path}\n") logging.info(f"Wrote batch submission script to {batch_script}") From fe87191c143e44e28e655df1709f23246ab9713d Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 12:32:33 -0500 Subject: [PATCH 4/8] Make FV3 background time YYYYmmddHHMMSS --- parm/atm/hofx/hofx_nomodel.yaml | 10 +++++----- parm/atm/variational/3dvar_dripcg.yaml | 10 +++++----- ush/ufsda/yamltools.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml index 74584108c..6eb79307b 100644 --- a/parm/atm/hofx/hofx_nomodel.yaml +++ b/parm/atm/hofx/hofx_nomodel.yaml @@ -5,11 +5,11 @@ state: datapath: $(BKG_DIR) filetype: fms restart datetime: $(BKG_ISOTIME) - filename_core: $(BKG_YYYYMMDDpHHMMSS).fv_core.res.nc - filename_trcr: $(BKG_YYYYMMDDpHHMMSS).fv_tracer.res.nc - filename_sfcd: $(BKG_YYYYMMDDpHHMMSS).sfc_data.nc - filename_sfcw: $(BKG_YYYYMMDDpHHMMSS).fv_srf_wnd.res.nc - filename_cplr: $(BKG_YYYYMMDDpHHMMSS).coupler.res + filename_core: $(BKG_YYYYmmddHHMMSS).fv_core.res.nc + filename_trcr: $(BKG_YYYYmmddHHMMSS).fv_tracer.res.nc + filename_sfcd: $(BKG_YYYYmmddHHMMSS).sfc_data.nc + filename_sfcw: $(BKG_YYYYmmddHHMMSS).fv_srf_wnd.res.nc + filename_cplr: $(BKG_YYYYmmddHHMMSS).coupler.res state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 042e5ddb1..829437482 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -8,11 +8,11 @@ cost function: datapath: $(BKG_DIR) filetype: fms restart datetime: $(BKG_ISOTIME) - filename_core: $(BKG_YYYYMMDDpHHMMSS).fv_core.res.nc - filename_trcr: $(BKG_YYYYMMDDpHHMMSS).fv_tracer.res.nc - filename_sfcd: $(BKG_YYYYMMDDpHHMMSS).sfc_data.nc - filename_sfcw: $(BKG_YYYYMMDDpHHMMSS).fv_srf_wnd.res.nc - filename_cplr: $(BKG_YYYYMMDDpHHMMSS).coupler.res + filename_core: $(BKG_YYYYmmddHHMMSS).fv_core.res.nc + filename_trcr: $(BKG_YYYYmmddHHMMSS).fv_tracer.res.nc + filename_sfcd: $(BKG_YYYYmmddHHMMSS).sfc_data.nc + filename_sfcw: $(BKG_YYYYmmddHHMMSS).fv_srf_wnd.res.nc + filename_cplr: $(BKG_YYYYmmddHHMMSS).coupler.res state variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis, slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph, u_srf,v_srf,f10m] diff --git a/ush/ufsda/yamltools.py b/ush/ufsda/yamltools.py index 5cc21d1fa..6b61c19ff 100644 --- a/ush/ufsda/yamltools.py +++ b/ush/ufsda/yamltools.py @@ -125,10 +125,10 @@ def calc_time_vars(config): "%Y-%m-%dT%H:%M:%SZ") else: raise KeyError("Neither $(valid_time) nor ${PDY}${cyc} defined") - config['YYYYMMDDpHHMMSS'] = valid_time_obj.strftime('%Y%m%d.%H%M%S') + config['YYYYmmddHHMMSS'] = valid_time_obj.strftime('%Y%m%d.%H%M%S') # for now bkg_time == valid_time, will change for fgat bkg_time_obj = valid_time_obj - config['BKG_YYYYMMDDpHHMMSS'] = bkg_time_obj.strftime('%Y%m%d.%H%M%S') + config['BKG_YYYYmmddHHMMSS'] = bkg_time_obj.strftime('%Y%m%d.%H%M%S') config['BKG_ISOTIME'] = bkg_time_obj.strftime('%Y-%m-%dT%H:%M:%SZ') if 'assim_freq' in os.environ: config['ATM_WINDOW_LENGTH'] = f"PT{os.environ['assim_freq']}H" From 84c2cfae62b8f2197ef3a44700f5edb01271b89a Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 14:14:54 -0500 Subject: [PATCH 5/8] Change exe_path to executable --- ush/run_fv3jedi_exe.py | 4 ++-- ush/ufsda/misc_utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/run_fv3jedi_exe.py b/ush/run_fv3jedi_exe.py index cce1113f3..11aedfba4 100755 --- a/ush/run_fv3jedi_exe.py +++ b/ush/run_fv3jedi_exe.py @@ -85,8 +85,8 @@ def run_fv3jedi_exe(yamlconfig): gdasfix = executable_subconfig['gdas_fix_root'] ufsda.stage.gdas_fix(gdasfix, workdir, var_config) # link executable - baseexe = os.path.basename(executable_subconfig['exe_path']) - ufsda.disk_utils.symlink(executable_subconfig['exe_path'], os.path.join(workdir, baseexe)) + baseexe = os.path.basename(executable_subconfig['executable']) + ufsda.disk_utils.symlink(executable_subconfig['executable'], os.path.join(workdir, baseexe)) # create output directories ufsda.disk_utils.mkdir(os.path.join(workdir, 'diags')) if app_mode in ['variational']: diff --git a/ush/ufsda/misc_utils.py b/ush/ufsda/misc_utils.py index 905fb94c3..46d3eb287 100644 --- a/ush/ufsda/misc_utils.py +++ b/ush/ufsda/misc_utils.py @@ -23,7 +23,7 @@ def isTrue(str_in): return status -def create_batch_job(job_config, working_dir, exe_path, yaml_path): +def create_batch_job(job_config, working_dir, executable, yaml_path): """ create a batch job submission shell script """ @@ -49,7 +49,7 @@ def create_batch_job(job_config, working_dir, exe_path, yaml_path): """ f.write(commands) if scheduler[job_config['machine']] == 'slurm': - f.write(f"srun -n $SLURM_NTASKS {exe_path} {yaml_path}\n") + f.write(f"srun -n $SLURM_NTASKS {executable} {yaml_path}\n") logging.info(f"Wrote batch submission script to {batch_script}") return batch_script From b764f32fc89679ab7ad8db95011878fe4428d46f Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 14:34:08 -0500 Subject: [PATCH 6/8] Make error message more verbose, move check for app mode earlier in script --- ush/run_fv3jedi_exe.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ush/run_fv3jedi_exe.py b/ush/run_fv3jedi_exe.py index 11aedfba4..417f8feec 100755 --- a/ush/run_fv3jedi_exe.py +++ b/ush/run_fv3jedi_exe.py @@ -18,6 +18,13 @@ def run_fv3jedi_exe(yamlconfig): logging.info(f'Loading configuration from {yamlconfig}') except Exception as e: logging.error(f'Error occurred when attempting to load: {yamlconfig}, error: {e}') + # check if the specified app mode is valid + app_mode = all_config_dict['GDASApp mode'] + supported_app_modes = ['hofx', 'variational'] + if app_mode not in supported_app_modes: + raise KeyError(f"'{app_mode}' not supported. " + + "Current GDASApp modes supported are: " + + f"{' | '.join(supported_app_modes)}") # create working directory workdir = all_config_dict['working directory'] try: @@ -30,9 +37,6 @@ def run_fv3jedi_exe(yamlconfig): sys.path.append(ufsda_path) import ufsda # compute config for YAML for executable - app_mode = all_config_dict['GDASApp mode'] - if app_mode not in ['hofx', 'variational']: - raise KeyError(f"{app_mode} not supported") executable_subconfig = all_config_dict['executable options'] valid_time = executable_subconfig['valid_time'] h = re.findall('PT(\\d+)H', executable_subconfig['atm_window_length'])[0] From e7aae3cc83dd6efcdc28c39a4170e165b198118e Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 14:56:02 -0500 Subject: [PATCH 7/8] Change from fv3jedi_run to jedi_run --- ush/{run_fv3jedi_exe.py => run_jedi_exe.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename ush/{run_fv3jedi_exe.py => run_jedi_exe.py} (98%) diff --git a/ush/run_fv3jedi_exe.py b/ush/run_jedi_exe.py similarity index 98% rename from ush/run_fv3jedi_exe.py rename to ush/run_jedi_exe.py index 417f8feec..2f3b161bc 100755 --- a/ush/run_fv3jedi_exe.py +++ b/ush/run_jedi_exe.py @@ -9,7 +9,7 @@ import yaml -def run_fv3jedi_exe(yamlconfig): +def run_jedi_exe(yamlconfig): logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') # open YAML file to get config try: @@ -108,4 +108,4 @@ def run_fv3jedi_exe(yamlconfig): parser = argparse.ArgumentParser() parser.add_argument('-c', '--config', type=str, help='Input YAML Configuration', required=True) args = parser.parse_args() - run_fv3jedi_exe(args.config) + run_jedi_exe(args.config) From 3640e0ef2bb250f1b222561be7a3022d50b25af2 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 15:14:49 -0500 Subject: [PATCH 8/8] add a purge --- ush/ufsda/misc_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/ufsda/misc_utils.py b/ush/ufsda/misc_utils.py index 46d3eb287..9dfa3f9b2 100644 --- a/ush/ufsda/misc_utils.py +++ b/ush/ufsda/misc_utils.py @@ -43,6 +43,7 @@ def create_batch_job(job_config, working_dir, executable, yaml_path): #SBATCH -t {job_config['walltime']}""" f.write(sbatch) commands = f""" +module purge module use {job_config['modulepath']} module load GDAS/{job_config['machine']} cd {working_dir}