From 8d7a698eade9a991dc23694dca1b94030ec09be9 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 5 Apr 2022 13:19:04 -0500 Subject: [PATCH 01/12] Add r2d2 config --- modulefiles/GDAS/orion.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua index f797f0c2e..af62bdd4a 100644 --- a/modulefiles/GDAS/orion.lua +++ b/modulefiles/GDAS/orion.lua @@ -53,6 +53,8 @@ local mpinproc = '-n' setenv('MPIEXEC_EXEC', mpiexec) setenv('MPIEXEC_NPROC', mpinproc) +setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml') + whatis("Name: ".. pkgName) whatis("Version: " .. pkgVersion) From cb92c95b890988764bc5ad3dc7b52a8b378bdc69 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 5 Apr 2022 15:07:46 -0500 Subject: [PATCH 02/12] save to change branch --- ush/run_single_atm_var_analysis.py | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 ush/run_single_atm_var_analysis.py diff --git a/ush/run_single_atm_var_analysis.py b/ush/run_single_atm_var_analysis.py new file mode 100644 index 000000000..412d96d34 --- /dev/null +++ b/ush/run_single_atm_var_analysis.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import argparse +import logging +import os +import subprocess +import ufsda +import yaml + + +def run_atm_var_analysis(yamlconfig): + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') + # open YAML file to get config + try: + with open(yamlconfig, 'r') as yamlconfig_opened: + all_config_dict = yaml.safe_load(yamlconfig_opened) + logging.info(f'Loading configuration from {yamlconfig}') + except Exception as e: + logging.error(f'Error occurred when attempting to load: {yamlconfig}, error: {e}') + # create working directory + workdir = all_config_dict['working directory'] + try: + os.makedirs(workdir, exist_ok = True) + logging.info(f'Created working directory {workdir}') + except OSError as error: + logging.error(f'Error creating {workdir}: {error}') + # use R2D2 to stage backgrounds, obs, bias correction files, etc. + # link additional fix files needed (CRTM, fieldsets, etc.) + # link executable + # generate YAML for executable based on input config + # generate job submission script + # submit job to queue + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--config', type=str, help='Input YAML Configuration', required=True) + args = parser.parse_args() + run_atm_var_analysis(args.config) From fc675ba45d6e1024e749ca626f623875fc222b7d Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 6 Apr 2022 11:49:02 -0500 Subject: [PATCH 03/12] Save before meetings --- ush/run_single_atm_var_analysis.py | 63 ++++++++++++++++++++++++++++-- ush/ufsda/__init__.py | 4 +- ush/ufsda/disk_utils.py | 19 ++++++++- ush/ufsda/misc_utils.py | 40 +++++++++++++++++++ ush/ufsda/stage.py | 45 ++++++++++++++++++++- ush/ufsda/yamltools.py | 2 +- 6 files changed, 164 insertions(+), 9 deletions(-) diff --git a/ush/run_single_atm_var_analysis.py b/ush/run_single_atm_var_analysis.py index 412d96d34..d133d1a2d 100644 --- a/ush/run_single_atm_var_analysis.py +++ b/ush/run_single_atm_var_analysis.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 import argparse +import datetime as dt import logging import os +import re import subprocess -import ufsda +import sys import yaml @@ -19,16 +21,71 @@ def run_atm_var_analysis(yamlconfig): # create working directory workdir = all_config_dict['working directory'] try: - os.makedirs(workdir, exist_ok = True) + os.makedirs(workdir, exist_ok=True) logging.info(f'Created working directory {workdir}') except OSError as error: logging.error(f'Error creating {workdir}: {error}') + # add your ufsda python package to path + ufsda_path = os.path.join(all_config_dict['GDASApp home'], 'ush') + sys.path.append(ufsda_path) + import ufsda + # compute config for YAML for executable + analysis_subconfig = all_config_dict['analysis options'] + valid_time = analysis_subconfig['valid_time'] + h = re.findall('PT(\\d+)H', analysis_subconfig['atm_window_length'])[0] + prev_cycle = valid_time - dt.timedelta(hours=int(h)) + cyc = valid_time.strftime("%H") + cdate = valid_time.strftime("%Y%m%d%H") + gcyc = prev_cycle.strftime("%H") + gdate = prev_cycle.strftime("%Y%m%d%H") + var_config = { + 'paths': analysis_subconfig['paths'], + 'OBS_LIST': analysis_subconfig['obs_list'], + 'atm': analysis_subconfig['atm'], + 'layout_x': str(analysis_subconfig['layout_x']), + 'layout_y': str(analysis_subconfig['layout_y']), + 'BKG_DIR': os.path.join(workdir, 'bkg'), + 'fv3jedi_fix_dir': os.path.join(workdir, 'Data', 'fv3files'), + 'fv3jedi_fieldset_dir': os.path.join(workdir, 'Data', 'fieldsets'), + 'ANL_DIR': os.path.join(workdir, 'anl'), + 'fv3jedi_staticb_dir': os.path.join(workdir, 'berror'), + 'BIAS_DIR': os.path.join(workdir, 'obs'), + 'CRTM_COEFF_DIR': os.path.join(workdir, 'crtm'), + 'BIAS_PREFIX': f"{analysis_subconfig['dump']}.t{gcyc}z", + 'BIAS_DATE': f"{gdate}", + 'DIAG_DIR': os.path.join(workdir, 'diags'), + 'OBS_DIR': os.path.join(workdir, 'obs'), + 'OBS_PREFIX': f"{analysis_subconfig['dump']}.t{cyc}z", + 'OBS_DATE': f"{cdate}", + 'valid_time': f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}", + 'atm_window_length': analysis_subconfig['atm_window_length'], + 'CASE': analysis_subconfig['case'], + 'CASE_ENKF': analysis_subconfig['case_enkf'], + 'DOHYBVAR': analysis_subconfig['dohybvar'], + 'LEVS': str(analysis_subconfig['levs']), + } + template = analysis_subconfig['var_yaml'] + output_file = os.path.join(workdir, 'fv3jedi_var.yaml') + # set some environment variables + os.environ['PARMgfs'] = os.path.join(all_config_dict['GDASApp home'], 'parm') + # generate YAML for executable based on input config + logging.info(f'Using YAML template {template}') + ufsda.yamltools.genYAML(var_config, template=template, output=output_file) + logging.info(f'Wrote Variational DA YAML file to {output_file}') # use R2D2 to stage backgrounds, obs, bias correction files, etc. # link additional fix files needed (CRTM, fieldsets, etc.) + gdasfix = analysis_subconfig['gdas_fix_root'] + ufsda.stage.gdas_fix(gdasfix, workdir, var_config) # link executable - # generate YAML for executable based on input config + varexe = os.path.join(all_config_dict['GDASApp home'], 'build', 'bin', 'fv3jedi_var.x') + ufsda.disk_utils.symlink(varexe, os.path.join(workdir, 'fv3jedi_var.x')) # generate job submission script + job_script = ufsda.misc_utils.create_batch_job(all_config_dict['job options'], + workdir, + os.path.join(workdir, 'fv3jedi_var.x'), + output_file) # submit job to queue + ufsda.misc_utils.submit_batch_job(all_config_dict['job options'], workdir, job_script) if __name__ == "__main__": diff --git a/ush/ufsda/__init__.py b/ush/ufsda/__init__.py index c2bacf30c..116f95c9d 100644 --- a/ush/ufsda/__init__.py +++ b/ush/ufsda/__init__.py @@ -1,7 +1,7 @@ -from .disk_utils import * +from .disk_utils import mkdir, symlink from .ufs_yaml import gen_yaml, parse_config import ufsda.stage import ufsda.r2d2 import ufsda.post import ufsda.yamltools -from .misc_utils import isTrue +from .misc_utils import isTrue, create_batch_job, submit_batch_job diff --git a/ush/ufsda/disk_utils.py b/ush/ufsda/disk_utils.py index 2766d04e0..964af6105 100644 --- a/ush/ufsda/disk_utils.py +++ b/ush/ufsda/disk_utils.py @@ -1,3 +1,7 @@ +import os +import logging + + def mkdir(dirpath): """ mkdir(dirpath) @@ -7,6 +11,17 @@ def mkdir(dirpath): import os try: os.makedirs(dirpath, exist_ok=True) - print(f"{dirpath} created successfully") + logging.info(f"{dirpath} created successfully") except OSError as error: - print(f"{dirpath} could not be created") + logging.info(f"{dirpath} could not be created") + + +def symlink(src, dest): + try: + os.symlink(src, dest) + logging.info(f"Symbolically linked {src} to {dest}") + except FileExistsError: + os.remove(dest) + logging.info(f"{dest} exists, removing...") + os.symlink(src, dest) + logging.info(f"Symbolically linked {src} to {dest}") diff --git a/ush/ufsda/misc_utils.py b/ush/ufsda/misc_utils.py index 41b1074ab..4d2411d36 100644 --- a/ush/ufsda/misc_utils.py +++ b/ush/ufsda/misc_utils.py @@ -1,3 +1,8 @@ +import logging +import os +import subprocess + + def isTrue(str_in): """ isTrue(str_in) - function to translate shell variables to python logical variables @@ -10,3 +15,38 @@ def isTrue(str_in): else: status = False return status + + +def create_batch_job(job_config, working_dir, exe_path, yaml_path): + """ + create a batch job submission shell script + """ + # open up a file for writing + batch_script = os.path.join(working_dir, 'submit_job.sh') + with open(batch_script, 'w') as f: + f.write('#!/bin/bash\n') + if job_config['machine'] in ['orion', 'hera']: + f.write('#SBATCH -J GDASApp\n') + f.write('#SBATCH -o GDASApp.o%J\n') + f.write(f"#SBATCH -A {job_config['account']}\n") + f.write(f"#SBATCH -q {job_config['queue']}\n") + f.write(f"#SBATCH -p {job_config['partition']}\n") + f.write(f"#SBATCH --ntasks={job_config['ntasks']}\n") + f.write(f"#SBATCH --cpus-per-task={job_config['cpus-per-task']}\n") + f.write(f"#SBATCH --exclusive\n") + f.write(f"#SBATCH -t {job_config['walltime']}\n") + f.write(f"module use {job_config['modulepath']}\n") + f.write(f"module load GDAS/{job_config['machine']}\n") + f.write(f"cd {working_dir}\n") + if job_config['machine'] in ['orion', 'hera']: + f.write(f"srun -n $SLURM_NTASKS {exe_path} {yaml_path}\n") + logging.info(f"Wrote batch submission script to {batch_script}") + return batch_script + + +def submit_batch_job(job_config, working_dir, job_script): + """ + submit a batch job + """ + if job_config['machine'] in ['orion', 'hera']: + subprocess.Popen(f"sbatch {job_script}", cwd=working_dir, shell=True) diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index de0bc4579..23d17d53c 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -6,8 +6,51 @@ import os import shutil import datetime as dt +import ufsda -__all__ = ['background', 'fv3jedi', 'obs', 'berror'] +__all__ = ['background', 'fv3jedi', 'obs', 'berror', 'gdas_fix'] + + +def gdas_fix(input_fix_dir, working_dir, config): + """ + gdas_fix(input_fix_dir, working_dir, config): + Stage fix files needed by FV3-JEDI for GDAS analyses + input_fix_dir - path to root fix file directory + working_dir - path to where files should be linked to + config - dict containing configuration + """ + # create output directories + ufsda.disk_utils.mkdir(config['fv3jedi_fieldset_dir']) + ufsda.disk_utils.mkdir(config['fv3jedi_fix_dir']) + # figure out analysis resolution + if config['DOHYBVAR']: + case_anl = config['CASE_ENKF'] + else: + case_anl = config['CASE'] + layers = int(config['LEVS'])-1 + # link static B files + ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'bump', case_anl), + config['fv3jedi_staticb_dir']) + # link akbk file + ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'fv3jedi', + 'fv3files', f"akbk{layers}.nc4"), + os.path.join(config['fv3jedi_fix_dir'], 'akbk.nc4')) + # link other fv3files + ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'fv3jedi', + 'fv3files', 'fmsmpp.nml'), + os.path.join(config['fv3jedi_fix_dir'], 'fmsmpp.nml')) + ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'fv3jedi', + 'fv3files', 'field_table_gfdl'), + os.path.join(config['fv3jedi_fix_dir'], 'field_table')) + # link fieldsets + fieldsets = ['dynamics.yaml', 'ufo.yaml'] + for fieldset in fieldsets: + ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'fv3jedi', + 'fieldsets', fieldset), + os.path.join(config['fv3jedi_fieldset_dir'], fieldset)) + # link CRTM coeff dir + ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'crtm', '2.3.0_jedi'), + config['CRTM_COEFF_DIR']) def background(config): diff --git a/ush/ufsda/yamltools.py b/ush/ufsda/yamltools.py index 30399c8e7..d009ceb27 100644 --- a/ush/ufsda/yamltools.py +++ b/ush/ufsda/yamltools.py @@ -67,7 +67,7 @@ def parse_config(input_config_dict, template=None, clean=True): def atmanl_case(config): # compute atm analysis case/res variables based on environment and/or config case = int(config.get('CASE', os.environ.get('CASE', 'C768'))[1:]) - case_enkf = int(config.get('CASE', os.environ.get('CASE_ENKF', 'C384'))[1:]) + case_enkf = int(config.get('CASE_ENKF', os.environ.get('CASE_ENKF', 'C384'))[1:]) levs = int(config.get('LEVS', os.environ.get('LEVS', '128'))) if 'DOHYBVAR' in config: dohybvar = config['DOHYBVAR'] From 75747072436811b1e54d91cfaf78899ee1f9b476 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 6 Apr 2022 16:22:15 -0500 Subject: [PATCH 04/12] End of day save --- parm/atm/berror/staticb_bump.yaml | 10 ++--- parm/atm/common/paths.yaml | 2 - parm/atm/obs/config/amsua_n19.yaml | 2 +- parm/atm/obs/config/sondes.yaml | 2 +- test/genYAML_prep.sh | 3 +- test/test_generate_yaml.py | 3 +- ush/run_single_atm_var_analysis.py | 7 ++- ush/ufsda/r2d2.py | 35 ++++++++++++++- ush/ufsda/stage.py | 71 +++++++++++++++++++++++++++++- 9 files changed, 121 insertions(+), 14 deletions(-) delete mode 100644 parm/atm/common/paths.yaml diff --git a/parm/atm/berror/staticb_bump.yaml b/parm/atm/berror/staticb_bump.yaml index ee2372081..7da801d0a 100644 --- a/parm/atm/berror/staticb_bump.yaml +++ b/parm/atm/berror/staticb_bump.yaml @@ -20,11 +20,11 @@ saber blocks: variables: [surface_pressure] universe radius: filetype: fms restart - datetime: 2021-12-26T00:00:00Z + datetime: 2021-12-22T00:00:00Z set datetime on read: true psinfile: true datapath: *staticb_dir - prefix: cor/20211226.000000 + prefix: cor/20211222.000000 filename_core: cor_rh.fv_core.res.nc filename_trcr: cor_rh.fv_tracer.res.nc filename_sfcd: cor_rh.sfc_data.nc @@ -36,11 +36,11 @@ saber blocks: active variables: *active_vars file: filetype: fms restart - datetime: 2021-12-26T00:00:00Z + datetime: 2021-12-22T00:00:00Z set datetime on read: true psinfile: true datapath: *staticb_dir - prefix: stddev/20211226.000000 + prefix: stddev/20211222.000000 filename_core: stddev.fv_core.res.nc filename_trcr: stddev.fv_tracer.res.nc filename_sfcd: stddev.sfc_data.nc @@ -57,7 +57,7 @@ saber blocks: universe_rad: 2000.0e3 load_vbal: true load_samp_local: true - fname_samp: vbal/vbal_2021122600_gfs_sampling + fname_samp: vbal/vbal_2021122200_gfs_sampling vbal_block: [true, true,false, true,false,false] - saber block name: BUMP_PsiChiToUV input variables: *control_vars diff --git a/parm/atm/common/paths.yaml b/parm/atm/common/paths.yaml deleted file mode 100644 index bf77b9999..000000000 --- a/parm/atm/common/paths.yaml +++ /dev/null @@ -1,2 +0,0 @@ -BERROR_YAML: ${PARMgfs}/atm/berror/staticb_bump.yaml -OBS_YAML_DIR: ${PARMgfs}/atm/obs/config diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index 7858cf1d5..2cb1d977a 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -14,7 +14,7 @@ obs operator: obs options: Sensor_ID: amsua_n19 EndianType: little_endian - CoefficientPath: $(CRTM_COEFF_DIR) + CoefficientPath: $(CRTM_COEFF_DIR)/ obs bias: input file: $(BIAS_DIR)/$(BIAS_PREFIX).amsua_n19.satbias.$(BIAS_DATE).nc4 output file: $(BIAS_DIR)/$(OBS_PREFIX).amsua_n19.satbias.$(OBS_DATE).nc4 diff --git a/parm/atm/obs/config/sondes.yaml b/parm/atm/obs/config/sondes.yaml index e72bfb2de..66ca3d067 100644 --- a/parm/atm/obs/config/sondes.yaml +++ b/parm/atm/obs/config/sondes.yaml @@ -3,7 +3,7 @@ obs space: obsdatain: obsfile: $(OBS_DIR)/$(OBS_PREFIX).sondes.$(OBS_DATE).nc4 obsgrouping: - group variables: ["station_id", "LaunchTime"] + group variables: ["station_id"] sort variable: "air_pressure" sort order: "descending" obsdataout: diff --git a/test/genYAML_prep.sh b/test/genYAML_prep.sh index ff11d0c8a..ccfe624b9 100755 --- a/test/genYAML_prep.sh +++ b/test/genYAML_prep.sh @@ -6,7 +6,8 @@ cat > testoutput/genYAML_example.yaml << EOF template: ${srcdir}/parm/atm/variational/3dvar_dripcg.yaml output: ${bindir}/test/testoutput/genYAML_output_3dvar.yaml config: - paths: $<< ${srcdir}/parm/atm/common/paths.yaml + BERROR_YAML: \${PARMgfs}/atm/berror/staticb_bump.yaml + OBS_YAML_DIR: \${PARMgfs}/atm/obs/config atm: true layout_x: '1' layout_y: '2' diff --git a/test/test_generate_yaml.py b/test/test_generate_yaml.py index 45306d900..be384c102 100644 --- a/test/test_generate_yaml.py +++ b/test/test_generate_yaml.py @@ -10,7 +10,8 @@ def test_yaml_gen_yaml(parm_dir): output_file = os.path.join(os.getcwd(), 'testoutput', 'test_yaml_gen.yaml') template = os.path.join(parm_dir, 'atm/variational/3dvar_dripcg.yaml') config_dict = { - 'paths': f'$<< {parm_dir}/atm/common/paths.yaml', + 'BERROR_YAML': '${PARMgfs}/atm/berror/staticb_bump.yaml', + 'OBS_YAML_DIR': '${PARMgfs}/atm/obs/config', 'atm': True, 'layout_x': '1', 'layout_y': '2', diff --git a/ush/run_single_atm_var_analysis.py b/ush/run_single_atm_var_analysis.py index d133d1a2d..b3eeb96a3 100644 --- a/ush/run_single_atm_var_analysis.py +++ b/ush/run_single_atm_var_analysis.py @@ -34,12 +34,14 @@ def run_atm_var_analysis(yamlconfig): valid_time = analysis_subconfig['valid_time'] h = re.findall('PT(\\d+)H', analysis_subconfig['atm_window_length'])[0] prev_cycle = valid_time - dt.timedelta(hours=int(h)) + window_begin = valid_time - dt.timedelta(hours=int(h)/2) cyc = valid_time.strftime("%H") cdate = valid_time.strftime("%Y%m%d%H") gcyc = prev_cycle.strftime("%H") gdate = prev_cycle.strftime("%Y%m%d%H") var_config = { - 'paths': analysis_subconfig['paths'], + 'BERROR_YAML': analysis_subconfig['berror_yaml'], + 'OBS_YAML_DIR': analysis_subconfig['obs_yaml_dir'], 'OBS_LIST': analysis_subconfig['obs_list'], 'atm': analysis_subconfig['atm'], 'layout_x': str(analysis_subconfig['layout_x']), @@ -58,6 +60,8 @@ def run_atm_var_analysis(yamlconfig): 'OBS_PREFIX': f"{analysis_subconfig['dump']}.t{cyc}z", 'OBS_DATE': f"{cdate}", 'valid_time': f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}", + 'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", + 'prev_valid_time': f"{prev_cycle.strftime('%Y-%m-%dT%H:%M:%SZ')}", 'atm_window_length': analysis_subconfig['atm_window_length'], 'CASE': analysis_subconfig['case'], 'CASE_ENKF': analysis_subconfig['case_enkf'], @@ -73,6 +77,7 @@ def run_atm_var_analysis(yamlconfig): ufsda.yamltools.genYAML(var_config, template=template, output=output_file) logging.info(f'Wrote Variational DA YAML file to {output_file}') # use R2D2 to stage backgrounds, obs, bias correction files, etc. + ufsda.stage.gdas_single_cycle(var_config) # link additional fix files needed (CRTM, fieldsets, etc.) gdasfix = analysis_subconfig['gdas_fix_root'] ufsda.stage.gdas_fix(gdasfix, workdir, var_config) diff --git a/ush/ufsda/r2d2.py b/ush/ufsda/r2d2.py index eec00f781..641f664f2 100644 --- a/ush/ufsda/r2d2.py +++ b/ush/ufsda/r2d2.py @@ -19,7 +19,7 @@ def store(config): dump = config.get('dump', 'gdas') source_dir = config['source_dir'] source_file_fmt = config['source_file_fmt'] - obs_types = config['obs_types'] + obs_types = config.get('obs_types', None) for time in times: year = Hour(time).format('%Y') @@ -39,3 +39,36 @@ def store(config): kwargs['step'] = config['forecast_steps'] kwargs['source_file'] = eval(f"f'{source_file_fmt}'"), r2d2.store(**kwargs) + + +def fetch(config): + kwargs = {} + kwargs['ignore_missing'] = False + for arg in config.keys(): + if arg in possible_args: + kwargs[arg] = config[arg] + type = config.type + times = date_sequence(config.start, config.end, config.step) + dump = config.get('dump', 'gdas') + obs_types = config.get('obs_types', None) + target_dir = config['target_dir'] + target_file_fmt = config['target_file_fmt'] + for time in times: + year = Hour(time).format('%Y') + month = Hour(time).format('%m') + day = Hour(time).format('%d') + hour = Hour(time).format('%H') + kwargs['date'] = time + if type in ['bc', 'ob']: + if type == 'ob': + kwargs['time_window'] = config['step'] + for obs_type in obs_types: + kwargs['target_file'] = eval(f"f'{target_file_fmt}'"), + kwargs['obs_type'] = obs_type + r2d2.fetch(**kwargs) + else: + kwargs['file_type'] = config.file_type_list + kwargs['step'] = config['forecast_steps'] + kwargs['target_file'] = eval(f"f'{target_file_fmt}'"), + r2d2.fetch(**kwargs) + diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 23d17d53c..138d458a4 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -3,12 +3,14 @@ from solo.date import Hour, DateIncrement from solo.logger import Logger from solo.stage import Stage +from solo.configuration import Configuration +from solo.nice_dict import NiceDict import os import shutil import datetime as dt import ufsda -__all__ = ['background', 'fv3jedi', 'obs', 'berror', 'gdas_fix'] +__all__ = ['background', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle'] def gdas_fix(input_fix_dir, working_dir, config): @@ -53,6 +55,73 @@ def gdas_fix(input_fix_dir, working_dir, config): config['CRTM_COEFF_DIR']) +def gdas_single_cycle(config): + # grab backgrounds first + r2d2_config = { + 'start': config['prev_valid_time'], + 'end': config['prev_valid_time'], + 'step': config['atm_window_length'], + 'forecast_steps': ['PT6H'], # 3DVar no FGAT for now + 'file_type_list': ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'], + 'target_dir': config['BKG_DIR'], + 'target_file_fmt': '{target_dir}/$(valid_date).$(file_type).tile$(tile).nc', + 'type': 'fc', + 'model': 'gfs', + 'resolution': config['CASE'].lower(), + 'database': 'shared', + 'dump': 'gdas', + 'experiment': 'oper_gdas', # change this here and other places to be oper_{dump} + 'tile': [1, 2, 3, 4, 5, 6], + 'user_date_format': '%Y%m%d.%H%M%S', + 'fc_date_rendering': 'analysis', + } + r2d2_config = NiceDict(r2d2_config) + ufsda.r2d2.fetch(r2d2_config) + # get gfs metadata + r2d2_config['model'] = 'gfs_metadata' + r2d2_config['target_file_fmt'] = '{target_dir}/$(valid_date).$(file_type)' + r2d2_config['file_type_list'] = ['coupler.res', 'fv_core.res.nc'] + del r2d2_config['tile'] + ufsda.r2d2.fetch(r2d2_config) + + # remove things from dict and add/change for obs + delvars = ['forecast_steps', 'model', 'resolution', 'fc_date_rendering', 'user_date_format'] + for d in delvars: + del r2d2_config[d] + # get list of obs to process and their output files + obs_list_yaml = config['OBS_LIST'] + obs_list_config = Configuration(obs_list_yaml) + obs_list_config = ufsda.yamltools.iter_config(config, obs_list_config) + for ob in obs_list_config['observations']: + # first get obs + r2d2_config.pop('file_type', None) + r2d2_config['type'] = 'ob' + r2d2_config['provider'] = 'ncdiag' + r2d2_config['start'] = config['window_begin'] + r2d2_config['end'] = r2d2_config['start'] + target_file = ob['obs space']['obsdatain']['obsfile'] + r2d2_config['target_file_fmt'] = target_file + r2d2_config['obs_types'] = [ob['obs space']['name']] + ufsda.r2d2.fetch(r2d2_config) + # get bias files if needed + if 'obs bias' in ob.keys(): + r2d2_config['type'] = 'bc' + r2d2_config['provider'] = 'gsi' + r2d2_config['start'] = config['prev_valid_time'] + r2d2_config['end'] = r2d2_config['start'] + r2d2_config['file_type'] = 'satbias' + target_file = ob['obs bias']['input file'] + r2d2_config['target_file_fmt'] = target_file + ufsda.r2d2.fetch(r2d2_config) + r2d2_config['file_type'] = 'tlapse' + target_file = target_file.replace('satbias', 'tlapse') + target_file = target_file.replace('nc4', 'txt') + r2d2_config['target_file_fmt'] = target_file + ufsda.r2d2.fetch(r2d2_config) + + + + def background(config): """ Stage backgrounds and create links for analysis From 54623ced2df5e11ba504072917831093c4b00514 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 7 Apr 2022 08:11:21 -0500 Subject: [PATCH 05/12] Add identity B YAML --- parm/atm/berror/staticb_identity.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 parm/atm/berror/staticb_identity.yaml diff --git a/parm/atm/berror/staticb_identity.yaml b/parm/atm/berror/staticb_identity.yaml new file mode 100644 index 000000000..6ff03bd1d --- /dev/null +++ b/parm/atm/berror/staticb_identity.yaml @@ -0,0 +1,2 @@ +covariance model: FV3JEDI-ID +date: $(window_begin) From b722e28b6442e90aec8fbc91f46812bcef65a0d9 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 7 Apr 2022 08:13:51 -0500 Subject: [PATCH 06/12] Add creation of output diags and anl dirs --- ush/run_single_atm_var_analysis.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/run_single_atm_var_analysis.py b/ush/run_single_atm_var_analysis.py index b3eeb96a3..78ce6d197 100644 --- a/ush/run_single_atm_var_analysis.py +++ b/ush/run_single_atm_var_analysis.py @@ -84,6 +84,9 @@ def run_atm_var_analysis(yamlconfig): # link executable varexe = os.path.join(all_config_dict['GDASApp home'], 'build', 'bin', 'fv3jedi_var.x') ufsda.disk_utils.symlink(varexe, os.path.join(workdir, 'fv3jedi_var.x')) + # create output directories + ufsda.disk_utils.mkdir(os.path.join(workdir, 'diags')) + ufsda.disk_utils.mkdir(os.path.join(workdir, 'anl')) # generate job submission script job_script = ufsda.misc_utils.create_batch_job(all_config_dict['job options'], workdir, From 2b401487a99274a8b98a1a6950d16c7808c0e68b Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 7 Apr 2022 08:40:21 -0500 Subject: [PATCH 07/12] Change path of output file so that code does not crash --- parm/atm/obs/config/amsua_n19.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index 2cb1d977a..975b0c3b5 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -44,7 +44,7 @@ obs bias: inflation: ratio: 1.1 ratio for small dataset: 2.0 - output file: $(BIAS_DIR)/$(OBS_PREFIX).amsua_n19.satbias.$(OBS_DATE).nc4 + output file: $(BIAS_DIR)/$(OBS_PREFIX).amsua_n19.satbias_cov.$(OBS_DATE).nc4 obs filters: - filter: BlackList filter variables: From a8a50c5a637606e407621a0496c19632b2104798 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 7 Apr 2022 08:41:14 -0500 Subject: [PATCH 08/12] coding norms --- ush/ufsda/r2d2.py | 1 - ush/ufsda/stage.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/ush/ufsda/r2d2.py b/ush/ufsda/r2d2.py index 641f664f2..ffb0a2c8a 100644 --- a/ush/ufsda/r2d2.py +++ b/ush/ufsda/r2d2.py @@ -71,4 +71,3 @@ def fetch(config): kwargs['step'] = config['forecast_steps'] kwargs['target_file'] = eval(f"f'{target_file_fmt}'"), r2d2.fetch(**kwargs) - diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 138d458a4..dd83e7df6 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -120,8 +120,6 @@ def gdas_single_cycle(config): ufsda.r2d2.fetch(r2d2_config) - - def background(config): """ Stage backgrounds and create links for analysis From 2707343d294733c1d4a9d17614d67f9cc9add56e Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 7 Apr 2022 09:44:59 -0500 Subject: [PATCH 09/12] Add ulimit -s unlimited to module --- modulefiles/GDAS/orion.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/modulefiles/GDAS/orion.lua b/modulefiles/GDAS/orion.lua index af62bdd4a..77e9800dd 100644 --- a/modulefiles/GDAS/orion.lua +++ b/modulefiles/GDAS/orion.lua @@ -55,6 +55,7 @@ setenv('MPIEXEC_NPROC', mpinproc) setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml') +execute{cmd="ulimit -s unlimited",modeA={"load"}} whatis("Name: ".. pkgName) whatis("Version: " .. pkgVersion) From 60a030710c4707601c954507fa4605e45811b88b Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 7 Apr 2022 10:22:22 -0500 Subject: [PATCH 10/12] kwargs to inputs to not use reserved name when functions are declared --- ush/ufsda/r2d2.py | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/ush/ufsda/r2d2.py b/ush/ufsda/r2d2.py index ffb0a2c8a..ecfb20476 100644 --- a/ush/ufsda/r2d2.py +++ b/ush/ufsda/r2d2.py @@ -9,11 +9,11 @@ def store(config): - kwargs = {} - kwargs['ignore_missing'] = True + inputs = {} + inputs['ignore_missing'] = True for arg in config.keys(): if arg in possible_args: - kwargs[arg] = config[arg] + inputs[arg] = config[arg] type = config.type times = date_sequence(config.start, config.end, config.step) dump = config.get('dump', 'gdas') @@ -26,27 +26,27 @@ def store(config): month = Hour(time).format('%m') day = Hour(time).format('%d') hour = Hour(time).format('%H') - kwargs['date'] = time + inputs['date'] = time if type in ['bc', 'ob']: if type == 'ob': - kwargs['time_window'] = config['step'] + inputs['time_window'] = config['step'] for obs_type in obs_types: - kwargs['source_file'] = eval(f"f'{source_file_fmt}'"), - kwargs['obs_type'] = obs_type - r2d2.store(**kwargs) + inputs['source_file'] = eval(f"f'{source_file_fmt}'"), + inputs['obs_type'] = obs_type + r2d2.store(**inputs) else: - kwargs['file_type'] = config.file_type_list - kwargs['step'] = config['forecast_steps'] - kwargs['source_file'] = eval(f"f'{source_file_fmt}'"), - r2d2.store(**kwargs) + inputs['file_type'] = config.file_type_list + inputs['step'] = config['forecast_steps'] + inputs['source_file'] = eval(f"f'{source_file_fmt}'"), + r2d2.store(**inputs) def fetch(config): - kwargs = {} - kwargs['ignore_missing'] = False + inputs = {} + inputs['ignore_missing'] = False for arg in config.keys(): if arg in possible_args: - kwargs[arg] = config[arg] + inputs[arg] = config[arg] type = config.type times = date_sequence(config.start, config.end, config.step) dump = config.get('dump', 'gdas') @@ -58,16 +58,16 @@ def fetch(config): month = Hour(time).format('%m') day = Hour(time).format('%d') hour = Hour(time).format('%H') - kwargs['date'] = time + inputs['date'] = time if type in ['bc', 'ob']: if type == 'ob': - kwargs['time_window'] = config['step'] + inputs['time_window'] = config['step'] for obs_type in obs_types: - kwargs['target_file'] = eval(f"f'{target_file_fmt}'"), - kwargs['obs_type'] = obs_type - r2d2.fetch(**kwargs) + inputs['target_file'] = eval(f"f'{target_file_fmt}'"), + inputs['obs_type'] = obs_type + r2d2.fetch(**inputs) else: - kwargs['file_type'] = config.file_type_list - kwargs['step'] = config['forecast_steps'] - kwargs['target_file'] = eval(f"f'{target_file_fmt}'"), - r2d2.fetch(**kwargs) + inputs['file_type'] = config.file_type_list + inputs['step'] = config['forecast_steps'] + inputs['target_file'] = eval(f"f'{target_file_fmt}'"), + r2d2.fetch(**inputs) From a90f2b41cf848157c691e1e46f60546e06839b1a Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 7 Apr 2022 15:38:50 -0500 Subject: [PATCH 11/12] Change to scheduler --- ush/ufsda/misc_utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ush/ufsda/misc_utils.py b/ush/ufsda/misc_utils.py index 4d2411d36..e7b665926 100644 --- a/ush/ufsda/misc_utils.py +++ b/ush/ufsda/misc_utils.py @@ -3,6 +3,12 @@ import subprocess +scheduler = { + 'orion': 'slurm', + 'hera': 'slurm', +} + + def isTrue(str_in): """ isTrue(str_in) - function to translate shell variables to python logical variables @@ -25,7 +31,7 @@ def create_batch_job(job_config, working_dir, exe_path, yaml_path): batch_script = os.path.join(working_dir, 'submit_job.sh') with open(batch_script, 'w') as f: f.write('#!/bin/bash\n') - if job_config['machine'] in ['orion', 'hera']: + if scheduler[job_config['machine']] == 'slurm': f.write('#SBATCH -J GDASApp\n') f.write('#SBATCH -o GDASApp.o%J\n') f.write(f"#SBATCH -A {job_config['account']}\n") @@ -38,7 +44,7 @@ def create_batch_job(job_config, working_dir, exe_path, yaml_path): f.write(f"module use {job_config['modulepath']}\n") f.write(f"module load GDAS/{job_config['machine']}\n") f.write(f"cd {working_dir}\n") - if job_config['machine'] in ['orion', 'hera']: + if scheduler[job_config['machine']] == 'slurm': f.write(f"srun -n $SLURM_NTASKS {exe_path} {yaml_path}\n") logging.info(f"Wrote batch submission script to {batch_script}") return batch_script @@ -48,5 +54,5 @@ def submit_batch_job(job_config, working_dir, job_script): """ submit a batch job """ - if job_config['machine'] in ['orion', 'hera']: + if scheduler[job_config['machine']] == 'slurm': subprocess.Popen(f"sbatch {job_script}", cwd=working_dir, shell=True) From ed5479446c9c150adf71744a914855953de07546 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 8 Apr 2022 08:41:15 -0500 Subject: [PATCH 12/12] change type to r2d2_type where appropriate --- ush/ufsda/r2d2.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/ufsda/r2d2.py b/ush/ufsda/r2d2.py index ecfb20476..06fbb6a96 100644 --- a/ush/ufsda/r2d2.py +++ b/ush/ufsda/r2d2.py @@ -14,7 +14,7 @@ def store(config): for arg in config.keys(): if arg in possible_args: inputs[arg] = config[arg] - type = config.type + r2d2_type = config.type times = date_sequence(config.start, config.end, config.step) dump = config.get('dump', 'gdas') source_dir = config['source_dir'] @@ -27,8 +27,8 @@ def store(config): day = Hour(time).format('%d') hour = Hour(time).format('%H') inputs['date'] = time - if type in ['bc', 'ob']: - if type == 'ob': + if r2d2_type in ['bc', 'ob']: + if r2d2_type == 'ob': inputs['time_window'] = config['step'] for obs_type in obs_types: inputs['source_file'] = eval(f"f'{source_file_fmt}'"), @@ -47,7 +47,7 @@ def fetch(config): for arg in config.keys(): if arg in possible_args: inputs[arg] = config[arg] - type = config.type + r2d2_type = config.type times = date_sequence(config.start, config.end, config.step) dump = config.get('dump', 'gdas') obs_types = config.get('obs_types', None) @@ -59,8 +59,8 @@ def fetch(config): day = Hour(time).format('%d') hour = Hour(time).format('%H') inputs['date'] = time - if type in ['bc', 'ob']: - if type == 'ob': + if r2d2_type in ['bc', 'ob']: + if r2d2_type == 'ob': inputs['time_window'] = config['step'] for obs_type in obs_types: inputs['target_file'] = eval(f"f'{target_file_fmt}'"),