From abfc560cad010c664069110ce119161e40ca0ebe Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Tue, 13 May 2025 15:40:19 +0000 Subject: [PATCH 01/10] stage and archive aerosol bias correction files --- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 3 +- parm/gdas/aero_stage_variational.yaml.j2 | 2 +- ush/python/pygfs/task/aero_analysis.py | 36 ++++++++++++++++-------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index 763c45b10e1..96368188efd 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -24,7 +24,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \ - COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL + COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL \ + COMIN_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL ############################################################### # Run relevant script diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 index afd0e1b9461..c9128cb979c 100644 --- a/parm/gdas/aero_stage_variational.yaml.j2 +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -13,7 +13,7 @@ {% set bkg_times = [] %} {% do bkg_times.append(current_cycle) %} {% endif %} -{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %} +{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.', 'sfc_data.', 'fv_srf_wnd.res.'] %} ###################################### mkdir: - "{{ DATA }}/anl" diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index b88ee6e4802..7417031ab4e 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -105,17 +105,18 @@ def initialize(self) -> None: logger.debug(f"Observation files:\n{pformat(obs_dict)}") # # stage bias corrections - # logger.info(f"Staging list of bias correction files") - # bias_dict = self.jedi_dict['aeroanlvar'].render_jcb(self.task_config, 'aero_bias_staging') - # if bias_dict['copy'] is None: - # logger.info(f"No bias correction files to stage") - # else: - # bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) - # FileHandler(bias_dict).sync() - # logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # # extract bias corrections - # Jedi.extract_tar_from_filehandler_dict(bias_dict) + logger.info(f"Staging list of bias correction files") + bias_dict = self.jedi_dict['aeroanlvar'].render_jcb(self.task_config, 'aero_bias_staging') + + if bias_dict['copy'] is None: + logger.info(f"No bias correction files to stage") + else: + bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract bias corrections + Jedi.extract_tar_from_filehandler_dict(bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") @@ -183,13 +184,24 @@ def finalize(self) -> None: self._add_fms_cube_sphere_increments() # tar up bias correction files - # NOTE TODO + bfile = f"{self.task_config.APREFIX}aero_varbc_params.tar" + aertar = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, bfile) + + # get lists of aerosol bias correction files to add to tarball + satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) # copy files back to COM logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}") aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config) FileHandler(aero_var_final_list).sync() + # tar aerosol bias correction files to ROTDIR + logger.info(f"Creating aerosol bias correction tar file {aertar}") + with tarfile.open(aertar, 'w') as aerbcor: + for satfile in satlist: + aerbcor.add(satfile, arcname=os.path.basename(satfile)) + logger.info(f"Add {aerbcor.getnames()}") + # open tar file for writing with tarfile.open(aerostat, "w|gz") as archive: for diagfile in diags: From b0bf872ec4422f2d3615c3b2ad05880704df0e35 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Wed, 21 May 2025 13:56:51 +0000 Subject: [PATCH 02/10] update hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 07d8d2c2747..2154dde198e 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 07d8d2c27477ec436bd900613615c319df038c21 +Subproject commit 2154dde198ea91be7d5f737b5b982eec2023ca8f From 96a7849c11395a69dcb970363ed5b2f2322aff2b Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 22 May 2025 20:30:09 +0000 Subject: [PATCH 03/10] stage aero bc file for aero DA CI test --- parm/stage/aero.yaml.j2 | 17 +++++++++++++++++ parm/stage/master_gfs.yaml.j2 | 9 +++++++++ scripts/exglobal_stage_ic.py | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 parm/stage/aero.yaml.j2 diff --git a/parm/stage/aero.yaml.j2 b/parm/stage/aero.yaml.j2 new file mode 100644 index 00000000000..422c67a0852 --- /dev/null +++ b/parm/stage/aero.yaml.j2 @@ -0,0 +1,17 @@ +aero: + mkdir: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_CHEM_ANALYSIS_MEM = COMOUT_CHEM_ANALYSIS_MEM_list[imem] %} + - "{{ COMOUT_CHEM_ANALYSIS_MEM }}" + {% endfor %} + link_req: + {% for mem in range(first_mem, last_mem + 1) %} + {% set imem = mem - first_mem %} + {% set COMOUT_CHEM_ANALYSIS_MEM = COMOUT_CHEM_ANALYSIS_MEM_list[imem] %} + {% for ftype in ["aero_varbc_params.tar"] %} + {% if path_exists(ICSDIR ~ "/" ~ COMOUT_CHEM_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} + - ["{{ ICSDIR }}/{{ COMOUT_CHEM_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_CHEM_ANALYSIS_MEM }}"] + {% endif %} + {% endfor %} + {% endfor %} # mem loop diff --git a/parm/stage/master_gfs.yaml.j2 b/parm/stage/master_gfs.yaml.j2 index 3e5418fcab2..d1438291137 100644 --- a/parm/stage/master_gfs.yaml.j2 +++ b/parm/stage/master_gfs.yaml.j2 @@ -78,6 +78,7 @@ {% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} {% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} {% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_CHEM_ANALYSIS_MEM_list = [] %} # Construct member COM directory lists # ------------------------------------ @@ -110,6 +111,7 @@ {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_CHEM_ANALYSIS_MEM = COM_CHEM_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} # Append the member COM directories {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} @@ -122,6 +124,7 @@ {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} + {% do COMOUT_CHEM_ANALYSIS_MEM_list.append(COMOUT_CHEM_ANALYSIS_MEM)%} {% endfor %} @@ -189,3 +192,9 @@ {% include "wave.yaml.j2" %} {% endfilter %} {% endif %} + +{% if DO_AERO_ANL %} +{% filter indent(width=4) %} +{% include "aero.yaml.j2" %} +{% endfilter %} +{% endif %} diff --git a/scripts/exglobal_stage_ic.py b/scripts/exglobal_stage_ic.py index bf4217f45f6..42ba2c55364 100755 --- a/scripts/exglobal_stage_ic.py +++ b/scripts/exglobal_stage_ic.py @@ -23,7 +23,7 @@ def main(): 'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'DO_JEDIATMVAR', 'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', 'DO_JEDIOCNVAR', 'DO_STARTMEM_FROM_JEDIICE', - 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'DO_CA', + 'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST', 'DO_CA', 'DO_AERO_ANL', 'USE_ATM_ENS_PERTURB_FILES', 'USE_OCN_ENS_PERTURB_FILES'] stage_dict = AttrDict() From a8d1b16bf21831ccd47d758bc729644e8df0fe56 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Fri, 23 May 2025 14:05:55 +0000 Subject: [PATCH 04/10] resolve conflicts --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 2154dde198e..cb4f83971cc 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 2154dde198ea91be7d5f737b5b982eec2023ca8f +Subproject commit cb4f83971cc8d2e3914c8b1db9ebff886fcd27fd From 7ee257d5798b3efb76a4cac4099426f7864eaae5 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Tue, 27 May 2025 13:00:59 +0000 Subject: [PATCH 05/10] add an exception for missing BC files --- ush/python/pygfs/task/aero_analysis.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 275c16f6bf0..7df4a59a064 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -107,12 +107,15 @@ def initialize(self) -> None: if bias_dict['copy'] is None: logger.info(f"No bias correction files to stage") else: - bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # extract bias corrections - Jedi.extract_tar_from_filehandler_dict(bias_dict) + try: + bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract bias corrections + Jedi.extract_tar_from_filehandler_dict(bias_dict) + except: + logger.debug(f"Bias correction files do not exist:\n{pformat(bias_dict)}") # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From c04236d4d5551404276dfd683ebcd997cd3d292f Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Tue, 27 May 2025 13:23:38 +0000 Subject: [PATCH 06/10] fix coding norms --- ush/python/pygfs/task/aero_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 7df4a59a064..6ee87fa92f9 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -114,8 +114,8 @@ def initialize(self) -> None: # extract bias corrections Jedi.extract_tar_from_filehandler_dict(bias_dict) - except: - logger.debug(f"Bias correction files do not exist:\n{pformat(bias_dict)}") + except FileNotFoundError: + logger.error(f"Bias correction files or directories do not exist:\n{pformat(bias_dict)}") # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 79f945d998ec8f7dc2ba6274c1838295c1fe85ca Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 4 Jun 2025 11:06:40 -0400 Subject: [PATCH 07/10] export DO_AERO_ANL for all apps --- dev/parm/config/gefs/config.base.j2 | 1 + dev/parm/config/sfs/config.base.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/dev/parm/config/gefs/config.base.j2 b/dev/parm/config/gefs/config.base.j2 index 19194a6c42b..95ea18032e5 100644 --- a/dev/parm/config/gefs/config.base.j2 +++ b/dev/parm/config/gefs/config.base.j2 @@ -131,6 +131,7 @@ export DO_OCN="NO" export DO_ICE="NO" export DO_EXTRACTVARS="{{ DO_EXTRACTVARS }}" # Option to process and extract a subset of products to save on disk export DO_AERO_FCST="NO" +export DO_AERO_ANL="NO" export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points export DOIBP_WAV="NO" # Option to create point outputs from input boundary points export FRAC_GRID=".true." diff --git a/dev/parm/config/sfs/config.base.j2 b/dev/parm/config/sfs/config.base.j2 index dc79c97cc4c..2fbd87cb96e 100644 --- a/dev/parm/config/sfs/config.base.j2 +++ b/dev/parm/config/sfs/config.base.j2 @@ -130,6 +130,7 @@ export DO_OCN="NO" export DO_ICE="NO" export DO_EXTRACTVARS="{{ DO_EXTRACTVARS }}" # Option to process and extract a subset of products to save on disk export DO_AERO_FCST="NO" +export DO_AERO_ANL="NO" export DOBNDPNT_WAVE="NO" # The SFS buoys file does not currently have any boundary points export DOIBP_WAV="NO" # Option to create point outputs from input boundary points export FRAC_GRID=".true." From 1424794c28ce64d43b1ebac0dc95e34852f54c7c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 12 Jun 2025 09:15:52 -0400 Subject: [PATCH 08/10] working no DA case --- dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml | 23 +++++++++++++++++++ .../cases/yamls/gcafs_cycled_defaults_ci.yaml | 1 + .../yamls/gcafs_cycled_noDA_defaults_ci.yaml | 6 +++++ dev/parm/config/gcafs/config.base.j2 | 6 +++-- dev/parm/config/gcafs/yaml/defaults.yaml | 2 ++ dev/workflow/applications/applications.py | 1 + dev/workflow/rocoto/gcafs_tasks.py | 7 +++--- 7 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml create mode 100644 dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml diff --git a/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml b/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml new file mode 100644 index 00000000000..f0d5dc39fe0 --- /dev/null +++ b/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml @@ -0,0 +1,23 @@ +experiment: + net: gcafs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: ATMA + resdetatmos: 96 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + idate: 2021122012 + edate: 2021122100 + nens: 0 + interval: 12 + start: cold + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml + +skip_ci_on_hosts: + - orion + - gaeac5 + - hercules + - awsepicglobalworkflow diff --git a/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml b/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml index c03466e4b7f..a344a12d16d 100644 --- a/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml +++ b/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml @@ -2,3 +2,4 @@ defaults: !INC {{ HOMEgfs }}/dev/parm/config/gcafs/yaml/defaults.yaml base: DO_TEST_MODE: "NO" + AERO_ANL_RUNS: "gcdas" diff --git a/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml b/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml new file mode 100644 index 00000000000..075e142113c --- /dev/null +++ b/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml @@ -0,0 +1,6 @@ +defaults: + !INC {{ HOMEgfs }}/dev/parm/config/gcafs/yaml/defaults.yaml +base: + DO_TEST_MODE: "NO" + USE_AERO_ANL: "NO" + AERO_ANL_RUNS: "gdas" diff --git a/dev/parm/config/gcafs/config.base.j2 b/dev/parm/config/gcafs/config.base.j2 index 5d4069fd340..2afd5b63395 100644 --- a/dev/parm/config/gcafs/config.base.j2 +++ b/dev/parm/config/gcafs/config.base.j2 @@ -178,7 +178,7 @@ export DO_OCN="NO" export DO_ICE="NO" DO_AERO="NO" export DO_PREP_OBS_AERO="NO" -aero_anl_runs="gdas gcdas" # When to run aerosol analysis: gdas, gfs, or both +aero_anl_runs="{{ AERO_ANL_RUNS }}" # When to run aerosol analysis: gcdas, gcafs or both export DO_AERO_FCST="YES" export DO_AERO_ANL="NO" export DOBNDPNT_WAVE="NO" @@ -267,6 +267,7 @@ if [[ "${DO_AERO}" == "YES" ]]; then export DO_AERO_ANL="YES" fi export DO_AERO_FCST="YES" + export USE_AERO_ANL="{{ USE_AERO_ANL }}" # Use aerosol analysis in forecast fi # Surface cycle update frequency @@ -363,6 +364,7 @@ export imp_physics=8 # DA engine export DO_JEDIATMVAR="{{ DO_JEDIATMVAR }}" export DO_JEDIATMENS="{{ DO_JEDIATMENS }}" +export DO_JEDIAEROVAR="{{ DO_JEDIAEROVAR }}" export DO_JEDIOCNVAR="{{ DO_JEDIOCNVAR }}" export DO_JEDISNOWDA="{{ DO_JEDISNOWDA }}" export DO_MERGENSST="{{ DO_MERGENSST }}" @@ -490,7 +492,7 @@ if [[ "${DO_JEDIATMVAR}" = "YES" ]]; then export DO_VMINMON="NO" # GSI minimization monitoring export DO_ANLSTAT="YES" # JEDI-based analysis statistics else - if [[ ${DO_AERO} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then + if [[ ${DO_AERO_ANL} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then export DO_ANLSTAT="YES" # JEDI-based analysis statistics fi fi diff --git a/dev/parm/config/gcafs/yaml/defaults.yaml b/dev/parm/config/gcafs/yaml/defaults.yaml index e6e3a89ffbe..ef82268c480 100644 --- a/dev/parm/config/gcafs/yaml/defaults.yaml +++ b/dev/parm/config/gcafs/yaml/defaults.yaml @@ -1,4 +1,6 @@ base: + USE_AERO_ANL: "YES" + AERO_ANL_RUNS: "gcdas" DOIAU: "NO" DO_JEDIATMVAR: "NO" DO_JEDIATMENS: "NO" diff --git a/dev/workflow/applications/applications.py b/dev/workflow/applications/applications.py index 57de0ebde59..ac97dedc3d8 100644 --- a/dev/workflow/applications/applications.py +++ b/dev/workflow/applications/applications.py @@ -195,6 +195,7 @@ def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: run_options[run]['do_ice'] = run_base.get('DO_ICE', False) run_options[run]['do_prep_obs_aero'] = run_base.get('DO_PREP_OBS_AERO', False) run_options[run]['do_aero_anl'] = run_base.get('DO_AERO_ANL', False) + run_options[run]['use_aero_anl'] = run_base.get('USE_AERO_ANL', False) run_options[run]['do_aero_fcst'] = run_base.get('DO_AERO_FCST', False) if run_options[run]['do_archcom'] and run_base.get('ARCHCOM_TO', "") == "globus_hpss": diff --git a/dev/workflow/rocoto/gcafs_tasks.py b/dev/workflow/rocoto/gcafs_tasks.py index 53276c9bb0a..c4e7b5b2b73 100644 --- a/dev/workflow/rocoto/gcafs_tasks.py +++ b/dev/workflow/rocoto/gcafs_tasks.py @@ -619,8 +619,9 @@ def _fcst_cycled(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} dependencies.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{anldep}_aeroanlfinal'} - dependencies.append(rocoto.add_dependency(dep_dict)) + if self.options['use_aero_anl']: + dep_dict = {'type': 'task', 'name': f'{anldep}_aeroanlfinal'} + dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) @@ -1240,7 +1241,7 @@ def arch_tars(self): if self.options['do_anlstat'] and self.options['do_aero_anl']: dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) - if self.run in ['gcdas']: + if self.run in ['gcdas'] and self.options['do_aero_anl']: dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlgenb'} deps.append(rocoto.add_dependency(dep_dict)) # Post job dependencies From 019fe84591963d5b4a7b70b0612b87bae1795c7c Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Mon, 16 Jun 2025 18:23:45 +0000 Subject: [PATCH 09/10] Revert "Merge pull request #1 from CoryMartin-NOAA/feature/do-aero-anl" This reverts commit 541dae539844def14d2e6f1111b12a11e365b729, reversing changes made to 3022988c6cd7ce0814a7989e69ca1b1f2e534ab3. --- dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml | 23 ------------------- .../cases/yamls/gcafs_cycled_defaults_ci.yaml | 1 - .../yamls/gcafs_cycled_noDA_defaults_ci.yaml | 6 ----- dev/parm/config/gcafs/config.base.j2 | 6 ++--- dev/parm/config/gcafs/yaml/defaults.yaml | 2 -- dev/parm/config/gefs/config.base.j2 | 1 - dev/parm/config/sfs/config.base.j2 | 1 - dev/workflow/applications/applications.py | 1 - dev/workflow/rocoto/gcafs_tasks.py | 7 +++--- 9 files changed, 5 insertions(+), 43 deletions(-) delete mode 100644 dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml delete mode 100644 dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml diff --git a/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml b/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml deleted file mode 100644 index f0d5dc39fe0..00000000000 --- a/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml +++ /dev/null @@ -1,23 +0,0 @@ -experiment: - net: gcafs - mode: cycled - -arguments: - pslot: {{ 'pslot' | getenv }} - app: ATMA - resdetatmos: 96 - comroot: {{ 'RUNTESTS' | getenv }}/COMROOT - expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 - idate: 2021122012 - edate: 2021122100 - nens: 0 - interval: 12 - start: cold - yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml - -skip_ci_on_hosts: - - orion - - gaeac5 - - hercules - - awsepicglobalworkflow diff --git a/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml b/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml index a344a12d16d..c03466e4b7f 100644 --- a/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml +++ b/dev/ci/cases/yamls/gcafs_cycled_defaults_ci.yaml @@ -2,4 +2,3 @@ defaults: !INC {{ HOMEgfs }}/dev/parm/config/gcafs/yaml/defaults.yaml base: DO_TEST_MODE: "NO" - AERO_ANL_RUNS: "gcdas" diff --git a/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml b/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml deleted file mode 100644 index 075e142113c..00000000000 --- a/dev/ci/cases/yamls/gcafs_cycled_noDA_defaults_ci.yaml +++ /dev/null @@ -1,6 +0,0 @@ -defaults: - !INC {{ HOMEgfs }}/dev/parm/config/gcafs/yaml/defaults.yaml -base: - DO_TEST_MODE: "NO" - USE_AERO_ANL: "NO" - AERO_ANL_RUNS: "gdas" diff --git a/dev/parm/config/gcafs/config.base.j2 b/dev/parm/config/gcafs/config.base.j2 index 3fefced7d82..97b2968288c 100644 --- a/dev/parm/config/gcafs/config.base.j2 +++ b/dev/parm/config/gcafs/config.base.j2 @@ -178,7 +178,7 @@ export DO_OCN="NO" export DO_ICE="NO" DO_AERO="NO" export DO_PREP_OBS_AERO="NO" -aero_anl_runs="{{ AERO_ANL_RUNS }}" # When to run aerosol analysis: gcdas, gcafs or both +aero_anl_runs="gdas gcdas" # When to run aerosol analysis: gdas, gfs, or both export DO_AERO_FCST="YES" export DO_AERO_ANL="NO" export DOBNDPNT_WAVE="NO" @@ -267,7 +267,6 @@ if [[ "${DO_AERO}" == "YES" ]]; then export DO_AERO_ANL="YES" fi export DO_AERO_FCST="YES" - export USE_AERO_ANL="{{ USE_AERO_ANL }}" # Use aerosol analysis in forecast fi # Surface cycle update frequency @@ -364,7 +363,6 @@ export imp_physics=8 # DA engine export DO_JEDIATMVAR="{{ DO_JEDIATMVAR }}" export DO_JEDIATMENS="{{ DO_JEDIATMENS }}" -export DO_JEDIAEROVAR="{{ DO_JEDIAEROVAR }}" export DO_JEDIOCNVAR="{{ DO_JEDIOCNVAR }}" export DO_JEDISNOWDA="{{ DO_JEDISNOWDA }}" export DO_MERGENSST="{{ DO_MERGENSST }}" @@ -496,7 +494,7 @@ if [[ "${DO_JEDIATMVAR}" = "YES" ]]; then export DO_VMINMON="NO" # GSI minimization monitoring export DO_ANLSTAT="YES" # JEDI-based analysis statistics else - if [[ ${DO_AERO_ANL} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then + if [[ ${DO_AERO} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then export DO_ANLSTAT="YES" # JEDI-based analysis statistics fi fi diff --git a/dev/parm/config/gcafs/yaml/defaults.yaml b/dev/parm/config/gcafs/yaml/defaults.yaml index fa4ca8d9b88..f45bddfc737 100644 --- a/dev/parm/config/gcafs/yaml/defaults.yaml +++ b/dev/parm/config/gcafs/yaml/defaults.yaml @@ -1,6 +1,4 @@ base: - USE_AERO_ANL: "YES" - AERO_ANL_RUNS: "gcdas" DOIAU: "NO" DO_JEDIATMVAR: "NO" DO_JEDIATMENS: "NO" diff --git a/dev/parm/config/gefs/config.base.j2 b/dev/parm/config/gefs/config.base.j2 index 01a5a3e85bb..d5c61fd0070 100644 --- a/dev/parm/config/gefs/config.base.j2 +++ b/dev/parm/config/gefs/config.base.j2 @@ -131,7 +131,6 @@ export DO_OCN="NO" export DO_ICE="NO" export DO_EXTRACTVARS="{{ DO_EXTRACTVARS }}" # Option to process and extract a subset of products to save on disk export DO_AERO_FCST="NO" -export DO_AERO_ANL="NO" export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points export DOIBP_WAV="NO" # Option to create point outputs from input boundary points export FRAC_GRID=".true." diff --git a/dev/parm/config/sfs/config.base.j2 b/dev/parm/config/sfs/config.base.j2 index 2fbd87cb96e..dc79c97cc4c 100644 --- a/dev/parm/config/sfs/config.base.j2 +++ b/dev/parm/config/sfs/config.base.j2 @@ -130,7 +130,6 @@ export DO_OCN="NO" export DO_ICE="NO" export DO_EXTRACTVARS="{{ DO_EXTRACTVARS }}" # Option to process and extract a subset of products to save on disk export DO_AERO_FCST="NO" -export DO_AERO_ANL="NO" export DOBNDPNT_WAVE="NO" # The SFS buoys file does not currently have any boundary points export DOIBP_WAV="NO" # Option to create point outputs from input boundary points export FRAC_GRID=".true." diff --git a/dev/workflow/applications/applications.py b/dev/workflow/applications/applications.py index e49d053c1e9..59529a16740 100644 --- a/dev/workflow/applications/applications.py +++ b/dev/workflow/applications/applications.py @@ -195,7 +195,6 @@ def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: run_options[run]['do_ice'] = run_base.get('DO_ICE', False) run_options[run]['do_prep_obs_aero'] = run_base.get('DO_PREP_OBS_AERO', False) run_options[run]['do_aero_anl'] = run_base.get('DO_AERO_ANL', False) - run_options[run]['use_aero_anl'] = run_base.get('USE_AERO_ANL', False) run_options[run]['do_aero_fcst'] = run_base.get('DO_AERO_FCST', False) if run_base.get('GEFSTYPE', "") == "near-real-time": diff --git a/dev/workflow/rocoto/gcafs_tasks.py b/dev/workflow/rocoto/gcafs_tasks.py index c4e7b5b2b73..53276c9bb0a 100644 --- a/dev/workflow/rocoto/gcafs_tasks.py +++ b/dev/workflow/rocoto/gcafs_tasks.py @@ -619,9 +619,8 @@ def _fcst_cycled(self): dep_dict = {'type': 'task', 'name': f'{self.run}_prep_emissions'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.options['use_aero_anl']: - dep_dict = {'type': 'task', 'name': f'{anldep}_aeroanlfinal'} - dependencies.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{anldep}_aeroanlfinal'} + dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) @@ -1241,7 +1240,7 @@ def arch_tars(self): if self.options['do_anlstat'] and self.options['do_aero_anl']: dep_dict = {'type': 'task', 'name': f'{self.run}_anlstat'} deps.append(rocoto.add_dependency(dep_dict)) - if self.run in ['gcdas'] and self.options['do_aero_anl']: + if self.run in ['gcdas']: dep_dict = {'type': 'task', 'name': f'{self.run}_aeroanlgenb'} deps.append(rocoto.add_dependency(dep_dict)) # Post job dependencies From 345e4e38e2008245915fb33bf9a6b2ba75e86abf Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Mon, 16 Jun 2025 18:30:47 +0000 Subject: [PATCH 10/10] set DO_AERO_ANL to NO in config of gefs and sfs --- dev/parm/config/gefs/config.base.j2 | 1 + dev/parm/config/sfs/config.base.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/dev/parm/config/gefs/config.base.j2 b/dev/parm/config/gefs/config.base.j2 index d5c61fd0070..01a5a3e85bb 100644 --- a/dev/parm/config/gefs/config.base.j2 +++ b/dev/parm/config/gefs/config.base.j2 @@ -131,6 +131,7 @@ export DO_OCN="NO" export DO_ICE="NO" export DO_EXTRACTVARS="{{ DO_EXTRACTVARS }}" # Option to process and extract a subset of products to save on disk export DO_AERO_FCST="NO" +export DO_AERO_ANL="NO" export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points export DOIBP_WAV="NO" # Option to create point outputs from input boundary points export FRAC_GRID=".true." diff --git a/dev/parm/config/sfs/config.base.j2 b/dev/parm/config/sfs/config.base.j2 index dc79c97cc4c..2fbd87cb96e 100644 --- a/dev/parm/config/sfs/config.base.j2 +++ b/dev/parm/config/sfs/config.base.j2 @@ -130,6 +130,7 @@ export DO_OCN="NO" export DO_ICE="NO" export DO_EXTRACTVARS="{{ DO_EXTRACTVARS }}" # Option to process and extract a subset of products to save on disk export DO_AERO_FCST="NO" +export DO_AERO_ANL="NO" export DOBNDPNT_WAVE="NO" # The SFS buoys file does not currently have any boundary points export DOIBP_WAV="NO" # Option to create point outputs from input boundary points export FRAC_GRID=".true."