From dd3dbd536e85b6c94abb161d15b86bbff6d7df63 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Feb 2025 10:53:53 -0500 Subject: [PATCH 1/7] update hash of wxflow --- sorc/wxflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/wxflow b/sorc/wxflow index a7b49e9cc76..eccb9a5a72a 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit a7b49e9cc76ef4b50cc1c28d4b7959ebde99c5f5 +Subproject commit eccb9a5a72a12ab05ae45d253ffd3c5eee62fd06 From 4296d03a67058b6e570bbc0c563d1ed49a3322ba Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Feb 2025 10:59:41 -0500 Subject: [PATCH 2/7] update copy to link in the stage_ic parms --- parm/stage/analysis.yaml.j2 | 2 +- parm/stage/atmosphere_cold.yaml.j2 | 2 +- parm/stage/atmosphere_ens_perturbations.yaml.j2 | 2 +- parm/stage/atmosphere_nest.yaml.j2 | 4 ++-- parm/stage/atmosphere_warm.yaml.j2 | 2 +- parm/stage/ice.yaml.j2 | 4 ++-- parm/stage/ocean.yaml.j2 | 2 +- parm/stage/ocean_ens_perturbations.yaml.j2 | 2 +- parm/stage/ocean_mediator.yaml.j2 | 2 +- parm/stage/ocean_rerun.yaml.j2 | 2 +- parm/stage/wave.yaml.j2 | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index 424bf2b5fe2..d2d96c5eed9 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -6,7 +6,7 @@ analysis: {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" {% endfor %} - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} diff --git a/parm/stage/atmosphere_cold.yaml.j2 b/parm/stage/atmosphere_cold.yaml.j2 index 9eeaaf4b9ec..39e5f3f92ce 100644 --- a/parm/stage/atmosphere_cold.yaml.j2 +++ b/parm/stage/atmosphere_cold.yaml.j2 @@ -5,7 +5,7 @@ atmosphere_cold: {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} - "{{ COMOUT_ATMOS_INPUT_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} diff --git a/parm/stage/atmosphere_ens_perturbations.yaml.j2 b/parm/stage/atmosphere_ens_perturbations.yaml.j2 index d9caf58b725..31968c301c8 100644 --- a/parm/stage/atmosphere_ens_perturbations.yaml.j2 +++ b/parm/stage/atmosphere_ens_perturbations.yaml.j2 @@ -5,7 +5,7 @@ atmosphere_ens_perturbation: {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem + 1, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} diff --git a/parm/stage/atmosphere_nest.yaml.j2 b/parm/stage/atmosphere_nest.yaml.j2 index d86942c1ba2..32f41c48d55 100644 --- a/parm/stage/atmosphere_nest.yaml.j2 +++ b/parm/stage/atmosphere_nest.yaml.j2 @@ -7,7 +7,7 @@ atmosphere_nest: {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} @@ -25,7 +25,7 @@ atmosphere_nest: {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} - "{{ COMOUT_ATMOS_INPUT_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %} diff --git a/parm/stage/atmosphere_warm.yaml.j2 b/parm/stage/atmosphere_warm.yaml.j2 index 44d6d78e30c..be4316bccec 100644 --- a/parm/stage/atmosphere_warm.yaml.j2 +++ b/parm/stage/atmosphere_warm.yaml.j2 @@ -5,7 +5,7 @@ atmosphere_warm: {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %} diff --git a/parm/stage/ice.yaml.j2 b/parm/stage/ice.yaml.j2 index 241f45a35b8..320145c4c82 100644 --- a/parm/stage/ice.yaml.j2 +++ b/parm/stage/ice.yaml.j2 @@ -13,7 +13,7 @@ ice: {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_ICE_ANALYSIS_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %} @@ -26,7 +26,7 @@ ice: {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_ICE_RESTART_PREV_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %} diff --git a/parm/stage/ocean.yaml.j2 b/parm/stage/ocean.yaml.j2 index b57c36d4ac1..0d1fc3a022a 100644 --- a/parm/stage/ocean.yaml.j2 +++ b/parm/stage/ocean.yaml.j2 @@ -5,7 +5,7 @@ ocean: {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %} diff --git a/parm/stage/ocean_ens_perturbations.yaml.j2 b/parm/stage/ocean_ens_perturbations.yaml.j2 index 586b9f66cb2..78a6e34ed81 100644 --- a/parm/stage/ocean_ens_perturbations.yaml.j2 +++ b/parm/stage/ocean_ens_perturbations.yaml.j2 @@ -5,7 +5,7 @@ ocean_ens_perturbation: {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem + 1, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} diff --git a/parm/stage/ocean_mediator.yaml.j2 b/parm/stage/ocean_mediator.yaml.j2 index c986b2e7461..140745de1a7 100644 --- a/parm/stage/ocean_mediator.yaml.j2 +++ b/parm/stage/ocean_mediator.yaml.j2 @@ -6,7 +6,7 @@ ocean_mediator: {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_MED_RESTART_PREV_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %} diff --git a/parm/stage/ocean_rerun.yaml.j2 b/parm/stage/ocean_rerun.yaml.j2 index 8b4042d730d..699cab715b3 100644 --- a/parm/stage/ocean_rerun.yaml.j2 +++ b/parm/stage/ocean_rerun.yaml.j2 @@ -5,7 +5,7 @@ ocean_rerun: {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} diff --git a/parm/stage/wave.yaml.j2 b/parm/stage/wave.yaml.j2 index 2788a24343c..cd3298602e3 100644 --- a/parm/stage/wave.yaml.j2 +++ b/parm/stage/wave.yaml.j2 @@ -5,7 +5,7 @@ wave: {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} - "{{ COMOUT_WAVE_RESTART_PREV_MEM }}" {% endfor %} # mem loop - copy: + link: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %} From 647b66edbc857ece38afbf646306d021f35f7e43 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Feb 2025 11:25:57 -0500 Subject: [PATCH 3/7] update the master yamls doc-block to include link as an option in addition to copy --- parm/stage/master_gefs.yaml.j2 | 6 +- parm/stage/master_gfs.yaml.j2 | 184 --------------------------------- 2 files changed, 5 insertions(+), 185 deletions(-) diff --git a/parm/stage/master_gefs.yaml.j2 b/parm/stage/master_gefs.yaml.j2 index 4f8f645b029..378f9015408 100644 --- a/parm/stage/master_gefs.yaml.j2 +++ b/parm/stage/master_gefs.yaml.j2 @@ -10,13 +10,17 @@ # - "COM directory to create" # copy: # - ["source_file", "destination_file"] +# link: +# - ["target", "link_name"] # key2: # mkdir: # - "COM directory to create" # copy: # - ["source_file", "destination_file"] +# link: +# - ["target", "link_name"] # -# Any number of keys with nested mkdir and copy are permitted +# Any number of keys with nested mkdir, and copy/link are permitted # Jinja is permitted in this yaml, as long as the keys are: # - COMOUT_ # - DO_ATM, DO_OCN, DO_ICE, etc. diff --git a/parm/stage/master_gfs.yaml.j2 b/parm/stage/master_gfs.yaml.j2 index e8620050947..e69de29bb2d 100644 --- a/parm/stage/master_gfs.yaml.j2 +++ b/parm/stage/master_gfs.yaml.j2 @@ -1,184 +0,0 @@ -################################################################### -# This is the master yaml for the GFS -# -# Cycle, member, and RUN settings are set before including each -# component yaml based on DO switches -# -# The included yamls are intended to be of the following structure: -# key1: -# mkdir: -# - "COM directory to create" -# copy: -# - ["source_file", "destination_file"] -# key2: -# mkdir: -# - "COM directory to create" -# copy: -# - ["source_file", "destination_file"] -# -# Any number of keys with nested mkdir and copy are permitted -# Jinja is permitted in this yaml, as long as the keys are: -# - COMOUT_ -# - DO_ATM, DO_OCN, DO_ICE, etc. -# For a full list see scripts/exglobal_stage_ic.py -################################################################### - -# Set cycle date variables -# ------------------------ -{% set half_window = assim_freq // 2 %} -{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} -{% set half_window_end = (half_window | string + "H") | to_timedelta %} -{% if DOIAU and MODE == "cycled" %} - {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} -{% else %} - {% if REPLAY_ICS %} - {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_end) %} - {% else %} - {% set model_start_date_current_cycle = current_cycle %} - {% endif %} -{% endif %} - -{% set current_cycle_YMD = current_cycle | to_YMD %} -{% set current_cycle_HH = current_cycle | strftime("%H") %} -{% set previous_cycle_YMD = previous_cycle | to_YMD %} -{% set previous_cycle_HH = previous_cycle | strftime("%H") %} -{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} -{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} - -# Determine restart RUN -# --------------------- -{% set rRUN = RUN %} -{% if RUN == "gfs" %} - {% set rRUN = "gdas" %} -{% endif %} - -# Set first/last mem for loop -# --------------------------- -{% if RUN == "enkfgdas" %} # Ensemble RUN - {% set first_mem = 1 %} - {% set last_mem = NMEM_ENS %} -{% else %} # Deterministic RUN - {% set first_mem = -1 %} - {% set last_mem = -1 %} -{% endif %} - -# Declare to-be-filled lists of member COM directories -# ---------------------------------------------------- -{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} -{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} -{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} -{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} -{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} -{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} - -# Construct member COM directory lists -# ------------------------------------ -{% for mem in range(first_mem, last_mem + 1) %} - - {% if mem >= 0 %} - {% set mem_char = 'mem%03d' | format(mem) %} - {% else %} - {% set mem_char = '' %} - {% endif %} - - {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':current_cycle_YMD, - '${HH}':current_cycle_HH, - '${MEMDIR}': mem_char }) %} - {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':rRUN, - '${YMD}':previous_cycle_YMD, - '${HH}':previous_cycle_HH, - '${MEMDIR}': mem_char }) %} - - {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} - {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} - - # Append the member COM directories - {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} - {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} - {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} - {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} - {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} - {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} - {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} - {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} - {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} - -{% endfor %} - -################################################################### -# Initial condition to stage - include components based on switches -################################################################### - -{% if MODE == "cycled" %} -{% filter indent(width=4) %} -{% include "analysis.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if EXP_WARM_START %} -{% filter indent(width=4) %} -{% include "atmosphere_warm.yaml.j2" %} -{% endfilter %} -{% else %} # cold start -{% filter indent(width=4) %} -{% include "atmosphere_cold.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if DO_NEST %} -{% filter indent(width=4) %} -{% include "atmosphere_nest.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if REPLAY_ICS %} -{% filter indent(width=4) %} -{% include "atmosphere_perturbation.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if DO_ICE %} -{% filter indent(width=4) %} -{% include "ice.yaml.j2" %} -{% endfilter %} -{% endif %} - -{% if DO_OCN %} -{% filter indent(width=4) %} -{% include "ocean.yaml.j2" %} -{% endfilter %} -{% if DO_JEDIOCNVAR %} -{% filter indent(width=4) %} -{% include "ocean_rerun.yaml.j2" %} -{% endfilter %} -{% endif %} -{% if REPLAY_ICS %} -{% filter indent(width=4) %} -{% include "ocean_replay.yaml.j2" %} -{% endfilter %} -{% endif %} -{% if EXP_WARM_START %} -{% filter indent(width=4) %} -{% include "ocean_mediator.yaml.j2" %} -{% endfilter %} -{% endif %} -{% endif %} - -{% if DO_WAVE %} -{% filter indent(width=4) %} -{% include "wave.yaml.j2" %} -{% endfilter %} -{% endif %} From f536a3f173e76a3e3c89692ff0f51ca5858d312b Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Feb 2025 11:31:24 -0500 Subject: [PATCH 4/7] update comments and doc-blocks --- jobs/JGLOBAL_STAGE_IC | 2 +- parm/stage/master_gefs.yaml.j2 | 2 +- ush/python/pygfs/task/stage_ic.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jobs/JGLOBAL_STAGE_IC b/jobs/JGLOBAL_STAGE_IC index b8126f8efe9..a0fdc4e14e0 100755 --- a/jobs/JGLOBAL_STAGE_IC +++ b/jobs/JGLOBAL_STAGE_IC @@ -10,7 +10,7 @@ err=$? ############################################################### # Check for errors and exit if any of the above failed if [[ "${err}" -ne 0 ]]; then - echo "FATAL ERROR: Unable to copy ICs to ${ROTDIR}; ABORT!" + echo "FATAL ERROR: Unable to stage ICs to ${ROTDIR}; ABORT!" exit "${err}" fi diff --git a/parm/stage/master_gefs.yaml.j2 b/parm/stage/master_gefs.yaml.j2 index 378f9015408..bdb427b1d5c 100644 --- a/parm/stage/master_gefs.yaml.j2 +++ b/parm/stage/master_gefs.yaml.j2 @@ -20,7 +20,7 @@ # link: # - ["target", "link_name"] # -# Any number of keys with nested mkdir, and copy/link are permitted +# Any number of keys with nested mkdir, copy, and link are permitted # Jinja is permitted in this yaml, as long as the keys are: # - COMOUT_ # - DO_ATM, DO_OCN, DO_ICE, etc. diff --git a/ush/python/pygfs/task/stage_ic.py b/ush/python/pygfs/task/stage_ic.py index 37cc4138f3f..379e7bae03d 100644 --- a/ush/python/pygfs/task/stage_ic.py +++ b/ush/python/pygfs/task/stage_ic.py @@ -56,9 +56,9 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None: # Add the glob.glob function for capturing filenames stage_dict['glob'] = glob.glob - # Parse stage yaml to get list of files to copy + # Parse staging yaml to get list of files to stage stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False) - # Copy files to ROTDIR + # stage files to ROTDIR for key in stage_set.keys(): FileHandler(stage_set[key]).sync() From b3beae1607d8bfae3c3cf14af96bcad38f07cf12 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Feb 2025 11:33:43 -0500 Subject: [PATCH 5/7] update comments and doc-blocks --- parm/stage/master_gfs.yaml.j2 | 188 ++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/parm/stage/master_gfs.yaml.j2 b/parm/stage/master_gfs.yaml.j2 index e69de29bb2d..04011f26126 100644 --- a/parm/stage/master_gfs.yaml.j2 +++ b/parm/stage/master_gfs.yaml.j2 @@ -0,0 +1,188 @@ +################################################################### +# This is the master yaml for the GFS +# +# Cycle, member, and RUN settings are set before including each +# component yaml based on DO switches +# +# The included yamls are intended to be of the following structure: +# key1: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# link: +# - ["target", "link_name"] +# key2: +# mkdir: +# - "COM directory to create" +# copy: +# - ["source_file", "destination_file"] +# link: +# - ["target", "link_name"] +# +# Any number of keys with nested mkdir, copy, and link are permitted +# Jinja is permitted in this yaml, as long as the keys are: +# - COMOUT_ +# - DO_ATM, DO_OCN, DO_ICE, etc. +# For a full list see scripts/exglobal_stage_ic.py +################################################################### + +# Set cycle date variables +# ------------------------ +{% set half_window = assim_freq // 2 %} +{% set half_window_begin = (-half_window | string + "H") | to_timedelta %} +{% set half_window_end = (half_window | string + "H") | to_timedelta %} +{% if DOIAU and MODE == "cycled" %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_begin) %} +{% else %} + {% if REPLAY_ICS %} + {% set model_start_date_current_cycle = current_cycle | add_to_datetime(half_window_end) %} + {% else %} + {% set model_start_date_current_cycle = current_cycle %} + {% endif %} +{% endif %} + +{% set current_cycle_YMD = current_cycle | to_YMD %} +{% set current_cycle_HH = current_cycle | strftime("%H") %} +{% set previous_cycle_YMD = previous_cycle | to_YMD %} +{% set previous_cycle_HH = previous_cycle | strftime("%H") %} +{% set p_prefix = previous_cycle | strftime("%Y%m%d.%H0000") %} +{% set m_prefix = model_start_date_current_cycle | strftime("%Y%m%d.%H0000") %} + +# Determine restart RUN +# --------------------- +{% set rRUN = RUN %} +{% if RUN == "gfs" %} + {% set rRUN = "gdas" %} +{% endif %} + +# Set first/last mem for loop +# --------------------------- +{% if RUN == "enkfgdas" %} # Ensemble RUN + {% set first_mem = 1 %} + {% set last_mem = NMEM_ENS %} +{% else %} # Deterministic RUN + {% set first_mem = -1 %} + {% set last_mem = -1 %} +{% endif %} + +# Declare to-be-filled lists of member COM directories +# ---------------------------------------------------- +{% set COMOUT_ATMOS_INPUT_MEM_list = [] %} +{% set COMOUT_ATMOS_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_ATMOS_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_ANALYSIS_MEM_list = [] %} +{% set COMOUT_ICE_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_OCEAN_ANALYSIS_MEM_list = [] %} +{% set COMOUT_MED_RESTART_PREV_MEM_list = [] %} +{% set COMOUT_WAVE_RESTART_PREV_MEM_list = [] %} + +# Construct member COM directory lists +# ------------------------------------ +{% for mem in range(first_mem, last_mem + 1) %} + + {% if mem >= 0 %} + {% set mem_char = 'mem%03d' | format(mem) %} + {% else %} + {% set mem_char = '' %} + {% endif %} + + {% set current_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle_YMD, + '${HH}':current_cycle_HH, + '${MEMDIR}': mem_char }) %} + {% set previous_cycle_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':rRUN, + '${YMD}':previous_cycle_YMD, + '${HH}':previous_cycle_HH, + '${MEMDIR}': mem_char }) %} + + {% set COMOUT_ATMOS_INPUT_MEM = COM_ATMOS_INPUT_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ATMOS_RESTART_PREV_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_ICE_RESTART_PREV_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_RESTART_PREV_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(current_cycle_dict) %} + {% set COMOUT_MED_RESTART_PREV_MEM = COM_MED_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + {% set COMOUT_WAVE_RESTART_PREV_MEM = COM_WAVE_RESTART_TMPL | replace_tmpl(previous_cycle_dict) %} + + # Append the member COM directories + {% do COMOUT_ATMOS_INPUT_MEM_list.append(COMOUT_ATMOS_INPUT_MEM)%} + {% do COMOUT_ATMOS_RESTART_PREV_MEM_list.append(COMOUT_ATMOS_RESTART_PREV_MEM)%} + {% do COMOUT_ATMOS_ANALYSIS_MEM_list.append(COMOUT_ATMOS_ANALYSIS_MEM)%} + {% do COMOUT_ICE_ANALYSIS_MEM_list.append(COMOUT_ICE_ANALYSIS_MEM)%} + {% do COMOUT_ICE_RESTART_PREV_MEM_list.append(COMOUT_ICE_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_RESTART_PREV_MEM_list.append(COMOUT_OCEAN_RESTART_PREV_MEM)%} + {% do COMOUT_OCEAN_ANALYSIS_MEM_list.append(COMOUT_OCEAN_ANALYSIS_MEM)%} + {% do COMOUT_MED_RESTART_PREV_MEM_list.append(COMOUT_MED_RESTART_PREV_MEM)%} + {% do COMOUT_WAVE_RESTART_PREV_MEM_list.append(COMOUT_WAVE_RESTART_PREV_MEM)%} + +{% endfor %} + +################################################################### +# Initial condition to stage - include components based on switches +################################################################### + +{% if MODE == "cycled" %} +{% filter indent(width=4) %} +{% include "analysis.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "atmosphere_warm.yaml.j2" %} +{% endfilter %} +{% else %} # cold start +{% filter indent(width=4) %} +{% include "atmosphere_cold.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_NEST %} +{% filter indent(width=4) %} +{% include "atmosphere_nest.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "atmosphere_perturbation.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_ICE %} +{% filter indent(width=4) %} +{% include "ice.yaml.j2" %} +{% endfilter %} +{% endif %} + +{% if DO_OCN %} +{% filter indent(width=4) %} +{% include "ocean.yaml.j2" %} +{% endfilter %} +{% if DO_JEDIOCNVAR %} +{% filter indent(width=4) %} +{% include "ocean_rerun.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if REPLAY_ICS %} +{% filter indent(width=4) %} +{% include "ocean_replay.yaml.j2" %} +{% endfilter %} +{% endif %} +{% if EXP_WARM_START %} +{% filter indent(width=4) %} +{% include "ocean_mediator.yaml.j2" %} +{% endfilter %} +{% endif %} +{% endif %} + +{% if DO_WAVE %} +{% filter indent(width=4) %} +{% include "wave.yaml.j2" %} +{% endfilter %} +{% endif %} From c81631a2e0d87ec5884b64c4d505f027ecca8112 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 4 Mar 2025 11:07:40 -0500 Subject: [PATCH 6/7] update w/ fix from @CatherineThomas-NOAA --- parm/stage/atmosphere_warm.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/stage/atmosphere_warm.yaml.j2 b/parm/stage/atmosphere_warm.yaml.j2 index be4316bccec..20f94fd6753 100644 --- a/parm/stage/atmosphere_warm.yaml.j2 +++ b/parm/stage/atmosphere_warm.yaml.j2 @@ -15,14 +15,14 @@ atmosphere_warm: {% for ftype in ["coupler.res", "fv_core.res.nc"] %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] {% endfor %} - {% for ftype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} {% for ntile in range(1, ntiles + 1) %} + {% for ftype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] + {% endfor %} # ftype {% if DO_CA %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.ca_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] {% endif %} {% endfor %} # ntile - {% endfor %} # ftype {% for ntile in range(1, ntiles + 1) %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ p_prefix ~ ".sfcanl_data.tile" ~ ntile ~ ".nc") %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ p_prefix }}.sfcanl_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"] From 52860ecd4099fce42c9c33e302bf287976b3148d Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Wed, 5 Mar 2025 10:24:37 -0500 Subject: [PATCH 7/7] update wxflow hash to handle existing files for links --- sorc/wxflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/wxflow b/sorc/wxflow index eccb9a5a72a..039d2c05828 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit eccb9a5a72a12ab05ae45d253ffd3c5eee62fd06 +Subproject commit 039d2c05828097d32896e1eb41795a7663d92a2a