NOAA-EMC · aerorahul · Mar 6, 2025 · Feb 28, 2025 · Feb 28, 2025 · Feb 28, 2025
@@ -10,7 +10,7 @@ err=$?
 ###############################################################
 # Check for errors and exit if any of the above failed
 if [[ "${err}" -ne 0 ]]; then
-  echo "FATAL ERROR: Unable to copy ICs to ${ROTDIR}; ABORT!"
+  echo "FATAL ERROR: Unable to stage ICs to ${ROTDIR}; ABORT!"
   exit "${err}"
 fi
 

@@ -6,7 +6,7 @@ analysis:
         {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %}
         - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"
         {% endfor %}
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %}

@@ -5,7 +5,7 @@ atmosphere_cold:
         {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %}
         - "{{ COMOUT_ATMOS_INPUT_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %}

@@ -5,7 +5,7 @@ atmosphere_ens_perturbation:
         {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %}
         - "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem + 1, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %}

@@ -7,7 +7,7 @@ atmosphere_nest:
         {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %}
         - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %}
@@ -25,7 +25,7 @@ atmosphere_nest:
         {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %}
         - "{{ COMOUT_ATMOS_INPUT_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_INPUT_MEM = COMOUT_ATMOS_INPUT_MEM_list[imem] %}

@@ -5,7 +5,7 @@ atmosphere_warm:
         {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %}
         - "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ATMOS_RESTART_PREV_MEM = COMOUT_ATMOS_RESTART_PREV_MEM_list[imem] %}
@@ -15,14 +15,14 @@ atmosphere_warm:
         {% for ftype in ["coupler.res", "fv_core.res.nc"] %}
         - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"]
         {% endfor %}
-        {% for ftype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %}
         {% for ntile in range(1, ntiles + 1) %}
+        {% for ftype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %}
         - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.{{ ftype }}.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"]
+        {% endfor %} # ftype
         {% if DO_CA %}
         - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.ca_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"]
         {% endif %}
         {% endfor %} # ntile
-        {% endfor %} # ftype
         {% for ntile in range(1, ntiles + 1) %}
         {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ p_prefix ~ ".sfcanl_data.tile" ~ ntile ~ ".nc") %}
         - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ p_prefix }}.sfcanl_data.tile{{ ntile }}.nc", "{{ COMOUT_ATMOS_RESTART_PREV_MEM }}"]

@@ -13,7 +13,7 @@ ice:
         {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %}
         - "{{ COMOUT_ICE_ANALYSIS_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ICE_ANALYSIS_MEM = COMOUT_ICE_ANALYSIS_MEM_list[imem] %}
@@ -26,7 +26,7 @@ ice:
         {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %}
         - "{{ COMOUT_ICE_RESTART_PREV_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_ICE_RESTART_PREV_MEM = COMOUT_ICE_RESTART_PREV_MEM_list[imem] %}

@@ -10,13 +10,17 @@
 #         - "COM directory to create"
 #     copy:
 #         - ["source_file", "destination_file"]
+#     link:
+#         - ["target", "link_name"]
 # key2:
 #     mkdir:
 #         - "COM directory to create"
 #     copy:
 #         - ["source_file", "destination_file"]
+#     link:
+#         - ["target", "link_name"]
 #
-# Any number of keys with nested mkdir and copy are permitted
+# Any number of keys with nested mkdir, copy, and link are permitted
 # Jinja is permitted in this yaml, as long as the keys are:
 # - COMOUT_
 # - DO_ATM, DO_OCN, DO_ICE, etc.

@@ -10,13 +10,17 @@
 #         - "COM directory to create"
 #     copy:
 #         - ["source_file", "destination_file"]
+#     link:
+#         - ["target", "link_name"]
 # key2:
 #     mkdir:
 #         - "COM directory to create"
 #     copy:
 #         - ["source_file", "destination_file"]
+#     link:
+#         - ["target", "link_name"]
 #
-# Any number of keys with nested mkdir and copy are permitted
+# Any number of keys with nested mkdir, copy, and link are permitted
 # Jinja is permitted in this yaml, as long as the keys are:
 # - COMOUT_
 # - DO_ATM, DO_OCN, DO_ICE, etc.

@@ -5,7 +5,7 @@ ocean:
         {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %}
         - "{{ COMOUT_OCEAN_RESTART_PREV_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_OCEAN_RESTART_PREV_MEM = COMOUT_OCEAN_RESTART_PREV_MEM_list[imem] %}

@@ -5,7 +5,7 @@ ocean_ens_perturbation:
         {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %}
         - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem + 1, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %}

@@ -6,7 +6,7 @@ ocean_mediator:
         {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %}
         - "{{ COMOUT_MED_RESTART_PREV_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_MED_RESTART_PREV_MEM = COMOUT_MED_RESTART_PREV_MEM_list[imem] %}

@@ -5,7 +5,7 @@ ocean_rerun:
         {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %}
         - "{{ COMOUT_OCEAN_ANALYSIS_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %}

@@ -5,7 +5,7 @@ wave:
         {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %}
         - "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"
         {% endfor %} # mem loop
-    copy:
+    link:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %}

@@ -56,9 +56,9 @@ def execute_stage(self, stage_dict: Dict[str, Any]) -> None:
         # Add the glob.glob function for capturing filenames
         stage_dict['glob'] = glob.glob
 
-        # Parse stage yaml to get list of files to copy
+        # Parse staging yaml to get list of files to stage
         stage_set = parse_j2yaml(self.task_config.STAGE_IC_YAML_TMPL, stage_dict, allow_missing=False)
 
-        # Copy files to ROTDIR
+        # stage files to ROTDIR
         for key in stage_set.keys():
             FileHandler(stage_set[key]).sync()
+1 −1		.github/workflows/pytests.yaml
+1 −1		src/wxflow/configuration.py
+33 −2		src/wxflow/file_utils.py
+4 −3		src/wxflow/fsutils.py
+2 −1		src/wxflow/schema.py
+7 −6		src/wxflow/sqlitedb.py
+177 −8		tests/test_file_utils.py
+1 −1		tests/test_sqlitedb.py