Decrease HPSS storage for GFS retros and address hpss bugs

dev/ci/cases/gfsv17/s2sw.yaml

@@ -17,7 +17,7 @@ base:
   DO_FIT2OBS: "YES"
   DO_VERFOZN: "YES"
   DO_VERFRAD: "YES"
-  EUPD_CYC: "both"
+  EUPD_CYC: "gdas"
   INTERVAL_GFS: 6
   SDATE_GFS: 2024111512
   FHMAX_GFS: 384

dev/ci/cases/gfsv17/s2sw_rdhpcs.yaml

@@ -17,7 +17,7 @@ base:
   DO_FIT2OBS: "YES"
   DO_VERFOZN: "YES"
   DO_VERFRAD: "YES"
-  EUPD_CYC: "both"
+  EUPD_CYC: "gdas"
   INTERVAL_GFS: 6
   SDATE_GFS: 2024111512
   FHMAX_GFS: 384

dev/parm/config/gfs/config.arch_tars

@@ -17,7 +17,10 @@ export ARCH_HASHES='YES'     # Archive the hashes of the GW and submodules and '
 export ARCH_DIFFS='NO'       # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR
 
 export ARCH_GAUSSIAN="YES"
-export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS:-}
+export ARCH_GAUSSIAN_FHMAX=0
+if [[ -z "${FHMAX_GFS+x}" ]]; then
+    export ARCH_GAUSSIAN_FHMAX=$(( FHMAX_GFS < 24 ? FHMAX_GFS : 24 ))
+fi
 export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS:-}
 
 # If we are running globus archiving, create tarballs in a temporary location

dev/parm/config/gfs/config.ufs

@@ -300,6 +300,7 @@ case "${fv3_res}" in
         elif [[ "${RUN}" = gfs ]]; then
           export layout_x=32
           export layout_y=32
+          export blocksize=36 # Note, default is 32 and MOD((resolution/layout_x)*(resolution/layout_y),blocksize) == 0
           export WRITE_GROUP=8
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=50
         elif [[ "${RUN}" = *gcafs ]]; then

dev/workflow/rocoto/gfs_tasks.py

@@ -2281,7 +2281,6 @@ def arch_tars(self):
 
             if self.options['do_wave']:
                 tarball_types.append('gdaswave')
-                tarball_types.append('gdaswave_restart')
 
             if self.app_config.mode == 'cycled':
                 # Add restart archives (timing logic handled in template)
@@ -2291,6 +2290,8 @@ def arch_tars(self):
                     tarball_types.append('gdasice_restart')
                 if self.options['do_ocean']:
                     tarball_types.append('gdasocean_restart')
+                if self.options['do_wave']:
+                    tarball_types.append('gdaswave_restart')
 
         # Create a metatask that contains all the individual archive jobs
         dependencies = self._arch_tars_deps()

parm/archive/enkf_marine_grp.yaml.j2

@@ -14,11 +14,9 @@ enkf_marine_grp:
 
         # Forecast data
         {% if is_gdas %}
-        {% for fhr in range(3, 10, 3) %}
-        - "{{ COMIN_OCEAN_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}inst.f{{ "%03d" % fhr }}.nc"
-        {% endfor %}
+        - "{{ COMIN_OCEAN_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}inst.f009.nc"
 
-        {% for body in ["ic", "inst.f006", "inst.f009"] %}
+        {% for body in ["ic", "inst.f009"] %}
         - "{{ COMIN_ICE_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}{{ body }}.nc"
         {% endfor %}
         {% endif %}
@@ -30,8 +28,6 @@ enkf_marine_grp:
         {% set anl_delta = "0H" | to_timedelta %}
         {% endif %}
         {% set anl_time = current_cycle | add_to_datetime(anl_delta) %}
-        - "{{ COMIN_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}mom6_increment.i006.nc"
-        - "{{ COMIN_ICE_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.analysis.cice_model.res.nc"
         {% if DOLETKF_OCN %}
         - "{{ COMIN_OCEAN_LETKF_MEM | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc"
         - "{{ COMIN_ICE_LETKF_MEM | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc"

parm/archive/enkf_restarta_grp.yaml.j2

@@ -5,12 +5,14 @@ enkf_restarta_grp:
         {% for mem in range(first_group_mem, last_group_mem + 1) %}
             {% set imem = mem - first_group_mem %}
         # Construct the pertinent member COM directories
-            {% set COMIN_ATMOS_ANALYSIS_MEM = COMIN_ATMOS_ANALYSIS_MEM_list[imem] %}
-            {% set COMIN_ATMOS_HISTORY_MEM = COMIN_ATMOS_HISTORY_MEM_list[imem] %}
-            {% set COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_MEM_list[imem] %}
-            {% set COMIN_OCEAN_RESTART_MEM = COMIN_OCEAN_RESTART_MEM_list[imem] %}
-            {% set COMIN_ICE_RESTART_MEM = COMIN_ICE_RESTART_MEM_list[imem] %}
-            {% set COMIN_MED_RESTART_MEM = COMIN_MED_RESTART_MEM_list[imem] %}
+             {% set COMIN_ATMOS_ANALYSIS_MEM = COMIN_ATMOS_ANALYSIS_MEM_list[imem] %}
+             {% set COMIN_OCEAN_ANALYSIS_MEM = COMIN_OCEAN_ANALYSIS_MEM_list[imem] %}
+             {% set COMIN_ICE_ANALYSIS_MEM = COMIN_ICE_ANALYSIS_MEM_list[imem] %}
+             {% set COMIN_ATMOS_HISTORY_MEM = COMIN_ATMOS_HISTORY_MEM_list[imem] %}
+             {% set COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_MEM_list[imem] %}
+             {% set COMIN_OCEAN_RESTART_MEM = COMIN_OCEAN_RESTART_MEM_list[imem] %}
+             {% set COMIN_ICE_RESTART_MEM = COMIN_ICE_RESTART_MEM_list[imem] %}
+             {% set COMIN_MED_RESTART_MEM = COMIN_MED_RESTART_MEM_list[imem] %}
 
         # Store bias data
             {% if not lobsdiag_forenkf %}
@@ -72,9 +74,9 @@ enkf_restarta_grp:
                     {% set rst_delta = "+6H" | to_timedelta %}
                 {% endif %} # DOIAU_ENKF
                 {% set rst_time = current_cycle | add_to_datetime(rst_delta) %}
-        - "{{ COMIN_OCEAN_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.MOM.r*.nc"
-        - "{{ COMIN_ICE_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.cice_model.res.nc"
         - "{{ COMIN_MED_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.ufs.cpld.cpl.r.nc"
+        - "{{ COMIN_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}mom6_increment.i006.nc"
+        - "{{ COMIN_ICE_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ anl_time | to_fv3time }}.analysis.cice_model.res.nc"
             {% endif %} # DOHYBVAR_OCN
 
         {% endfor %}  # first_group_mem to last_group_mem

parm/archive/enkf_restartb_grp.yaml.j2

@@ -48,7 +48,6 @@ enkf_restartb_grp:
         {% set rst_delta = "+3H" | to_timedelta %}
         {% set rst_time = current_cycle | add_to_datetime(rst_delta) %}
         - "{{ COMIN_OCEAN_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.MOM.r*.nc"
-        - "{{ COMIN_ICE_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.cice_model.res.nc"
         - "{{ COMIN_MED_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.ufs.cpld.cpl.r.nc"
         {% endif %}
         {% endfor %}

parm/archive/gdasice.yaml.j2

@@ -3,7 +3,5 @@ gdasice:
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar"
     required:
         - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.ic.nc"
-        {% for fhr in range(FHOUT_ICE, FHMAX+1, FHOUT_ICE) %}
-        - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f{{ '%03d' % fhr }}.nc"
-        {% endfor %}
+        - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f006.nc"
         - '{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.ice_in'

parm/archive/gdasocean.yaml.j2

@@ -3,11 +3,6 @@ gdasocean:
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar"
     # the FHMIN-hour forecast is skipped when DOIAU="NO"
     required:
-        {% if DOIAU %}
-        - "{{ COMIN_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f{{ '%03d' % FHMIN }}.nc"
-        {% endif %}
-        {% for fhr in range(FHMIN + FHOUT_OCN, FHMAX + 1, FHOUT_OCN) %}
-        - "{{ COMIN_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f{{ '%03d' % fhr }}.nc"
-        {% endfor %}
+        - "{{ COMIN_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f006.nc"
         - '{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.MOM_input'
         - '{{ COMIN_CONF | relpath(ROTDIR) }}/MOM_parameter_doc.all'

parm/archive/gdasocean_analysis.yaml.j2

@@ -16,28 +16,29 @@ gdasocean_analysis:
         - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.analysis.cice_model.res.nc'
         - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc'
         - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_increment.i006.nc'
-        - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ head }}pp_increment.i006.nc'
 
         # static background error
         - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}bkgerr_parametric_stddev.nc'
         - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}bkgerr_parametric_stddev.nc'
 
         # ensemble background error
         {% if NMEM_ENS > 2 %}
-        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ens_weights.nc'
         - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}recentering_error.nc'
-        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}bkgerr_ens_stddev.nc'
-        - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}ens_weights.nc'
-        - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}recentering_error.nc'
-        - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}bkgerr_ens_stddev.nc'
-        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ssh_steric_stddev.nc'
-        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ssh_unbal_stddev.nc'
-        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ssh_total_stddev.nc'
-        - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ssh_explained_variance.nc'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_ensb.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_ensweights.yaml'
         {% endif %}
 
         # runtime configs
-        - '{{ COMIN_CONF | relpath(ROTDIR) }}/*.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_setcorscales.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_chgres.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_diagb.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}gridgen.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_vtscales.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}bmat_fields_metadata.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/var.yaml'
+        # soca_parameters_diffusion_vt, and soca_parameters_diffusion_hz when present
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_parameters_diffusion_??.yaml'
+        - '{{ COMIN_CONF | relpath(ROTDIR) }}/soca_incpostproc.yaml'
     optional:
         # obs space diags
         - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/{{ head }}ocn.*.stats.csv'

parm/archive/gdaswave.yaml.j2

@@ -3,7 +3,11 @@ gdaswave:
     name: "GDASWAVE"
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar"
     required:
-        # TODO explicitly name the wave grid/station files to archive
+        #Save HAFS restart file: 
+        {% set offset_dt = current_cycle | add_to_datetime("+6H" | to_timedelta) %}
+        - "{{ COMIN_WAVE_RESTART | relpath(ROTDIR) }}/{{ offset_dt | to_fv3time }}.restart.ww3.nc"
+
+        # TODO explicitly name the wave grid files to archive
         {% set WAVE_OUT_GRIDS_list = WAVE_OUT_GRIDS.split(' ') %}
         {% for grd in WAVE_OUT_GRIDS_list %}
         {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
@@ -15,4 +19,8 @@ gdaswave:
         {% set file_path = COM_WAVE_GRID_RES_TMPL | replace_tmpl(tmpl_dict) %}
         - "{{ file_path | relpath(ROTDIR) }}/{{ head }}*"
         {% endfor %}
-        - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}*"
+
+        # Wave point output
+        - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull.tar"
+        - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull.tar"
+        - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec.tar.gz"

parm/archive/gdaswave_restart.yaml.j2

@@ -5,4 +5,10 @@ gdaswave_restart:
         # TODO explicitly name the wave restart files to archive
         # NOTE: there are 2 restart files, one is needed to be saved at the same time as restartb is saved
         #  The other which is +6 from the cycle date is needed every cycle for downstream jobs
-        - "{{ COMIN_WAVE_RESTART | relpath(ROTDIR) }}/*"
+        # If IAU is on, grab the beginning of the window.
+        {% if DOIAU %}
+        {% set offset_dt = current_cycle | add_to_datetime("+3H" | to_timedelta) %}
+        {% else %}
+        {% set offset_dt = current_cycle | add_to_datetime("+6H" | to_timedelta) %}
+        {% endif %}
+        - "{{ COMIN_WAVE_RESTART | relpath(ROTDIR) }}/{{ offset_dt | to_fv3time }}.restart.ww3.nc"

parm/archive/gfsa.yaml.j2

@@ -6,7 +6,7 @@ gfsa:
         # Logs
         # TODO explicitly name all logs to include
         {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %}
-        {% if not "gfs_arch_tars.log" in log %}
+        {% if not "tar" in log %}
         - "{{ log }}"
         {% endif %}
         {% endfor %}

parm/archive/master_enkf.yaml.j2

@@ -86,13 +86,6 @@ datasets:
 {% include "enkf_grp.yaml.j2" %}
     {% endfilter %}
 
-    # Archive marine member data
-    {% if DOHYBVAR_OCN %}
-        {% filter indent(width=4) %}
-{% include "enkf_marine_grp.yaml.j2" %}
-        {% endfilter %}
-    {% endif %}
-
     # Determine if restarts should be saved
     {% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %}
 

ush/python/pygfs/task/archive.py

@@ -706,16 +706,16 @@ def _arch_restart(self, arch_dict: Dict) -> bool:
 
         # Restart archiving for gdas RUN
         if run == "gdas":
-            # Always archive gdas ocean restarts (for GEFS)
-            if tar_type == "gdasocean_restart":
+            # TODO: Always archive gdas ocean restarts (for GEFSv13 when project restarts)
+            if tar_type == "gdasocean_restart" and arch_warm_ics:
                 return True
 
             # Archive warm atmosphere and ice increments if requested
             elif (tar_type == "gdas_restarta" or tar_type == "gdasice_restart") and arch_increments:
                 return True
 
             # Archive warm atmosphere ICs if requested
-            elif tar_type == "gdas_restartb" and arch_warm_ics:
+            elif (tar_type == "gdas_restartb" or tar_type == "gdaswave_restart") and arch_warm_ics:
                 return True
 
             else: