Use of ICs for SFS Beta Version

dev/ci/cases/sfs/C192mx025_S2S_CPC_ICS.yaml

@@ -0,0 +1,24 @@
+experiment:
+  net: sfs
+  mode: forecast-only
+  idate: 1994050100
+  edate: 1994050100
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 192
+  resensatmos: 192
+  resdetocean: 0.25
+  start: 'cold'
+  nens: 10
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'TOPICDIR' | getenv }}/CPC/C192mx025
+  yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/sfs_full_C192mx025_CPC_ICS.yaml
+
+workflow:
+  engine: rocoto
+  rocoto:
+    maxtries: 2
+    cyclethrottle: 3
+    taskthrottle: 25
+    verbosity: 2

dev/ci/cases/sfs/C192mx025_S2S_REPLAY_ICS.yaml

@@ -0,0 +1,24 @@
+experiment:
+  net: sfs
+  mode: forecast-only
+  idate: 1994050100
+  edate: 1994050100
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 192
+  resensatmos: 192
+  resdetocean: 0.25
+  start: 'cold'
+  nens: 10
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'TOPICDIR' | getenv }}/REPLAY/C192mx025
+  yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/sfs_full_C192mx025_REPLAY_ICS.yaml
+
+workflow:
+  engine: rocoto
+  rocoto:
+    maxtries: 2
+    cyclethrottle: 3
+    taskthrottle: 25
+    verbosity: 2

dev/ci/cases/sfs/C96mx100_S2S.yaml → dev/ci/cases/sfs/C96mx100_S2S_CPC_ICS.yaml

@@ -12,8 +12,8 @@ experiment:
   nens: 10
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610
-  yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/sfs_full.yaml
+  icsdir: {{ 'TOPICDIR' | getenv }}/CPC/C96mx100
+  yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/sfs_full_C96mx100_CPC_ICS.yaml
 
 workflow:
   engine: rocoto

dev/ci/cases/sfs/C96mx100_S2S_REPLAY_ICS.yaml

@@ -0,0 +1,24 @@
+experiment:
+  net: sfs
+  mode: forecast-only
+  idate: 1994050100
+  edate: 1994050100
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 96
+  resensatmos: 96
+  resdetocean: 1
+  start: 'cold'
+  nens: 10
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'TOPICDIR' | getenv }}/REPLAY/C96mx100
+  yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/sfs_full_C96mx100_REPLAY_ICS.yaml
+
+workflow:
+  engine: rocoto
+  rocoto:
+    maxtries: 2
+    cyclethrottle: 3
+    taskthrottle: 25
+    verbosity: 2

dev/ci/cases/yamls/sfs_full_C192mx025_CPC_ICS.yaml

@@ -0,0 +1,10 @@
+defaults:
+  !INC {{ HOMEgfs }}/dev/parm/config/sfs/yaml/defaults.yaml
+base:
+  FHMAX_GFS: 8784
+  FCST_BREAKPOINTS: "2400, 4800, 7200"
+ocn:
+  MOM6_INTERP_ICS: "YES"
+nsst:
+  NST_SPINUP: 1
+

dev/ci/cases/yamls/sfs_full_C192mx025_REPLAY_ICS.yaml

@@ -0,0 +1,7 @@
+defaults:
+  !INC {{ HOMEgfs }}/dev/parm/config/sfs/yaml/defaults.yaml
+base:
+  FHMAX_GFS: 8784
+  FCST_BREAKPOINTS: "2400, 4800, 7200"
+ocn:
+  MOM6_INTERP_ICS: "NO"

dev/ci/cases/yamls/sfs_full_C96mx100_CPC_ICS.yaml

@@ -0,0 +1,9 @@
+defaults:
+  !INC {{ HOMEgfs }}/dev/parm/config/sfs/yaml/defaults.yaml
+base:
+  FHMAX_GFS: 8784
+  FCST_BREAKPOINTS: "2400, 4800, 7200"
+ocn:
+  MOM6_INTERP_ICS: "NO"
+nsst:
+  NST_SPINUP: 1

dev/ci/cases/yamls/sfs_full_C96mx100_REPLAY_ICS.yaml

@@ -0,0 +1,7 @@
+defaults:
+  !INC {{ HOMEgfs }}/dev/parm/config/sfs/yaml/defaults.yaml
+base:
+  FHMAX_GFS: 8784
+  FCST_BREAKPOINTS: "2400, 4800, 7200"
+ocn:
+  MOM6_INTERP_ICS: "YES"

dev/parm/config/gefs/config.ufs

@@ -94,8 +94,8 @@ case "${fv3_res}" in
         ;;
     "C96")
         export DELTIM=600
-        export layout_x=2
-        export layout_y=2
+        export layout_x=6
+        export layout_y=8
         export nthreads_fv3=1
         export nthreads_ufs=1
         export xr_cnvcld=".true."  # Pass conv. clouds to Xu-Randall cloud fraction
@@ -113,8 +113,8 @@ case "${fv3_res}" in
         ;;
     "C192")
         export DELTIM=600
-        export layout_x=4
-        export layout_y=6
+        export layout_x=8
+        export layout_y=8
         export nthreads_fv3=1
         export nthreads_ufs=1
         export cdmbgwd="0.23,1.5,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
@@ -364,7 +364,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       FRUNOFF="runoff.daitren.clim.${NX_GLB}x${NY_GLB}.v20180328.nc"
       CHLCLIM="seawifs-clim-1997-2010.${NX_GLB}x${NY_GLB}.v20180328.nc"
       MOM6_RIVER_RUNOFF='True'
-      MOM6_RESTART_SETTING="r"
+      MOM6_RESTART_SETTING=${MOM6_RESTART_SETTING:-'r'}
       eps_imesh="1.0e-1"
       case ${RUN} in
         gfs|gefs|sfs)
@@ -430,7 +430,11 @@ if [[ "${skip_cice6}" == "false" ]]; then
       cice6_processor_shape="slenderX2"
       ;;
     "025")
-      ntasks_cice6=120
+      if [[ "${fv3_res}" ==  "C192" ]]; then
+        ntasks_cice6=90
+      else
+        ntasks_cice6=120
+      fi
       cice6_processor_shape="slenderX2"
       ;;
     *)

dev/parm/config/gfs/config.globus

@@ -17,7 +17,7 @@ export STAGE_DIR="${DATAROOT}/archive_rotdir/${PSLOT}"
 export SERVER_NAME="mercury"
 
 # Username on the server
-export SERVER_USERNAME=""
+export SERVER_USERNAME="Neil.Barton"
 # General delivery location on Mercury (staging area for data on server)
 export SERVER_HOME="/collab2/data/${SERVER_USERNAME}/${PSLOT}"
 

dev/parm/config/gfs/config.ocn.j2

@@ -24,7 +24,7 @@ else
   export ODA_INCUPD_NHOURS="3.0"
 fi
 
-MOM6_INTERP_ICS="{{ MOM6_INTERP_ICS }}"
+export MOM6_INTERP_ICS="{{ MOM6_INTERP_ICS }}"
 if [[ "${MOM6_INTERP_ICS}" == "YES" ]]; then
     export MOM6_RESTART_SETTING='n'
     export MOM6_WARMSTART_FILE="MOM.res.nc"

dev/parm/config/sfs/config.base.j2

@@ -157,10 +157,10 @@ case "${CASE}" in
     "C48")
         export waveGRD='glo_500'
         ;;
-    "C96" | "C192")
+    "C96")
         export waveGRD='glo_100'
         ;;
-    "C384")
+    "C192" | "C384")
         export waveGRD='glo_025'
         ;;
     "C768" | "C1152")
@@ -328,7 +328,8 @@ export DO_FIT2OBS="NO"      # Run fit to observations package
 
 # Archiving options
 export HPSSARCH="{{ HPSSARCH }}"        # save data to HPSS archive
-export LOCALARCH="{{ LOCALARCH }}"        # save data to local archive
+export LOCALARCH="{{ LOCALARCH }}"      # save data to local archive
+export ARCHCOM_TO="{{ ARCHCOM_TO }}"    # where to save data
 if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then
    echo "Both HPSS and local archiving selected.  Please choose one or the other."
    exit 3

dev/parm/config/sfs/config.efcs

@@ -27,7 +27,7 @@ source "${EXPDIR}/config.ufs" ${string}
 source "${EXPDIR}/config.resources" efcs
 
 # Stochastic physics parameters (only for ensemble forecasts)
-if [[ "${CASE}" == "C96" ]] ; then
+if [[ "${TYPE}" == "hydro" ]] ; then
     export DO_SKEB="NO" # SKEB turned off for C96
     export SKEB=-999.
 else

dev/parm/config/sfs/config.fcst.j2

@@ -254,6 +254,8 @@ export FSICS="0"
 # Write more variables to output
 if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
   export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_replay"
+elif [[ "${RUN}" == "sfs" ]]; then
+  export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_sfs"
 else
   export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table"
 fi

dev/parm/config/sfs/config.nsst.j2

@@ -0,0 +1,32 @@
+#! /usr/bin/env bash
+
+########## config.nsst ##########
+# NSST specific
+
+echo "BEGIN: config.nsst"
+
+# NSST parameters contained within nstf_name
+
+# nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled
+export NST_MODEL=2
+
+# nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON,
+export NST_SPINUP="{{ NST_SPINUP }}"
+
+# nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON
+export NST_RESV=0
+
+# nstf_name(4,5) : ZSEA1, ZSEA2 the two depths to apply vertical average (bias correction)
+export ZSEA1=0
+export ZSEA2=0
+
+export NST_GSI=3          # default 0: No NST info at all;
+                          #         1: Input NST info but not used in GSI;
+                          #         2: Input NST info, used in CRTM simulation, no Tr analysis
+                          #         3: Input NST info, used in both CRTM simulation and Tr analysis
+export NSTINFO=0          # number of elements added in obs. data array (default = 0)
+if [[ ${NST_GSI} -gt 0 ]]; then 
+  export NSTINFO=4
+fi
+
+echo "END: config.nsst"

dev/parm/config/sfs/config.resources

@@ -316,6 +316,15 @@ case ${step} in
     export memory="4096M"
     ;;
 
+  "globus")
+    export walltime="08:00:00"
+    export ntasks=1
+    export tasks_per_node=1
+    export threads_per_task=1
+    export memory="4096M"
+    ;;
+
+
   "cleanup")
     export walltime="00:30:00"
     export ntasks=1

dev/parm/config/sfs/yaml/defaults.yaml

@@ -19,8 +19,8 @@ base:
   FHOUT_ICE_GFS: 24
   HPSSARCH: "NO"
   LOCALARCH: "NO"
-  USE_OCN_ENS_PERTURB_FILES: "NO"
-  USE_ATM_ENS_PERTURB_FILES: "NO"
+  USE_OCN_ENS_PERTURB_FILES: "YES"
+  USE_ATM_ENS_PERTURB_FILES: "YES"
   DO_TEST_MODE: "NO"
 fcst:
   CHECK_LAND_RESTART_OROG: "YES"
@@ -30,6 +30,9 @@ fcst:
   MONO: "mono"
 ocn:
   MOM6_INTERP_ICS: "NO"
+nsst:
+  NST_SPINUP: 0
+
 # config.aero has just a system-specific path to add.
 # This is handled by the setup_expt.py, but it has to be told to write to it.
 aero: {}

dev/workflow/applications/sfs.py

@@ -93,6 +93,8 @@ def _get_app_configs(self, run):
 
         if options['do_archcom']:
             configs += ['arch_tars']
+            if options['do_globusarch']:
+                configs += ['globus']
 
         configs += ['arch_vrfy', 'cleanup']
 
@@ -164,7 +166,11 @@ def get_task_names(self):
         if options['do_extractvars']:
             tasks += ['extractvars']
 
-        # TODO: Add archive
+        if options['do_archcom']:
+            tasks += ['arch_tars']
+            if options['do_globusarch']:
+                tasks += ['globus']
+
         tasks += ['cleanup']
 
         return {f"{self.run}": tasks}

dev/workflow/hosts/hercules.yaml

@@ -5,7 +5,7 @@ BASE_DATA: '/work2/noaa/global/role-global/data'
 BASE_IC: '/work2/noaa/global/role-global/data/ICSDIR'
 AERO_INPUTS_DIR: '/work2/noaa/global/role-global/data/GEFS_ExtData/20250310'
 PACKAGEROOT: '/work2/noaa/global/role-global/nwpara'
-HOMEDIR: '/work2/noaa/global/${USER}'
+HOMEDIR: '/work2/noaa/stmp/${USER}'
 STMP: '/work2/noaa/stmp/${USER}/${machine^^}'
 PTMP: '/work2/noaa/stmp/${USER}/${machine^^}'
 NOSCRUB: '${HOMEDIR}'

dev/workflow/rocoto/sfs_tasks.py

@@ -626,6 +626,30 @@ def arch_tars(self):
         task = rocoto.create_task(task_dict)
 
         return task
+
+    # Globus transfer for HPSS archiving 
+    def globus(self):
+        deps = []
+        dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+
+        resources = self.get_resource('globus')
+        task_name = f'{self.run}_globus_arch'
+        task_dict = {'task_name': task_name,
+                     'resources': resources,
+                     'dependency': dependencies,
+                     'envars': self.envars,
+                     'cycledef': self.run,
+                     'command': f'{self.HOMEgfs}/dev/jobs/globus_arch.sh',
+                     'job_name': f'{self.pslot}_{task_name}_@H',
+                     'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log',
+                     'maxtries': '&MAXTRIES;'
+                     }
+
+        task = rocoto.create_task(task_dict)
+
+        return task
 
     def cleanup(self):
         deps = []
@@ -652,6 +676,13 @@ def cleanup(self):
         if self.options['do_extractvars']:
             dep_dict = {'type': 'metatask', 'name': f'{self.run}_extractvars'}
             deps.append(rocoto.add_dependency(dep_dict))
+        if self.options['do_archcom']:
+            if self.options['do_globusarch']:
+                dep_dict = {'type': 'task', 'name': f'{self.run}_globus_arch'}
+            else:
+                dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'}
+            deps.append(rocoto.add_dependency(dep_dict))
+            dependencies = rocoto.create_dependency(dep=deps)
         dependencies = rocoto.create_dependency(dep=deps, dep_condition='and')
         resources = self.get_resource('cleanup')
         task_name = f'{self.run}_cleanup'

jobs/JGLOBAL_STAGE_IC

@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "stage_ic" -c "base stage_ic ocn"
 
 # Execute staging
 "${SCRgfs}/exglobal_stage_ic.py"