Add GEFS C48 support on AWS

env/AWSPW.env

@@ -43,6 +43,17 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${APRUN}"
+
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
@@ -52,6 +63,15 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     export NTHREADS_ECEN=${NTHREADSmax}

parm/config/gefs/config.resources

@@ -41,7 +41,7 @@ case ${machine} in
     ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
-    max_tasks_per_node=40
+    max_tasks_per_node=36
     ;;
   *)
     echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}"

parm/config/gefs/config.resources.AWSPW

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+# AWS-specific job resources
+
+export is_exclusive="True"
+export memory=None
+
+# shellcheck disable=SC2312
+for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
+  unset "${mem_var}"
+done

parm/config/gfs/config.resources.AWSPW

@@ -3,6 +3,7 @@
 # AWS-specific job resources
 
 export is_exclusive="True"
+export memory=None
 
 # shellcheck disable=SC2312
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do

scripts/exgfs_wave_post_pnt.sh

@@ -393,14 +393,14 @@ source "${USHgfs}/preamble.sh"
     then
       export dtspec=3600.
       # Construct the wave_outp_spec (spec) command to run on each buoy in buoy_lst.txt
-      sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}\/spec_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}"
+      sed "s?^\(.*\)$/${escaped_USHgfs}?wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}\/spec_\1.out 2>\&1?" buoy_lst.txt >> "tmpcmdfile.${FH3}"
     fi
 
     if [ "$DOBLL_WAV" = 'YES' ]
     then
       export dtspec=3600.
       # Construct the wave_outp_spec (bull) command to run on each buoy in buoy_lst.txt
-      sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}\/bull_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}"
+      sed "s?^\(.*\)$/${escaped_USHgfs}?wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}\/bull_\1.out 2>\&1?" buoy_lst.txt >> "tmpcmdfile.${FH3}"
     fi
 
     split -n l/1/10  tmpcmdfile.$FH3 > cmdfile.${FH3}.01
@@ -517,13 +517,13 @@ source "${USHgfs}/preamble.sh"
   if [ "$DOSPC_WAV" = 'YES' ]
   then
     # Construct wave_outp_cat (spec) call for each buoy in buoy_lst.txt
-    sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${escaped_CATOUTDIR}\/spec_cat_\1.out 2>\&1/" buoy_lst.txt >> cmdfile.buoy
+    sed "s?^\(.*\)$/${escaped_USHgfs}?wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${escaped_CATOUTDIR}\/spec_cat_\1.out 2>\&1?" buoy_lst.txt >> cmdfile.buoy
   fi
 
   if [ "$DOBLL_WAV" = 'YES' ]
   then
     # Construct wave_outp_cat (bull) call for each buoy in buoy_lst.txt
-    sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${escaped_CATOUTDIR}\/bull_cat_\1.out 2>\&1/" buoy_lst.txt >> cmdfile.buoy
+    sed "s?^\(.*\)$/${escaped_USHgfs}?wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${escaped_CATOUTDIR}\/bull_cat_\1.out 2>\&1?" buoy_lst.txt >> cmdfile.buoy
   fi
 
   if [ ${CFP_MP:-"NO"} = "YES" ]; then

workflow/rocoto/rocoto.py

@@ -146,7 +146,8 @@ def _create_innermost_task(task_dict: Dict[str, Any]) -> List[str]:
     strings.append(f'\t<walltime>{walltime}</walltime>\n')
     strings.append(f'\t<nodes>{nodes}:ppn={ppn}:tpp={threads}</nodes>\n')
     if memory is not None:
-        strings.append(f'\t<memory>{memory}</memory>\n')
+        if memory != 'None':
+            strings.append(f'\t<memory>{memory}</memory>\n')
     if native is not None:
         strings.append(f'\t<native>{native}</native>\n')
     strings.append('\n')

workflow/rocoto/workflow_xml.py

@@ -162,8 +162,7 @@ def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None:
         # AWS need 'SHELL', and 'BASH_ENV' defined, or, the crontab job won't start.
         if os.environ.get('PW_CSP', None) in ['aws', 'azure', 'google']:
             strings.extend([f'SHELL="/bin/bash"',
-                            f'BASH_ENV="/etc/bashrc"'
-                            ])
+                            f'BASH_ENV="/etc/bashrc"'])
         strings.extend([f'{cronintstr} {rocotorunstr}',
                         '#################################################################',
                         ''])