diff --git a/jobs/JFV3CAM_NEST_ENVIR b/jobs/JFV3CAM_NEST_ENVIR index 0fa76f277..7ec51a076 100755 --- a/jobs/JFV3CAM_NEST_ENVIR +++ b/jobs/JFV3CAM_NEST_ENVIR @@ -16,7 +16,7 @@ export CRES=768 #-- FV3 equivalent to 13-km global resolution export CASE=C${CRES} export gtype=nest # grid type = uniform, stretch, nest, or regional -export HOMEfv3=/gpfs/dell2/emc/modeling/noscrub/$USER/fv3cam_workflow +export HOMEfv3=/gpfs/dell2/emc/modeling/noscrub/$USER/regional_workflow export PARMfv3=$HOMEfv3/parm export EXECfv3=$HOMEfv3/exec export USHfv3=$HOMEfv3/ush diff --git a/jobs/JFV3CAM_SAR_CHGRES_FCSTBNDY b/jobs/JFV3CAM_SAR_CHGRES_FCSTBNDY index 3284dab12..27bd0fe61 100755 --- a/jobs/JFV3CAM_SAR_CHGRES_FCSTBNDY +++ b/jobs/JFV3CAM_SAR_CHGRES_FCSTBNDY @@ -53,7 +53,7 @@ mkdir -p $COMOUT export NWGES=${COMROOT}/${NET}/${envir}/${RUN}.${PDY}/${cyc} mkdir -p $NWGES -export INPdir=${COMOUT}/anl.${tmmark} +export INPdir=${COMOUT}/anl.${dom}.${tmmark} mkdir -p ${INPdir} ##################################### diff --git a/jobs/JFV3CAM_SAR_CHGRES_FIRSTGUESS b/jobs/JFV3CAM_SAR_CHGRES_FIRSTGUESS index 7b4939dfc..337652f01 100755 --- a/jobs/JFV3CAM_SAR_CHGRES_FIRSTGUESS +++ b/jobs/JFV3CAM_SAR_CHGRES_FIRSTGUESS @@ -53,7 +53,7 @@ mkdir -p $NWGES if [ $tmmark = tm12 ] ; then # SAR-DA export INPdir=$COMOUT/gfsanl.$tmmark elif [ $tmmark = tm00 ] ; then # SAR without DA - export INPdir=$COMOUT/anl.$tmmark + export INPdir=$COMOUT/anl.${dom}.$tmmark fi mkdir -p $INPdir diff --git a/jobs/JFV3CAM_SAR_ENVIR b/jobs/JFV3CAM_SAR_ENVIR index cf9116ebf..40dad8176 100755 --- a/jobs/JFV3CAM_SAR_ENVIR +++ b/jobs/JFV3CAM_SAR_ENVIR @@ -16,13 +16,13 @@ export CRES=768 #-- FV3 equivalent to 13-km global resolution export CASE=C${CRES} export gtype=regional # grid type = uniform, stretch, nest, or regional -export HOMEfv3=/gpfs/dell2/emc/modeling/noscrub/$USER/fv3cam_workflow +export HOMEfv3=/gpfs/dell2/emc/modeling/noscrub/$USER/regional_workflow export PARMfv3=$HOMEfv3/parm export EXECfv3=$HOMEfv3/exec export USHfv3=$HOMEfv3/ush export FIXfv3=$HOMEfv3/fix -export FIXsar=$FIXfv3/fix_sar +export FIXsar=$FIXfv3/fix_sar/${dom} export FIXam=$FIXfv3/fix_am export FIXco2=$FIXam/fix_co2_proj diff --git a/jobs/JFV3CAM_SAR_FCST b/jobs/JFV3CAM_SAR_FCST index 2d7592b38..432db602c 100755 --- a/jobs/JFV3CAM_SAR_FCST +++ b/jobs/JFV3CAM_SAR_FCST @@ -61,7 +61,7 @@ export CYCfcst=`echo $CYCLEfcst | cut -c 9-10` export GUESSdir=$COMOUT/guess.${tmmark_next} mkdir -p $GUESSdir -export ANLdir=$COMOUT/anl.${tmmark_next} +export ANLdir=$COMOUT/anl.${dom}.${tmmark_next} mkdir -p $ANLdir ##################################### diff --git a/jobs/JFV3CAM_SAR_POST b/jobs/JFV3CAM_SAR_POST index e8139a60b..109e1fbc2 100755 --- a/jobs/JFV3CAM_SAR_POST +++ b/jobs/JFV3CAM_SAR_POST @@ -26,7 +26,7 @@ export WGRIB2=${EXECfv3}/wgrib2new ##################################### # Working directories ##################################### -export INPUT_DATA=${INPUT_DATA:-${mainroot}/${tmpdir}/${USER}/tmpnwprd/forecast_${tmmark}_${PDY}_${cyc}} +export INPUT_DATA=${INPUT_DATA:-${mainroot}/${tmpdir}/${USER}/tmpnwprd/forecast_${tmmark}_${dom}_${PDY}_${cyc}} export DATA=${DATA:-${mainroot}/${tmpdir}/${USER}/tmpnwprd/${job}_${cyc}} if [ -d $DATA ]; then rm -rf $DATA diff --git a/jobs/JFV3CAM_SAR_POST_GOES b/jobs/JFV3CAM_SAR_POST_GOES index d804d3e63..6f03dc245 100755 --- a/jobs/JFV3CAM_SAR_POST_GOES +++ b/jobs/JFV3CAM_SAR_POST_GOES @@ -26,7 +26,7 @@ export WGRIB2=${EXECfv3}/wgrib2new ##################################### # Working directories ##################################### -export INPUT_DATA=${INPUT_DATA:-${mainroot}/${tmpdir}/${USER}/tmpnwprd/forecast_${tmmark}_${PDY}_${cyc}} +export INPUT_DATA=${INPUT_DATA:-${mainroot}/${tmpdir}/${USER}/tmpnwprd/forecast_${tmmark}_${dom}_${PDY}_${cyc}} export DATA=${DATA:-${mainroot}/${tmpdir}/${USER}/tmpnwprd/${job}_${cyc}} if [ -d $DATA ]; then rm -rf $DATA diff --git a/jobs/JFV3CAM_SAR_SENDINIBC b/jobs/JFV3CAM_SAR_SENDINIBC index 5ff5c5481..76998caf1 100755 --- a/jobs/JFV3CAM_SAR_SENDINIBC +++ b/jobs/JFV3CAM_SAR_SENDINIBC @@ -3,7 +3,7 @@ set -x . $HOMEfv3/jobs/JFV3CAM_SAR_ENVIR -export DATA=${mainroot}/${comdir}/${USER}/com/${NET}/${envir}/${RUN}.${PDY}/${cyc}/anl.tm00 +export DATA=${mainroot}/${comdir}/${USER}/com/${NET}/${envir}/${RUN}.${PDY}/${cyc}/anl.${dom}.tm00 cd $DATA export pid=$$ diff --git a/jobs/launch.ksh b/jobs/launch.ksh index 3904d1c6d..a44439fde 100755 --- a/jobs/launch.ksh +++ b/jobs/launch.ksh @@ -6,7 +6,7 @@ COMMAND=$1 # load modulefile and set up the environment for job runnning ############################################################# -MODULEFILES=${MODULEFILES:-/gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/modulefiles} +MODULEFILES=${MODULEFILES:-/gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/modulefiles} if [ "$machine" = "DELL" ] ; then . /usrx/local/prod/lmod/lmod/init/sh diff --git a/parm/input_sar.nml_template b/parm/input_sar.nml_template new file mode 100755 index 000000000..860a23e98 --- /dev/null +++ b/parm/input_sar.nml_template @@ -0,0 +1,284 @@ + &amip_interp_nml + interp_oi_sst = .true. + use_ncep_sst = .true. + use_ncep_ice = .false. + no_anom_sst = .false. + data_set = 'reynolds_oi', + date_out_of_range = 'climo', +/ + + &atmos_model_nml + blocksize = 24 + chksum_debug = .false. + dycore_only = .false. + fdiag = 1 + avg_max_length = 3600. +/ + +&diag_manager_nml + prepend_date = .false. +/ + + &fms_io_nml + checksum_required = .false. + max_files_r = 100, + max_files_w = 100, +/ + + &fms_nml + clock_grain = 'ROUTINE', + domains_stack_size = 1800200, + print_memory_usage = .false. +/ + + &fv_grid_nml + grid_file = 'INPUT/grid_spec.nc' +/ + + &fv_core_nml + layout = _TASK_X_,_TASK_Y_ + io_layout = 1,1 + npx = _NX_ + npy = _NY_ + ntiles = 1, + npz = 64 +! grid_type = -1 + make_nh = .T. + fv_debug = .F. + range_warn = .F. + reset_eta = .F. + n_sponge = 24 + nudge_qv = .F. + tau = 5. + rf_cutoff = 20.e2 + d2_bg_k1 = 0.20 + d2_bg_k2 = 0.04 + kord_tm = -11 + kord_mt = 11 + kord_wz = 11 + kord_tr = 11 + hydrostatic = .F. + phys_hydrostatic = .F. + use_hydro_pressure = .F. + beta = 0. + a_imp = 1. + p_fac = 0.1 + k_split = 6 + n_split = 6 + nwat = 6 + na_init = 1 + d_ext = 0.0 + dnats = 1 + fv_sg_adj = 300 + d2_bg = 0. + nord = 2 + dddmp = 0.1 + d4_bg = 0.15 + vtdm4 = 0.075 + delt_max = 0.008 + ke_bg = 0. + do_vort_damp = .true. + external_ic = .T. + external_eta = .T. + gfs_phil = .false. + nggps_ic = .T. + mountain = .F. + ncep_ic = .F. + d_con = 1.0 + hord_mt = 6 + hord_vt = 6 + hord_tm = 6 + hord_dp = -6 + hord_tr = 8 + adjust_dry_mass = .F. + consv_te = 0. + do_sat_adj = .T. + consv_am = .F. + fill = .T. + dwind_2d = .F. + print_freq = 6 + warm_start = .F. + no_dycore = .false. + z_tracer = .T. + read_increment = .F. + res_latlon_dynamics = "fv3_increment.nc" + + do_schmidt = .true. + target_lat = _TARG_LAT_ + target_lon = _TARG_LON_ + stretch_fac = 1.5 + regional = .true. + bc_update_interval = 3 + agrid_vel_rst = .false. + + full_zs_filter = .F. !unreleased feature + + nord_zs_filter = 4 + n_zs_filter = 0 ! safety +/ + +&surf_map_nml + zero_ocean = .F. + cd4 = 0.12 + cd2 = -1 + n_del2_strong = 0 + n_del2_weak = 2 + n_del4 = 1 + max_slope = 0.4 + peak_fac = 1. +/ + + &external_ic_nml + filtered_terrain = .true. + levp = 65 + gfs_dwinds = .true. + checker_tr = .F. + nt_checker = 0 +/ + + &gfs_physics_nml + fhzero = 1. + ldiag3d = .false. + lradar = .true. + avg_max_length = 3600. + h2o_phys = .true. + fhcyc = 24. + nst_anl = .true. + use_ufo = .true. + pre_rad = .false. + ncld = 5 + imp_physics = 11 + pdfcld = .false. + fhswr = 3600. + fhlwr = 3600. + ialb = 1 + iems = 1 + IAER = 111 + ico2 = 2 + isubc_sw = 2 + isubc_lw = 2 + isol = 2 + lwhtr = .true. + swhtr = .true. + cnvgwd = .false. + cal_pre = .false. + redrag = .true. + dspheat = .true. + hybedmf = .true. + random_clds = .false. + trans_trac = .true. + cnvcld = .false. + imfshalcnv = 2 + imfdeepcnv = 2 + cdmbgwd = 3.5, 0.01 ! NCEP default + prslrd0 = 0. + ivegsrc = 1 + isot = 1 + debug = .false. + nstf_name = 2,0,0,0,0 + iau_delthrs = 6 + iaufhrs = 30 + iau_inc_files = '' + shal_cnv = .false. !Shallow convection + do_deep = .false. + lgfdlmprad = .true. + effr_in = .true. +/ + + &gfdl_cloud_microphysics_nml + sedi_transport = .false. + do_sedi_heat = .false. + rad_snow = .true. + rad_graupel = .true. + rad_rain = .true. + const_vi = .F. + const_vs = .F. + const_vg = .F. + const_vr = .F. + vi_max = 1. + vs_max = 2. + vg_max = 12. + vr_max = 12. + qi_lim = 1. + prog_ccn = .false. + do_qa = .true. + fast_sat_adj = .true. + tau_l2v = 180. + tau_v2l = 90. + tau_g2v = 900. + rthresh = 10.e-6 ! This is a key parameter for cloud water + dw_land = 0.16 + dw_ocean = 0.10 + ql_gen = 1.0e-3 + ql_mlt = 1.0e-3 + qi0_crt = 8.0E-5 + qs0_crt = 1.0e-3 + tau_i2s = 1000. + c_psaci = 0.05 + c_pgacs = 0.01 + rh_inc = 0.30 + rh_inr = 0.30 + rh_ins = 0.30 + ccn_l = 300. + ccn_o = 100. + c_paut = 0.5 + c_cracw = 0.8 + use_ppm = .false. + use_ccn = .true. + mono_prof = .true. + z_slope_liq = .true. + z_slope_ice = .true. + de_ice = .false. + fix_negative = .true. + icloud_f = 1 + mp_time = 90. +/ + &interpolator_nml + interp_method = 'conserve_great_circle' +/ + +&namsfc + FNALBC2 = "C768.facsf.tileX.nc", + FNALBC = "C768.snowfree_albedo.tileX.nc", + FNTG3C = "C768.substrate_temperature.tileX.nc", + FNVEGC = "C768.vegetation_greenness.tileX.nc", + FNVETC = "C768.vegetation_type.tileX.nc", + FNSOTC = "C768.soil_type.tileX.nc", + FNVMNC = "C768.vegetation_greenness.tileX.nc", + FNVMXC = "C768.vegetation_greenness.tileX.nc", + FNSLPC = "C768.slope_type.tileX.nc", + FNABSC = "C768.maximum_snow_albedo.tileX.nc", + FNGLAC = "global_glacier.2x2.grb", + FNMXIC = "global_maxice.2x2.grb", + FNTSFC = "RTGSST.1982.2012.monthly.clim.grb", + FNSNOC = "global_snoclim.1.875.grb", + FNZORC = "igbp" + FNAISC = "CFSR.SEAICE.1982.2012.monthly.clim.grb", + FNSMCC = "global_soilmgldas.t1534.3072.1536.grb", + FNMSKH = "seaice_newland.grb", + FNTSFA = "", + FNACNA = "", + FNSNOA = "", + LDEBUG =.true., + FSMCL(2) = 99999 + FSMCL(3) = 99999 + FSMCL(4) = 99999 + FTSFS = 90 + FAISS = 99999 + FSNOL = 99999 + FSICL = 99999 + FTSFL = 99999, + FAISL = 99999, + FVETL = 99999, + FSOTL = 99999, + FvmnL = 99999, + FvmxL = 99999, + FSLPL = 99999, + FABSL = 99999, + FSNOS = 99999, + FSICS = 99999, +/ +&nam_stochy +/ +&nam_sfcperts +/ diff --git a/parm/model_configure_sar.tmp_template b/parm/model_configure_sar.tmp_template new file mode 100755 index 000000000..fda6db15c --- /dev/null +++ b/parm/model_configure_sar.tmp_template @@ -0,0 +1,58 @@ +print_esmf: .false. +total_member: 1 +PE_MEMBER01: NTASKS +start_year: YR +start_month: MN +start_day: DY +start_hour: H_R +start_minute: 0 +start_second: 0 +nhours_fcst: NHRS +RUN_CONTINUE: .false. +ENS_SPS: .false. +dt_atmos: 90 +cpl: .false. +calendar: 'julian' +memuse_verbose: .false. +atmos_nthreads: NTHRD +use_hyper_thread: .false. +ncores_per_node: NCNODE +debug_affinity: .true. +restart_interval: 12 +output_1st_tstep_rst: .false. + +quilting: .true. +write_groups: _WG_ +write_tasks_per_group: _WTPG_ +num_files: 2 +filename_base: 'dyn''phy' +output_file: 'netcdf' +write_nemsioflip: .false. +write_fsyncflag: .false. + +output_grid: 'rotated_latlon' +cen_lon: _CEN_LON_ # central longitude +cen_lat: _CEN_LAT_ # central latitude +lon1: _LON1_ # longitude of lower-left point in rotated coordinate system (in degrees) +lat1: _LAT1_ # latitude of lower-left ... +lon2: _LON2_ # longitude of upper-right ... +lat2: _LAT2_ # latitude of upper-right ... +dlon: _DLON_ +dlat: _DLAT_ + +#output_grid: 'lambert_conformal' +#cen_lon: -97.5 # central longitude +#cen_lat: 38.5 # central latitude +#stdlat1: 38.5 +#stdlat2: 38.5 +#nx: 1799 # number of points along x-axis. +#ny: 1059 # number of points along y-axis. +#lon1: -122.719528 # longitude of lower-left point (in degrees) +#lat1: 21.138123 # latitude of lower-left point (in degrees) +#dx: 3000.0 # x-direction grid length +#dy: 3000.0 # y-direction grid length + +nfhout: 1 +nfhmax_hf: 60 +nfhout_hf: 1 +nsout: -1 diff --git a/rocoto/build_rocoto_and_namelist.sh b/rocoto/build_rocoto_and_namelist.sh new file mode 100755 index 000000000..35cae104a --- /dev/null +++ b/rocoto/build_rocoto_and_namelist.sh @@ -0,0 +1,231 @@ +#! /bin/sh + +dom=${1} + +username=$USER + +# bcnodes = 21 if want to run all simultaneously +# bcnodes = 11 should get in queue faster and still run pretty quickly + +if [ $dom == 'conus' ] +then + +## rocoto items + +fcstnodes=76 +bcnodes=11 +postnodes=2 +goespostnodes=15 +goespostthrottle=3 +sh=00 +eh=12 + +## namelist items + +task_layout_x=16 +task_layout_y=48 +npx=1921 +npy=1297 +target_lat=38.5 +target_lon=-97.5 + +## model config items + +write_groups=3 +write_tasks_per_group=48 +cen_lon=$target_lon +cen_lat=$target_lat +lon1=-25.0 +lat1=-15.0 +lon2=25.0 +lat2=15.0 +dlon=0.02 +dlat=0.02 + + + +elif [ $dom == 'ak' ] +then + +## rocoto items + +fcstnodes=68 +bcnodes=11 +postnodes=2 +goespostnodes=5 +goespostthrottle=6 +sh=06 +eh=18 + +## namelist items + +task_layout_x=16 +task_layout_y=48 +npx=1345 +npy=1153 +target_lat=61.0 +target_lon=-153.0 + +## model config items + +write_groups=2 +write_tasks_per_group=24 +cen_lon=$target_lon +cen_lat=$target_lat +lon1=-18.0 +lat1=-14.79 +lon2=18.0 +lat2=14.79 +dlon=0.03 +dlat=0.03 + +elif [ $dom == 'hi' ] +then + +## rocoto items + +fcstnodes=7 +bcnodes=11 +postnodes=1 +goespostnodes=1 +goespostthrottle=9 +sh=00 +eh=12 + +## namelist items + +task_layout_x=6 +task_layout_y=12 +npx=433 +npy=361 +target_lat=20.0 +target_lon=-157.0 + +## model config items + +write_groups=1 +write_tasks_per_group=12 +cen_lon=$target_lon +cen_lat=$target_lat +lon1=-5.4 +lat1=-4.3 +lon2=5.4 +lat2=4.3 +dlon=0.025 +dlat=0.025 + +elif [ $dom == 'pr' ] +then + +## rocoto items + +fcstnodes=10 +bcnodes=11 +postnodes=1 +goespostnodes=1 +goespostthrottle=9 +sh=06 +eh=18 + +## namelist items + +task_layout_x=6 +task_layout_y=18 +npx=577 +npy=433 +target_lat=18.0 +target_lon=-69.0 + +## model config items + +write_groups=1 +write_tasks_per_group=12 +cen_lon=$target_lon +cen_lat=$target_lat +lon1=-7.8 +lat1=-5.2 +lon2=7.8 +lat2=5.2 +dlon=0.025 +dlat=0.025 + +elif [ $dom == 'guam' ] +then + +## rocoto items + +fcstnodes=7 +bcnodes=11 +ername is Matthew.Pyle +ostnodes=1 +goespostnodes=1 +goespostthrottle=9 +sh=00 +eh=12 + + +## namelist items + +task_layout_x=6 +task_layout_y=12 +npx=433 +npy=361 +target_lat=15.0 +target_lon=146.0 + +## model config items + +write_groups=1 +write_tasks_per_group=12 +cen_lon=$target_lon +cen_lat=$target_lat +lon1=-5.6 +lat1=-4.8 +lon2=5.6 +lat2=4.8 +dlon=0.025 +dlat=0.025 + +else + +echo "Bad entry...domain ${dom} not recognized" +echo "Use one of conus, ak, hi, pr, or guam" + +exit + +fi + +echo username is $username + +cat drive_fv3sar_template.xml \ + | sed s:_USER_:${username}:g \ + | sed s:_DOMAIN_:${dom}:g \ + | sed s:_SH_:${sh}:g \ + | sed s:_EH_:${eh}:g \ + | sed s:_BCNODES_:${bcnodes}:g \ + | sed s:_FCSTNODES_:${fcstnodes}:g \ + | sed s:_POSTNODES_:${postnodes}:g \ + | sed s:_GOESPOSTNODES_:${goespostnodes}:g \ + | sed s:_GOESPOSTTHROTTLE_:${goespostthrottle}:g > drive_fv3sar_${dom}.xml + + +cat ../parm/input_sar.nml_template \ + | sed s:_TASK_X_:${task_layout_x}:g \ + | sed s:_TASK_Y_:${task_layout_y}:g \ + | sed s:_NX_:${npx}:g \ + | sed s:_NY_:${npy}:g \ + | sed s:_TARG_LAT_:${target_lat}:g \ + | sed s:_TARG_LON_:${target_lon}:g > ../parm/input_sar.nml_${dom}_test + + +cat ../parm/model_configure_sar.tmp_template \ + | sed s:_WG_:${write_groups}:g \ + | sed s:_WTPG_:${write_tasks_per_group}:g \ + | sed s:_CEN_LAT_:${cen_lat}:g \ + | sed s:_CEN_LON_:${cen_lon}:g \ + | sed s:_LON1_:${lon1}:g \ + | sed s:_LAT1_:${lat1}:g \ + | sed s:_LON2_:${lon2}:g \ + | sed s:_LAT2_:${lat2}:g \ + | sed s:_DLON_:${dlon}:g \ + | sed s:_DLAT_:${dlat}:g > ../parm/model_configure_sar.tmp_${dom}_test diff --git a/rocoto/drive_fv3nest.xml b/rocoto/drive_fv3nest.xml index 3f36b7383..5afa24035 100755 --- a/rocoto/drive_fv3nest.xml +++ b/rocoto/drive_fv3nest.xml @@ -16,7 +16,7 @@ - + diff --git a/rocoto/drive_fv3nest_retro.xml b/rocoto/drive_fv3nest_retro.xml index 8f04c7088..9a83e3134 100755 --- a/rocoto/drive_fv3nest_retro.xml +++ b/rocoto/drive_fv3nest_retro.xml @@ -16,7 +16,7 @@ - + diff --git a/rocoto/drive_fv3sar.xml b/rocoto/drive_fv3sar.xml index 1e4c4d115..2f4991e21 100755 --- a/rocoto/drive_fv3sar.xml +++ b/rocoto/drive_fv3sar.xml @@ -16,8 +16,9 @@ - + + @@ -29,7 +30,7 @@ 00:10:00'> 00:15:00-R affinity[core]'> -00:12:00-R affinity[core]'> +00:15:00-R affinity[core]'> 01:40:00-R affinity[core\(2\):distribute=balance]'> 00:20:00-R affinity[core]'> 00:10:00-R affinity[core]'> @@ -68,6 +69,10 @@ HOMEfv3 &HOMEfv3; + + dom + &domain; + job chgres_firstguess @@ -120,7 +125,7 @@ &RESERVATION; &CHGRES_BC_RESOURCES; - 20:ppn=1 + 11:ppn=1 &JOBS;/launch.ksh &JOBS;/JFV3CAM_SAR_CHGRES_FCSTBNDY chgres_fcstbndy_@H &OUTDIR;/logfiles_sar/chgres_fcstbndy_@H.log @@ -129,6 +134,10 @@ HOMEfv3 &HOMEfv3; + + dom + &domain; + job chgres_fcstbndy @@ -184,16 +193,20 @@ &FCST_RESOURCES; 76:ppn=12 &JOBS;/launch.ksh &JOBS;/JFV3CAM_SAR_FCST - forecast_tm00_@H + forecast_tm00_&domain;_@H &OUTDIR;/logfiles_sar/forecast_tm00_@H.log HOMEfv3 &HOMEfv3; + + dom + &domain; + job - forecast_tm00_@Y@m@d + forecast_tm00_&domain;_@Y@m@d machine @@ -246,6 +259,10 @@ HOMEfv3 &HOMEfv3; + + dom + &domain; + job postf#fhr# @@ -301,6 +318,10 @@ HOMEfv3 &HOMEfv3; + + dom + &domain; + job postgoesf#fhr# diff --git a/rocoto/drive_fv3sar_da.xml b/rocoto/drive_fv3sar_da.xml index 62ada9472..bb29e434f 100755 --- a/rocoto/drive_fv3sar_da.xml +++ b/rocoto/drive_fv3sar_da.xml @@ -16,7 +16,7 @@ - + diff --git a/rocoto/drive_fv3sar_da_retro.xml b/rocoto/drive_fv3sar_da_retro.xml index 7b39c0c0f..95e479323 100755 --- a/rocoto/drive_fv3sar_da_retro.xml +++ b/rocoto/drive_fv3sar_da_retro.xml @@ -16,7 +16,7 @@ - + diff --git a/rocoto/drive_fv3sar_retro.xml b/rocoto/drive_fv3sar_retro.xml index 2c8c941f5..c925eeb6b 100755 --- a/rocoto/drive_fv3sar_retro.xml +++ b/rocoto/drive_fv3sar_retro.xml @@ -16,7 +16,7 @@ - + diff --git a/rocoto/drive_fv3sar_template.xml b/rocoto/drive_fv3sar_template.xml new file mode 100755 index 000000000..207a225cf --- /dev/null +++ b/rocoto/drive_fv3sar_template.xml @@ -0,0 +1,639 @@ + + + + + + + + + + + + + + + + + + + + + + + + +dev&ACCOUNT;'> +dev_transfer&ACCOUNT;'> + +00:10:00'> +00:15:00-R affinity[core]'> +00:12:00-R affinity[core]'> +01:40:00-R affinity[core\(2\):distribute=balance]'> +00:20:00-R affinity[core]'> +00:10:00-R affinity[core]'> +00:30:005G-R affinity[core]'> + +]> + + + + + + + + + &STARTYEAR;&STARTMONTH;&STARTDAY;&STARTHOUR;00 &ENDYEAR;&ENDMONTH;&ENDDAY;&ENDHOUR;00 12:00:00 + + + &OUTDIR;/logfiles_sar/workflow_regional_&domain;_@Y@m@d@H.log + + + + + + + + + + &RESERVATION; + &CHGRES_IC_RESOURCES; + 1 + &JOBS;/launch.ksh &JOBS;/JFV3CAM_SAR_CHGRES_FIRSTGUESS + chgres_firstguess_&domain;_@H + &OUTDIR;/logfiles_sar/chgres_firstguess_&domain;_@H.log + + + dom + &domain; + + + + HOMEfv3 + &HOMEfv3; + + + job + chgres_firstguess_&domain; + + + machine + &machine; + + + USER + &USER; + + + CDATE + @Y@m@d@H + + + PDY + @Y@m@d + + + cyc + @H + + + tmmark + tm00 + + + + + + &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf000.nemsio + &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio + &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.sfcanl.nemsio + + + &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf000.nemsio + &COMgfs2;/gfs.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio + &COMgfs2;/gfs.@Y@m@d/@H/gfs.t@Hz.sfcanl.nemsio + + + + + + + + + + + &RESERVATION; + &CHGRES_BC_RESOURCES; + _BCNODES_:ppn=1 + &JOBS;/launch.ksh &JOBS;/JFV3CAM_SAR_CHGRES_FCSTBNDY + chgres_fcstbndy_&domain;_@H + &OUTDIR;/logfiles_sar/chgres_fcstbndy_&domain;_@H.log + + + HOMEfv3 + &HOMEfv3; + + + job + chgres_fcstbndy_&domain; + + + machine + &machine; + + + USER + &USER; + + + CDATE + @Y@m@d@H + + + PDY + @Y@m@d + + + cyc + @H + + + tmmark + tm00 + + + dom + &domain; + + + + + + &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf057.nemsio + &COMgfs;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf060.nemsio + + + &COMgfs2;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf009.nemsio + &COMgfs2;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf060.nemsio + &COMgfs2;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf063.nemsio + &COMgfs2;/gfs.@Y@m@d/@H/gfs.t@Hz.atmf066.nemsio + + + + + + + + + + + + &RESERVATION; + &FCST_RESOURCES; + _FCSTNODES_:ppn=12 + &JOBS;/launch.ksh &JOBS;/JFV3CAM_SAR_FCST + forecast_tm00_&domain;_@H + &OUTDIR;/logfiles_sar/forecast_tm00_&domain;_@H.log + + + HOMEfv3 + &HOMEfv3; + + + job + forecast_tm00_&domain;_@Y@m@d + + + machine + &machine; + + + USER + &USER; + + + CDATE + @Y@m@d@H + + + PDY + @Y@m@d + + + cyc + @H + + + tmmark + tm00 + + + dom + &domain; + + + + + + + + + + + + + + + + 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 + + &RESERVATION; + &POST_RESOURCES; + _POSTNODES_:ppn=14 + &JOBS;/launch.ksh &JOBS;/JFV3CAM_SAR_POST + fv3_post_&domain;_f#fhr#_@Hz + &OUTDIR;/logfiles_sar/post_&domain;_f#fhr#_@H_tm00.log + + + HOMEfv3 + &HOMEfv3; + + + job + post_&domain;_f#fhr# + + + machine + &machine; + + + USER + &USER; + + + CDATE + @Y@m@d@H + + + PDY + @Y@m@d + + + cyc + @H + + + fhr + #fhr# + + + tmmark + tm00 + + + dom + &domain; + + + + + &DATAROOT;/forecast_tm00_&domain;_@Y@m@d_@H/logf0#fhr# + + + + + + + 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 + + &RESERVATION; + &POSTGOES_RESOURCES; + _GOESPOSTNODES_:ppn=14 + &JOBS;/launch.ksh &JOBS;/JFV3CAM_SAR_POST_GOES + fv3_postgoes_&domain;_f#fhr#_@Hz + &OUTDIR;/logfiles_sar/postgoes_&domain;_f#fhr#_@H_tm00.log + + + HOMEfv3 + &HOMEfv3; + + + job + postgoes_&domain;_f#fhr# + + + machine + &machine; + + + USER + &USER; + + + CDATE + @Y@m@d@H + + + PDY + @Y@m@d + + + cyc + @H + + + fhr + #fhr# + + + tmmark + tm00 + + + dom + &domain; + + + + + &DATAROOT;/forecast_tm00_&domain;_@Y@m@d_@H/logf0#fhr# + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rocoto/run_fv3nest.sh b/rocoto/run_fv3nest.sh index 242c43f08..04a307a63 100755 --- a/rocoto/run_fv3nest.sh +++ b/rocoto/run_fv3nest.sh @@ -9,4 +9,4 @@ module load lsf/10.1 module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles module load ruby/2.5.1 rocoto/1.2.4 -rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3nest.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3nest.db +rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3nest.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3nest.db diff --git a/rocoto/run_fv3nest_retro.sh b/rocoto/run_fv3nest_retro.sh index b8fe26c29..ff45a3dd8 100755 --- a/rocoto/run_fv3nest_retro.sh +++ b/rocoto/run_fv3nest_retro.sh @@ -9,4 +9,4 @@ module load lsf/10.1 module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles module load ruby/2.5.1 rocoto/1.2.4 -rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3nest_retro.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3nest_retro.db +rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3nest_retro.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3nest_retro.db diff --git a/rocoto/run_fv3sar.sh b/rocoto/run_fv3sar.sh index 8ed111c97..9d94eb531 100755 --- a/rocoto/run_fv3sar.sh +++ b/rocoto/run_fv3sar.sh @@ -5,8 +5,9 @@ set -x module load impi/18.0.1 module load lsf/10.1 +module load NetCDF/4.5.0 module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/ module load ruby/2.5.1 rocoto/1.2.4 -rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3sar.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3sar.db +rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3sar.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3sar.db diff --git a/rocoto/run_fv3sar_da.sh b/rocoto/run_fv3sar_da.sh index 38cb8402c..6ccb4eb27 100755 --- a/rocoto/run_fv3sar_da.sh +++ b/rocoto/run_fv3sar_da.sh @@ -9,4 +9,4 @@ module load lsf/10.1 module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/ module load ruby/2.5.1 rocoto/1.2.4 -rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3sar_da.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3sar_da.db +rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3sar_da.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3sar_da.db diff --git a/rocoto/run_fv3sar_da_retro.sh b/rocoto/run_fv3sar_da_retro.sh index 7c59ee41e..59d872ac7 100755 --- a/rocoto/run_fv3sar_da_retro.sh +++ b/rocoto/run_fv3sar_da_retro.sh @@ -9,4 +9,4 @@ module load lsf/10.1 module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/ module load ruby/2.5.1 rocoto/1.2.4 -rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3sar_da_retro.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3sar_da_retro.db +rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3sar_da_retro.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3sar_da_retro.db diff --git a/rocoto/run_fv3sar_multi.sh b/rocoto/run_fv3sar_multi.sh new file mode 100755 index 000000000..05c61a1e3 --- /dev/null +++ b/rocoto/run_fv3sar_multi.sh @@ -0,0 +1,21 @@ +#!/bin/bash -l +set +x +. /usrx/local/prod/lmod/lmod/init/sh +set -x + +module load impi/18.0.1 +module load lsf/10.1 + +module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/ +module load ruby/2.5.1 rocoto/1.2.4 + +doms="hi pr" + + +dir="/gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto" + +for dom in $doms +do +rocotorun -v 10 -w ${dir}/drive_fv3sar_${dom}.xml -d ${dir}/drive_fv3sar_${dom}.db +sleep 30 +done diff --git a/rocoto/run_fv3sar_retro.sh b/rocoto/run_fv3sar_retro.sh index 704d55d05..6b574c4fb 100755 --- a/rocoto/run_fv3sar_retro.sh +++ b/rocoto/run_fv3sar_retro.sh @@ -9,4 +9,4 @@ module load lsf/10.1 module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/ module load ruby/2.5.1 rocoto/1.2.4 -rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3sar_retro.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/rocoto/drive_fv3sar_retro.db +rocotorun -v 10 -w /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3sar_retro.xml -d /gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/rocoto/drive_fv3sar_retro.db diff --git a/scripts/exfv3cam_post.sh b/scripts/exfv3cam_post.sh index 1a62c7418..fd6c9ca70 100755 --- a/scripts/exfv3cam_post.sh +++ b/scripts/exfv3cam_post.sh @@ -52,8 +52,25 @@ mpirun ${POSTGPEXEC} < itag > $pgmout 2> err export err=$?; err_chk # Run wgrib2 -domain=conus +domain=${dom} + +if [ $domain = conus ] +then gridspecs="lambert:262.5:38.5:38.5 237.280:1799:3000 21.138:1059:3000" +elif [ $domain = "ak" ] +then +gridspecs="nps:210:60 185.5:825:5000 44.8:603:5000" +elif [ $domain = pr ] +then +gridspecs="latlon 283.41:340:.045 13.5:208:.045" +elif [ $domain = hi ] +then +gridspecs="latlon 197.65:223:.045 16.4:170:.045" +elif [ $domain = guam ] +then +gridspecs="latlon 141.0:223:.045 11.7:170:.045" +fi + compress_type=c3 if [ $fhr -eq 00 ] ; then diff --git a/scripts/exfv3cam_post_goes.sh b/scripts/exfv3cam_post_goes.sh index 2f8ae77b6..2ba4e2bff 100755 --- a/scripts/exfv3cam_post_goes.sh +++ b/scripts/exfv3cam_post_goes.sh @@ -74,15 +74,32 @@ mpirun ${POSTGPEXEC} < itag > $pgmout 2>err export err=$?; err_chk # RUN wgrib2 -domain=conus +domain=${dom} + +if [ $domain = "conus" ] +then gridspecs="lambert:262.5:38.5:38.5 237.280:1799:3000 21.138:1059:3000" +elif [ $domain = "ak" ] +then +gridspecs="nps:210:60 185.5:825:5000 44.8:603:5000" +elif [ $domain = "pr" ] +then +gridspecs="latlon 283.41:340:.045 13.5:208:.045" +elif [ $domain = "hi" ] +then +gridspecs="latlon 197.65:223:.045 16.4:170:.045" +elif [ $domain = guam ] +then +gridspecs="latlon 141.0:223:.045 11.7:170:.045" +fi + compress_type=c3 $WGRIB2 BGGOES${fhr}.${tmmark} -set_bitmap 1 -set_grib_type ${compress_type} -new_grid_winds grid -new_grid_vectors "UGRD:VGRD:USTM:VSTM" \ -new_grid_interpolation neighbor \ -new_grid ${gridspecs} ${domain}fv3.goestb${fhr}.${tmmark} -mv ${domain}fv3.goestb${fhr}.${tmmark} $COMOUT/${RUN}.t${cyc}z.conusgoestb.f${fhr}.grib2 +mv ${domain}fv3.goestb${fhr}.${tmmark} $COMOUT/${RUN}.t${cyc}z.${dom}goestb.f${fhr}.grib2 echo done > ${INPUT_DATA}/postgoestbdone${fhr}.${tmmark} diff --git a/scripts/exfv3cam_sar_fcst.sh b/scripts/exfv3cam_sar_fcst.sh index 27dcd47a2..36c53ecf2 100755 --- a/scripts/exfv3cam_sar_fcst.sh +++ b/scripts/exfv3cam_sar_fcst.sh @@ -19,7 +19,7 @@ ulimit -s unlimited ulimit -a mkdir -p INPUT RESTART -cp ${NWGES}/anl.${tmmark}/*.nc INPUT +cp ${NWGES}/anl.${dom}.${tmmark}/*.nc INPUT numbndy=`ls -l INPUT/gfs_bndy.tile7*.nc | wc -l` let "numbndy_check=$NHRS/3+1" @@ -102,20 +102,41 @@ cd .. # input.nml, input_nest02.nml, model_configure, and nems.configure #------------------------------------------------------------------- + +echo in fcst script we know that dom is $dom + if [ $tmmark = tm00 ] ; then # Free forecast with DA (warm start) if [ $model = fv3sar_da ] ; then cp ${PARMfv3}/input_sar_da.nml input.nml # Free forecast without DA (cold start) elif [ $model = fv3sar ] ; then - cp ${PARMfv3}/input_sar.nml input.nml + cp ${PARMfv3}/input_sar_${dom}.nml input.nml fi - cp ${PARMfv3}/model_configure_sar.tmp model_configure.tmp - nodes=76 - ncnode=24 - let nctsk=ncnode/OMP_NUM_THREADS # 12 tasks per node with 2 threads - let ntasks=nodes*nctsk - echo nctsk = $nctsk and ntasks = $ntasks + cp ${PARMfv3}/model_configure_sar.tmp_${dom} model_configure.tmp + +if [ $dom = "conus" ] +then + nodes=76 +elif [ $dom = "ak" ] +then + nodes=68 +elif [ $dom = "pr" ] +then + nodes=10 +elif [ $dom = "hi" ] +then + nodes=7 +elif [ $dom = "guam" ] +then + nodes=7 +fi + +ncnode=24 +let nctsk=ncnode/OMP_NUM_THREADS # 12 tasks per node with 2 threads +let ntasks=nodes*nctsk +echo nctsk = $nctsk and ntasks = $ntasks + # Submit post manager here else diff --git a/ush/global_chgres.sh b/ush/global_chgres.sh index 5e0ee4a3a..7f4b75b11 100755 --- a/ush/global_chgres.sh +++ b/ush/global_chgres.sh @@ -260,7 +260,7 @@ gfs_ver=${gfs_ver:-v15.0.0} BASEDIR=${BASEDIR:-${NWROOT:-/nwprod2}} HOMEgfs=${HOMEgfs:-$BASEDIR/gfs.${gfs_ver}} EXECgfs=${EXECgfs:-$HOMEgfs/exec} -FIXfv3=${FIXfv3:-/gpfs/dell2/emc/modeling/noscrub/${USER}/fv3cam_workflow/fix} +FIXfv3=${FIXfv3:-/gpfs/dell2/emc/modeling/noscrub/${USER}/regional_workflow/fix} FIXsar=${FIXsar:-$FIXfv3/fix_sar} FIXnest=${FIXnest:-$FIXfv3/fix_nest} FIXam=${FIXam:-$FIXfv3/fix_am}