diff --git a/.gitignore b/.gitignore index eca261d36e..fd91149de3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ configure.wrf* *.backup *.f90 +#added for macOS environment (KSA) +**/.DS_Store diff --git a/testcases/conus_12-km/namelist_v4.5.2_conus12km_restart.input b/testcases/conus_12-km/namelist_v4.5.2_conus12km_restart.input new file mode 100644 index 0000000000..285714d469 --- /dev/null +++ b/testcases/conus_12-km/namelist_v4.5.2_conus12km_restart.input @@ -0,0 +1,107 @@ + &time_control + run_days = 5, + run_hours = 0, + run_minutes = 0, + run_seconds = 0, + start_year = 2008, + start_month = 07, + start_day = 15, + start_hour = 00, + start_minute = 00, + start_second = 00, + end_year = 2008, + end_month = 07, + end_day = 21, + end_hour = 00, + end_minute = 00, + end_second = 00, + interval_seconds = 21600 + input_from_file = .true., + history_interval_m = 360, + frames_per_outfile = 1, + restart = .true., + restart_interval = 7200, + io_form_history = 11, + io_form_restart = 11, + io_form_input = 11, + io_form_boundary = 11, + debug_level = 0, + nocolons = .true., + override_restart_timers = .true., + / + + &domains + perturb_input = .false. + time_step = 72, + time_step_fract_num = 0, + time_step_fract_den = 1, + max_dom = 1, + s_we = 1, + e_we = 425, + e_sn = 300, + e_vert = 50, + dx = 12000, + dy = 12000, + grid_id = 1, + parent_id = 0, + i_parent_start = 0, + j_parent_start = 0, + parent_grid_ratio = 1, + parent_time_step_ratio = 1, + feedback = 1, + smooth_option = 0 + num_metgrid_levels = 34, + num_metgrid_soil_levels = 4, + numtiles = 4, + nproc_x = 8, + nproc_y = 24, + / + + &physics + physics_suite = 'conus' + mp_physics = -1, + ra_lw_physics = -1, + ra_sw_physics = -1, + radt = 12, + sf_sfclay_physics = -1, + sf_surface_physics = -1, + bl_pbl_physics = -1, + bldt = 0, + cu_physics = -1, + cudt = 0, + isfflx = 1, + ifsnow = 0, + icloud = 1, + surface_input_source = 1, + num_soil_layers = 1, + maxiens = 1, + maxens = 1, + maxens2 = 1, + maxens3 = 1, + ensdim = 1, + / + + &dynamics + w_damping = 1, + diff_opt = 1, + km_opt = 4, + khdif = 0, + kvdif = 0, + non_hydrostatic = .true., + moist_adv_opt = 1, + scalar_adv_opt = 1, + use_baseparam_fr_nml = .t. + / + + &bdy_control + spec_bdy_width = 5, + spec_zone = 1, + relax_zone = 4, + specified = .true., + nested = .false., + / + + &namelist_quilt + nio_tasks_per_group = 0, + nio_groups = 1, + / diff --git a/testcases/conus_12-km/readme.md b/testcases/conus_12-km/readme.md new file mode 100644 index 0000000000..599b4c3b3a --- /dev/null +++ b/testcases/conus_12-km/readme.md @@ -0,0 +1,49 @@ +# CONUS12km benchmark case + +## Source + +Original files were downloaded from NCAR MMM website: + +https://www2.mmm.ucar.edu/wrf/users/benchmark/benchdata_v422.html + +## Files + +The following three files + tar.gz archive file provide everything needed to run the test case. + +Only thing a user needs to do is to run the shell script `setup_rundir_WRF.sh` + +- `namelist_v4.5.2_conus12km_restart.input`: wrf namelist for the test case +- `sub_wrf_pm_testcase.sh` : an example batch script to run a test case on Perlmutter +- `setup_rundir_WRF.sh` : createa a directory to run the test simulation, copy input files, and submit a job after editing (sed) a copy of `sub_wrf_pm_testcase.sh` batch script. +- Initial and boundary conditions, and look-up tables for various model modules/kernels are archived and +stored in a publicly available space under WRF-SIG repository. The `setup_rundir_WRF.sh` script downloads this file +by wget command. + +### setup_rundir_WRF.sh + +Edit the following three lines and then execute. +``` +casename="codee_test01" #identifier for the new simulation + +#email address to receive notifications from the slurm system +myemail="Koichi.Sakaguchi@pnnl.gov" + +#directory where we run wrf.exe and writes output +rundir="/pscratch/sd/k/ksa/simulation/WRF/WRFSIG/${casename}" + +``` + +## Test Case + +The original CONUS12km case runs only for one hour to simulate a strong winter storm case. + +For more robust statistics and higher sensitivity to source code changes, the simulation period is extended +to 5 days (can be as long as 10 days) and to the summer season (July) in 2008; this time period is chosen +just for the convenience of K. Sakaguchi, who uses this period for his own research). +In the summer the influence of lateral boundary forcing +(e.g., winds coming in from the west) is weaker than in the winter, therefore processes originated +locally (within the model domain) have more impact. +So changes in the model executable would appear sooner and stronger. + +The 5-day simulation typically ends within 15 minutes using 194 MPI tasks, 4 threads in each task, +on 3 CPU nodes of Perlmutter. diff --git a/testcases/conus_12-km/setup_rundir_WRF.sh b/testcases/conus_12-km/setup_rundir_WRF.sh new file mode 100755 index 0000000000..1ee6a7e78d --- /dev/null +++ b/testcases/conus_12-km/setup_rundir_WRF.sh @@ -0,0 +1,45 @@ +#! /bin/bash +set -e + +#edit these lines ---------------------------------------- + +casename="codee_test01" #identifier for the new simulation + +#email address to receive notifications from the slurm system +myemail="Koichi.Sakaguchi@pnnl.gov" + +#directory where we run wrf.exe and writes output +rundir="/pscratch/sd/k/ksa/simulation/WRF/WRFSIG/${casename}" + +# ------------------- +scriptdir=$(pwd) + +mkdir -p ${rundir} + +cd ${rundir} +pwd + +#get input files from NERSC's WRFSIG CFS space through Science Gateway service +echo "downloading input file archive" + +wget https://portal.nersc.gov/cfs/m4232pub/testcase_input/testcase_input_conus12km_v4.5.2.tar.gz + +echo "expanding the input file arhive" + +tar -zxf testcase_input_conus12km_v4.5.2.tar.gz + +echo "finished copying the input files" + + +rm -rf ${rundir}/namelist.input #remove the default namelist.input file +#copy WRF namelist +cp ${scriptdir}/namelist_v4.5.2_conus12km_restart.input ${rundir}/namelist.input + +#copy sbatch script +cp ${scriptdir}/sub_wrf_pm_testcase.sh ${rundir}/sub_testcase.sh + +#edit the email address and job name in the sbatch script +sed -i "s/elvis@nersc.gov/${myemail}/" sub_testcase.sh +sed -i "s/elvis_test01/${casename}/" sub_testcase.sh + +sbatch sub_testcase.sh \ No newline at end of file diff --git a/testcases/conus_12-km/sub_wrf_pm_testcase.sh b/testcases/conus_12-km/sub_wrf_pm_testcase.sh new file mode 100644 index 0000000000..dff0291956 --- /dev/null +++ b/testcases/conus_12-km/sub_wrf_pm_testcase.sh @@ -0,0 +1,43 @@ +#!/bin/bash +#SBATCH -N 3 +#SBATCH -q debug +#SBATCH -t 00:30:00 +#SBATCH -J elvis_test01 +#SBATCH -A m4232 +#SBATCH --mail-type=END,FAIL +#SBATCH --mail-user=elvis@nersc.gov +#SBATCH -L scratch,cfs +#SBATCH -C cpu +#SBATCH --tasks-per-node=64 +##SBATCH -S 4 + +ntile=4 #number of OpenMP threads per MPI task + +#Modules -------------------------------------------------------------------- +module load contrib +module load wrf/4.5.2 + +#if to run with a wrf executable from modified source codes: +#1. don't load the wrf module +#2. the modified executable (wrf.exe) has to be placed in the rundir + +#OpenMP settings: +export OMP_NUM_THREADS=$ntile +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + +#run simulation +srun -n 192 -c 4 --cpu_bind=cores wrf.exe #3 nodes + +#capture error code +srunval=$? + +#rename and save the process 0 out and err files +cp rsl.error.0000 rsl.error_0_$SLURM_JOB_ID +cp rsl.out.0000 rsl.out_0_$SLURM_JOB_ID + +if [ $srunval -ne 0 ]; then + echo "run failed" + exit 10 +fi +