From 4f857462e9999d805b0a36a034ca7be2cad07534 Mon Sep 17 00:00:00 2001 From: Koichi Sakaguchi Date: Mon, 5 Feb 2024 08:26:55 -0800 Subject: [PATCH 1/3] added testcases/conus_12-km directory and added DS_Store files (in macOS environment) in gitignore --- .gitignore | 2 ++ testcases/conus_12-km/readme.md | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 testcases/conus_12-km/readme.md diff --git a/.gitignore b/.gitignore index eca261d36e..fd91149de3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ configure.wrf* *.backup *.f90 +#added for macOS environment (KSA) +**/.DS_Store diff --git a/testcases/conus_12-km/readme.md b/testcases/conus_12-km/readme.md new file mode 100644 index 0000000000..1a506e5dc1 --- /dev/null +++ b/testcases/conus_12-km/readme.md @@ -0,0 +1,3 @@ +### CONUS12km benchmark case +Downloaded from NCAR MMM website: +https://www2.mmm.ucar.edu/wrf/users/benchmark/benchdata_v422.html From 55d03516bd5a47e53237ec66d4fe5a95fcf85f8e Mon Sep 17 00:00:00 2001 From: Koichi Sakaguchi Date: Tue, 13 Feb 2024 16:14:35 -0800 Subject: [PATCH 2/3] added namelist, batch script, and a bash script to set up a directory to run the testcase simulation --- .../namelist_v4.5.2_conus12km_restart.input | 107 ++++++++++++++++++ testcases/conus_12-km/setup_rundir_WRF.sh | 31 +++++ testcases/conus_12-km/sub_wrf_pm_testcase.sh | 44 +++++++ 3 files changed, 182 insertions(+) create mode 100644 testcases/conus_12-km/namelist_v4.5.2_conus12km_restart.input create mode 100755 testcases/conus_12-km/setup_rundir_WRF.sh create mode 100644 testcases/conus_12-km/sub_wrf_pm_testcase.sh diff --git a/testcases/conus_12-km/namelist_v4.5.2_conus12km_restart.input b/testcases/conus_12-km/namelist_v4.5.2_conus12km_restart.input new file mode 100644 index 0000000000..285714d469 --- /dev/null +++ b/testcases/conus_12-km/namelist_v4.5.2_conus12km_restart.input @@ -0,0 +1,107 @@ + &time_control + run_days = 5, + run_hours = 0, + run_minutes = 0, + run_seconds = 0, + start_year = 2008, + start_month = 07, + start_day = 15, + start_hour = 00, + start_minute = 00, + start_second = 00, + end_year = 2008, + end_month = 07, + end_day = 21, + end_hour = 00, + end_minute = 00, + end_second = 00, + interval_seconds = 21600 + input_from_file = .true., + history_interval_m = 360, + frames_per_outfile = 1, + restart = .true., + restart_interval = 7200, + io_form_history = 11, + io_form_restart = 11, + io_form_input = 11, + io_form_boundary = 11, + debug_level = 0, + nocolons = .true., + override_restart_timers = .true., + / + + &domains + perturb_input = .false. + time_step = 72, + time_step_fract_num = 0, + time_step_fract_den = 1, + max_dom = 1, + s_we = 1, + e_we = 425, + e_sn = 300, + e_vert = 50, + dx = 12000, + dy = 12000, + grid_id = 1, + parent_id = 0, + i_parent_start = 0, + j_parent_start = 0, + parent_grid_ratio = 1, + parent_time_step_ratio = 1, + feedback = 1, + smooth_option = 0 + num_metgrid_levels = 34, + num_metgrid_soil_levels = 4, + numtiles = 4, + nproc_x = 8, + nproc_y = 24, + / + + &physics + physics_suite = 'conus' + mp_physics = -1, + ra_lw_physics = -1, + ra_sw_physics = -1, + radt = 12, + sf_sfclay_physics = -1, + sf_surface_physics = -1, + bl_pbl_physics = -1, + bldt = 0, + cu_physics = -1, + cudt = 0, + isfflx = 1, + ifsnow = 0, + icloud = 1, + surface_input_source = 1, + num_soil_layers = 1, + maxiens = 1, + maxens = 1, + maxens2 = 1, + maxens3 = 1, + ensdim = 1, + / + + &dynamics + w_damping = 1, + diff_opt = 1, + km_opt = 4, + khdif = 0, + kvdif = 0, + non_hydrostatic = .true., + moist_adv_opt = 1, + scalar_adv_opt = 1, + use_baseparam_fr_nml = .t. + / + + &bdy_control + spec_bdy_width = 5, + spec_zone = 1, + relax_zone = 4, + specified = .true., + nested = .false., + / + + &namelist_quilt + nio_tasks_per_group = 0, + nio_groups = 1, + / diff --git a/testcases/conus_12-km/setup_rundir_WRF.sh b/testcases/conus_12-km/setup_rundir_WRF.sh new file mode 100755 index 0000000000..173879c70c --- /dev/null +++ b/testcases/conus_12-km/setup_rundir_WRF.sh @@ -0,0 +1,31 @@ +#! /bin/bash +casename="codee_test01" #identifier for the new simulation +myemail="Koichi.Sakaguchi@pnnl.gov" + +rundir="/pscratch/sd/k/ksa/simulation/WRF/WRFSIG/${casename}" + +mkdir -p ${rundir} + +#copy WRF namelist +cp namelist_v4.5.2_conus12km_restart.input ${rundir}/namelist.input + +#copy sbatch script +cp sub_wrf_pm_testcase.sh ${rundir}/sub_wrf_pm_testcase.sh + +cd ${rundir} +#edit the email address in the sbatch script +sed -i "s/elvis@nersc.gov/${mymail}/" sub_wrf_pm_testcase.sh + +#get input files from NERSC's WRFSIG CFS space +wget https://portal.nersc.gov/cfs/m4232pub/testcase_input/testcase_input_conus12km_v4.5.2.tar.gz + +tar -zxf testcase_input_conus12km_v4.5.2.tar.gz + + +echo " ----- namelist.input ----- " +more namelist.input + +echo " ----- sbatch script ----- " +more sub_wrf_pm_testcase.sh + +echo sbatch sub_wrf_pm_testcase.sh \ No newline at end of file diff --git a/testcases/conus_12-km/sub_wrf_pm_testcase.sh b/testcases/conus_12-km/sub_wrf_pm_testcase.sh new file mode 100644 index 0000000000..4246ceb7e0 --- /dev/null +++ b/testcases/conus_12-km/sub_wrf_pm_testcase.sh @@ -0,0 +1,44 @@ +#!/bin/bash +#SBATCH -N 3 +#SBATCH -q debug +#SBATCH -t 00:30:00 +#SBATCH -J codee_test01 +#SBATCH -A m4232 +#SBATCH --mail-type=END,FAIL +#SBATCH --mail-user=elvis@nersc.gov +#SBATCH -L scratch,cfs +#SBATCH -C cpu +#SBATCH --tasks-per-node=64 +##SBATCH -S 4 + + +casename="codee_test01" +rundir="/pscratch/sd/k/ksa/simulation/WRF/WRFSIG/${casename}" + + +ntile=4 #number of OpenMP threads per MPI task + +#Modules -------------------------------------------------------------------- + +module load contrib +module load wrf/4.5.2 + +#OpenMP settings: +export OMP_NUM_THREADS=$ntile +export OMP_PLACES=threads +export OMP_PROC_BIND=spread + + + +cd ${rundir} +mkdir -p logs + + +#run simulation +srun -n 192 -c 4 --cpu_bind=cores wrf.exe #3 nodes + +#rename and save the process 0 out and err files +cp rsl.error.0000 rsl.error_0_$SLURM_JOB_ID +cp rsl.out.0000 rsl.out_0_$SLURM_JOB_ID + + From 4445b8fbdbf3c2a08686111215c2823dba42d6ce Mon Sep 17 00:00:00 2001 From: Koichi Sakaguchi Date: Tue, 13 Feb 2024 20:44:55 -0800 Subject: [PATCH 3/3] finalize csetup_rundir_WRF.sh and sub_wrf_pm_testcase.sh after testing on Perlmutter. Also added the content of readme.md --- testcases/conus_12-km/readme.md | 50 +++++++++++++++++++- testcases/conus_12-km/setup_rundir_WRF.sh | 44 +++++++++++------ testcases/conus_12-km/sub_wrf_pm_testcase.sh | 25 +++++----- 3 files changed, 89 insertions(+), 30 deletions(-) diff --git a/testcases/conus_12-km/readme.md b/testcases/conus_12-km/readme.md index 1a506e5dc1..599b4c3b3a 100644 --- a/testcases/conus_12-km/readme.md +++ b/testcases/conus_12-km/readme.md @@ -1,3 +1,49 @@ -### CONUS12km benchmark case -Downloaded from NCAR MMM website: +# CONUS12km benchmark case + +## Source + +Original files were downloaded from NCAR MMM website: + https://www2.mmm.ucar.edu/wrf/users/benchmark/benchdata_v422.html + +## Files + +The following three files + tar.gz archive file provide everything needed to run the test case. + +Only thing a user needs to do is to run the shell script `setup_rundir_WRF.sh` + +- `namelist_v4.5.2_conus12km_restart.input`: wrf namelist for the test case +- `sub_wrf_pm_testcase.sh` : an example batch script to run a test case on Perlmutter +- `setup_rundir_WRF.sh` : createa a directory to run the test simulation, copy input files, and submit a job after editing (sed) a copy of `sub_wrf_pm_testcase.sh` batch script. +- Initial and boundary conditions, and look-up tables for various model modules/kernels are archived and +stored in a publicly available space under WRF-SIG repository. The `setup_rundir_WRF.sh` script downloads this file +by wget command. + +### setup_rundir_WRF.sh + +Edit the following three lines and then execute. +``` +casename="codee_test01" #identifier for the new simulation + +#email address to receive notifications from the slurm system +myemail="Koichi.Sakaguchi@pnnl.gov" + +#directory where we run wrf.exe and writes output +rundir="/pscratch/sd/k/ksa/simulation/WRF/WRFSIG/${casename}" + +``` + +## Test Case + +The original CONUS12km case runs only for one hour to simulate a strong winter storm case. + +For more robust statistics and higher sensitivity to source code changes, the simulation period is extended +to 5 days (can be as long as 10 days) and to the summer season (July) in 2008; this time period is chosen +just for the convenience of K. Sakaguchi, who uses this period for his own research). +In the summer the influence of lateral boundary forcing +(e.g., winds coming in from the west) is weaker than in the winter, therefore processes originated +locally (within the model domain) have more impact. +So changes in the model executable would appear sooner and stronger. + +The 5-day simulation typically ends within 15 minutes using 194 MPI tasks, 4 threads in each task, +on 3 CPU nodes of Perlmutter. diff --git a/testcases/conus_12-km/setup_rundir_WRF.sh b/testcases/conus_12-km/setup_rundir_WRF.sh index 173879c70c..1ee6a7e78d 100755 --- a/testcases/conus_12-km/setup_rundir_WRF.sh +++ b/testcases/conus_12-km/setup_rundir_WRF.sh @@ -1,31 +1,45 @@ #! /bin/bash +set -e + +#edit these lines ---------------------------------------- + casename="codee_test01" #identifier for the new simulation -myemail="Koichi.Sakaguchi@pnnl.gov" -rundir="/pscratch/sd/k/ksa/simulation/WRF/WRFSIG/${casename}" +#email address to receive notifications from the slurm system +myemail="Koichi.Sakaguchi@pnnl.gov" -mkdir -p ${rundir} +#directory where we run wrf.exe and writes output +rundir="/pscratch/sd/k/ksa/simulation/WRF/WRFSIG/${casename}" -#copy WRF namelist -cp namelist_v4.5.2_conus12km_restart.input ${rundir}/namelist.input +# ------------------- +scriptdir=$(pwd) -#copy sbatch script -cp sub_wrf_pm_testcase.sh ${rundir}/sub_wrf_pm_testcase.sh +mkdir -p ${rundir} cd ${rundir} -#edit the email address in the sbatch script -sed -i "s/elvis@nersc.gov/${mymail}/" sub_wrf_pm_testcase.sh +pwd + +#get input files from NERSC's WRFSIG CFS space through Science Gateway service +echo "downloading input file archive" -#get input files from NERSC's WRFSIG CFS space wget https://portal.nersc.gov/cfs/m4232pub/testcase_input/testcase_input_conus12km_v4.5.2.tar.gz +echo "expanding the input file arhive" + tar -zxf testcase_input_conus12km_v4.5.2.tar.gz +echo "finished copying the input files" + -echo " ----- namelist.input ----- " -more namelist.input +rm -rf ${rundir}/namelist.input #remove the default namelist.input file +#copy WRF namelist +cp ${scriptdir}/namelist_v4.5.2_conus12km_restart.input ${rundir}/namelist.input + +#copy sbatch script +cp ${scriptdir}/sub_wrf_pm_testcase.sh ${rundir}/sub_testcase.sh -echo " ----- sbatch script ----- " -more sub_wrf_pm_testcase.sh +#edit the email address and job name in the sbatch script +sed -i "s/elvis@nersc.gov/${myemail}/" sub_testcase.sh +sed -i "s/elvis_test01/${casename}/" sub_testcase.sh -echo sbatch sub_wrf_pm_testcase.sh \ No newline at end of file +sbatch sub_testcase.sh \ No newline at end of file diff --git a/testcases/conus_12-km/sub_wrf_pm_testcase.sh b/testcases/conus_12-km/sub_wrf_pm_testcase.sh index 4246ceb7e0..dff0291956 100644 --- a/testcases/conus_12-km/sub_wrf_pm_testcase.sh +++ b/testcases/conus_12-km/sub_wrf_pm_testcase.sh @@ -2,7 +2,7 @@ #SBATCH -N 3 #SBATCH -q debug #SBATCH -t 00:30:00 -#SBATCH -J codee_test01 +#SBATCH -J elvis_test01 #SBATCH -A m4232 #SBATCH --mail-type=END,FAIL #SBATCH --mail-user=elvis@nersc.gov @@ -11,34 +11,33 @@ #SBATCH --tasks-per-node=64 ##SBATCH -S 4 - -casename="codee_test01" -rundir="/pscratch/sd/k/ksa/simulation/WRF/WRFSIG/${casename}" - - ntile=4 #number of OpenMP threads per MPI task #Modules -------------------------------------------------------------------- - module load contrib module load wrf/4.5.2 +#if to run with a wrf executable from modified source codes: +#1. don't load the wrf module +#2. the modified executable (wrf.exe) has to be placed in the rundir + #OpenMP settings: export OMP_NUM_THREADS=$ntile export OMP_PLACES=threads export OMP_PROC_BIND=spread - - -cd ${rundir} -mkdir -p logs - - #run simulation srun -n 192 -c 4 --cpu_bind=cores wrf.exe #3 nodes +#capture error code +srunval=$? + #rename and save the process 0 out and err files cp rsl.error.0000 rsl.error_0_$SLURM_JOB_ID cp rsl.out.0000 rsl.out_0_$SLURM_JOB_ID +if [ $srunval -ne 0 ]; then + echo "run failed" + exit 10 +fi