diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile new file mode 100644 index 00000000..d05d5828 --- /dev/null +++ b/.cicd/Jenkinsfile @@ -0,0 +1,59 @@ +pipeline { + agent none + environment { + GITHUB_TOKEN = credentials('GithubJenkinsNew') + AWS_ACCOUNT_ID = credentials('AWS_PROD_ACCOUNT_ID') + } + stages { + stage('Run Unit Tests') { + agent { + label 'hera' + } + steps { + script { + cleanWs() + checkout scm + sh ''' + cd unit_tests + echo "Running Unit Tests on ${NODE_NAME}" + ./compile_standalone.hera_gnu + cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT + srun --label -A epic -n 24 standalone_stochy.x + mkdir stochy_out + mv workg* stochy_out + cd stochy_out/ + ls + ''' + } + } + } + } + post { + always { + node('built-in') { + sh ''' + GIT_OWNER=$(echo $CHANGE_URL | cut -d '/' -f4) + GIT_REPO_NAME=$(echo $CHANGE_URL | cut -d '/' -f5 | cut -d '.' -f1) + echo "Testing successfully completed...removing label(s)" + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}' + ''' + } + } + success { + node('built-in') { + echo 'This will run only if successful.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_ACCOUNT_ID}:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' + ''' + } + } + failure { + node('built-in') { + echo 'This will run only if the run was marked as unstable.' + sh ''' + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_ACCOUNT_ID}:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' + ''' + } + } + } +} diff --git a/docs/source/developers_guide.rst b/docs/source/developers_guide.rst index 7990c09e..2169704f 100644 --- a/docs/source/developers_guide.rst +++ b/docs/source/developers_guide.rst @@ -12,3 +12,7 @@ If you intend to make modifications to the stochastic physics source code, there Full model tests """""""""""""""" The code updates are not expected to change existing results, so the full model regression tests need to be run. All of the tests must pass, although only a sub-set of tests are needed to consider adding changes to the stochastic_physics repository: fv3_control, fv3_stochy, fv3_ccpp_control, and fv3_ccpp_stochy. If the results are expected to change, then there needs to be scientific evidience that the change in results are what is expected. + +Unit Test Pipeline +""""""""""""""""" +The Jenkins Pipeline for running the standalone unit test can be started automatically by adding the "run-unit-tests" Git label to the PR. Pass/Fail results can be viewed from the footer of the PR in the Git Actions. A Passing test run will show "All checks have passed". This section can be expanded and selected to show test log output within the Jenkins console. Jenkins access is required to view the console output and can be provided by the EPIC Platform team upon request. diff --git a/stochastic_physics.F90 b/stochastic_physics.F90 index 1a198c4d..af78691c 100644 --- a/stochastic_physics.F90 +++ b/stochastic_physics.F90 @@ -164,9 +164,15 @@ subroutine init_stochastic_physics(levs, blksz, dtp, sppt_amp, input_nml_file_in endif enddo if (sppt_sfclimit) then - do k=1,7 - vfact_sppt(k)=pbl_taper(k) - enddo + if (levs .le. 7) then + do k=1,levs + vfact_sppt(k)=pbl_taper(k) + enddo + else + do k=1,7 + vfact_sppt(k)=pbl_taper(k) + enddo + endif endif if (is_rootpe()) then do k=1,levs @@ -525,34 +531,21 @@ subroutine run_stochastic_physics_ocn(sppt_wts, skeb_wts, t_rp1, t_rp2) real(kind_dbl_prec), intent(inout) :: sppt_wts(:,:), t_rp1(:,:), t_rp2(:,:), skeb_wts(:,:) real(kind_dbl_prec), allocatable :: tmp_wts(:,:) -if (pert_epbl .OR. do_ocnsppt) then - allocate(tmp_wts(gis_stochy_ocn%nx, gis_stochy_ocn%ny)) - if (pert_epbl) then - call get_random_pattern_scalar(rpattern_epbl1, nepbl, gis_stochy_ocn, tmp_wts) - t_rp1(:,:) = 2.0 / (1.0 + exp(-1 * tmp_wts)) - call get_random_pattern_scalar(rpattern_epbl2, nepbl, gis_stochy_ocn, tmp_wts) - t_rp2(:,:) = 2.0 / (1.0 + exp(-1 * tmp_wts)) - else - t_rp1(:,:) = 1.0 - t_rp2(:,:) = 1.0 - endif - if (do_ocnsppt) then - call get_random_pattern_scalar(rpattern_ocnsppt, nocnsppt, gis_stochy_ocn, tmp_wts) - sppt_wts = 2.0 / (1.0 + exp(-1 * tmp_wts)) - else - sppt_wts = 1.0 - endif - deallocate(tmp_wts) -else - sppt_wts(:,:) = 1.0 - t_rp1(:,:) = 1.0 - t_rp2(:,:) = 1.0 +allocate(tmp_wts(gis_stochy_ocn%nx, gis_stochy_ocn%ny)) +if (pert_epbl) then + call get_random_pattern_scalar(rpattern_epbl1, nepbl, gis_stochy_ocn, tmp_wts) + t_rp1(:,:) = 2.0 / (1.0 + exp(-1 * tmp_wts)) + call get_random_pattern_scalar(rpattern_epbl2, nepbl, gis_stochy_ocn, tmp_wts) + t_rp2(:,:) = 2.0 / (1.0 + exp(-1 * tmp_wts)) +endif +if (do_ocnsppt) then + call get_random_pattern_scalar(rpattern_ocnsppt, nocnsppt, gis_stochy_ocn, tmp_wts) + sppt_wts = 2.0 / (1.0 + exp(-1 * tmp_wts)) endif +deallocate(tmp_wts) if (do_ocnskeb) then call get_random_pattern_scalar(rpattern_ocnskeb, nocnskeb, gis_stochy_ocn_skeb, skeb_wts, normalize=.true.) -else - skeb_wts(:,:) = 1.0 endif end subroutine run_stochastic_physics_ocn diff --git a/unit_tests/compile_standalone.hera_gnu b/unit_tests/compile_standalone.hera_gnu index 46dfed8b..3011e62c 100755 --- a/unit_tests/compile_standalone.hera_gnu +++ b/unit_tests/compile_standalone.hera_gnu @@ -5,8 +5,33 @@ DEBUG=YES source ./module-setup.sh module purge module use $( pwd -P ) -if [ $DEBUG == 'YES' ]; then - module load modules.stoch_gnu_dbg +if [ $DEBUG == 'NO' ]; then +module use /scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/modulefiles +module use /scratch2/NCEPDEV/stmp1/role.epic/installs/openmpi/modulefiles +module use /scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/fms-2024.01/install/modulefiles/Core +module load stack-gcc/13.3.0 +module load stack-openmpi/4.1.6 +module load cmake/3.23.1 +module load nccmp/1.9.0.1 +module load jasper/2.0.32 + module load zlib/1.2.13 + module load libpng/1.6.37 + module load hdf5/1.14.0 + module load netcdf-c/4.9.2 + module load netcdf-fortran/4.6.1 + module load parallelio/2.5.10 + module load esmf/8.6.0 + module load fms/2024.01 + module load bacio/2.4.1 + module load crtm/2.4.0 + module load g2/3.5.1 + module load g2tmpl/1.13.0 + module load ip/4.3.0 + module load sp/2.5.0 + module load w3emc/2.10.0 + module load gftl-shared/1.6.1 + module load mapl/2.40.3-esmf-8.6.0 + module load scotch/7.0.4 else module load modules.stoch_gnu fi diff --git a/unit_tests/input.nml b/unit_tests/input.nml new file mode 100644 index 00000000..f24f2eba --- /dev/null +++ b/unit_tests/input.nml @@ -0,0 +1,101 @@ +&amip_interp_nml + interp_oi_sst = .true. + use_ncep_sst = .true. + use_ncep_ice = .false. + no_anom_sst = .false. + data_set = 'reynolds_oi', + date_out_of_range = 'climo', +/ + +&atmos_model_nml + blocksize = 32 + chksum_debug = .false. + dycore_only = .false. + fhout = 1 + fhmax = 800 + fhmaxhf = -1 + ccpp_suite = 'FV3_GFS_v15p2_coupled' +/ + +&diag_manager_nml + prepend_date = .F. +/ + +&fms_io_nml + checksum_required = .false. + max_files_r = 100, + max_files_w = 100, +/ + +&fms_nml + clock_grain = 'ROUTINE', + domains_stack_size = 40000000, + print_memory_usage = .false. +/ + +&fv_grid_nml + grid_file = 'INPUT/grid_spec.nc' +/ + +&fv_core_nml + layout = 1,4 + io_layout = 1,1 + npx = 97, + npy = 97, + ntiles = 6, + grid_type = -1 + +/ + +&gfs_physics_nml + do_spp = .true. + do_sppt = .true. + do_shum = .true. + do_skeb = .true. + lndp_type = 2 + n_var_lndp = 2 + n_var_spp = 5 + +/ + +&nam_sppperts + iseed_spp = 4,4,4,4,4 + spp_lscale = 250000.0,250000.0,250000.0,250000.0,250000.0 + spp_prt_list = 0.2,0.2,0.75,0.2,0.2 + spp_sigtop1 = 0.1,0.1,0.1,0.1,0.1 + spp_sigtop2 = 0.025,0.025,0.025,0.025,0.025 + spp_stddev_cutoff = 1.5,1.5,2.5,1.5,1.5 + spp_tau = 21600.0,21600.0,21600.0,21600.0,21600.0 + spp_var_list = 'pbl','sfc','mp','rad','gwd' +/ + +&nam_stochy + SKEBNORM=1, + SKEB_NPASS=30, + SKEB_VDOF=5, + SKEB=0.5, + SKEB_TAU=2.16E4, + SKEB_LSCALE=1000.E3, + SHUM=0.01, + SHUM_TAU=21600, + SHUM_LSCALE=500000, + SPPT=0.8,0.4,0.1 + SPPT_TAU=21600,86400,216000 + SPPT_LSCALE=500000,1000000,2000000 + SPPT_LOGIT=.TRUE., + SPPT_SFCLIMIT=.TRUE., + ISEED_SHUM=1, + ISEED_SKEB=2, + ISEED_SPPT=3, + STOCHINI=.false. + +/ +&nam_sfcperts + lndp_type=2 + lndp_var_list='vgf','smc' + lndp_prt_list=1.0,0.5 + iseed_lndp=4,5 + lndp_tau= 86400, + lndp_lscale = 250000, + lndp_model_type = 1 +/ diff --git a/unit_tests/run_standalone.sh b/unit_tests/run_standalone.sh index 2ec28588..bb81516b 100755 --- a/unit_tests/run_standalone.sh +++ b/unit_tests/run_standalone.sh @@ -1,7 +1,7 @@ #!/bin/sh #SBATCH -e err #SBATCH -o out -#SBATCH --account=gsienkf +#SBATCH --account=epic #SBATCH --qos=debug #SBATCH --nodes=1 #SBATCH --ntasks-per-node=40 @@ -25,7 +25,7 @@ fi # copy input directory if [ ! -d INPUT ]; then - cp -r /scratch2/BMC/gsienkf/Philip.Pegion/stochastic_physics_unit_tests/input_data INPUT + cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT fi mkdir -p RESTART @@ -40,6 +40,6 @@ sed -i -e "s/RES/$RES/g" input.nml sed -i -e "s/_STOCHINI_/.false./g" input.nml export OMP_NUM_THREADS=2 module list -time srun --label -n 24 standalone_stochy.x +time srun --label -n 24 standalone_stochy.x mkdir stochy_out mv workg* stochy_out diff --git a/unit_tests/standalone_stochy.F90 b/unit_tests/standalone_stochy.F90 index d91e3683..aec584c5 100644 --- a/unit_tests/standalone_stochy.F90 +++ b/unit_tests/standalone_stochy.F90 @@ -1,5 +1,5 @@ program standalone_stochy - +use mpi_f08 use stochastic_physics, only : init_stochastic_physics,run_stochastic_physics use get_stochy_pattern_mod, only : write_stoch_restart_atm @@ -16,6 +16,7 @@ program standalone_stochy implicit none integer, parameter :: nlevs=3 integer, parameter :: max_n_var_lndp = 6 +integer, parameter :: max_n_var_spp = 6 integer :: ntasks,fid integer :: nthreads integer :: ncid,xt_dim_id,yt_dim_id,time_dim_id,xt_var_id,yt_var_id,time_var_id,var_id_lat,var_id_lon,var_id_tile @@ -24,9 +25,11 @@ program standalone_stochy integer :: zt_dim_id,zt_var_id character*2 :: strid -character(len=3), dimension(max_n_var_lndp) :: lndp_var_list -real(kind=kind_dbl_prec), dimension(max_n_var_lndp) :: lndp_prt_list -include 'mpif.h' +character(len=3), allocatable :: lndp_var_list(:) +real(kind=kind_dbl_prec), allocatable :: lndp_prt_list(:) +character(len=10), allocatable :: spp_var_list(:) +real(kind=kind_dbl_prec), allocatable :: spp_prt_list(:) +real(kind=kind_dbl_prec), allocatable :: spp_stddev_cutoff(:) include 'netcdf.inc' real :: ak(nlevs+1),bk(nlevs+1) real(kind=4) :: ts,undef @@ -42,7 +45,7 @@ program standalone_stochy integer :: i1,i2,j1,npts,istart,tpt character*80 :: fname character*1 :: ntile_out_str -integer :: comm +type(MPI_Comm) :: comm real(kind=4),allocatable,dimension(:,:) :: workg,tile_number real(kind=4),allocatable,dimension(:,:,:) :: workg3d @@ -58,18 +61,19 @@ program standalone_stochy real (kind=kind_phys),allocatable :: skebu_wts (:,:,:) real (kind=kind_phys),allocatable :: skebv_wts (:,:,:) real (kind=kind_phys),allocatable :: sfc_wts (:,:,:) +real (kind=kind_phys),allocatable :: spp_wts (:,:,:,:) integer,allocatable :: blksz(:) integer :: me !< MPI rank designator integer :: root_pe !< MPI rank of root atmosphere processor real(kind=kind_phys) :: dtp !< physics timestep in seconds real(kind=kind_phys) :: fhour !< previous forecast hour real(kind=kind_phys) :: sppt_amp !< amplitude of sppt (to go to cld scheme) -logical :: do_sppt,do_shum,do_skeb,use_zmtnblck -integer :: skeb_npass,n_var_lndp, lndp_type +logical :: do_sppt,do_shum,do_skeb,do_spp,use_zmtnblck +integer :: skeb_npass,n_var_lndp, lndp_type, n_var_spp character(len=65) :: fn_nml !< namelist filename character(len=256),allocatable :: input_nml_file(:) !< character string containing full namelist - namelist /gfs_physics_nml/do_sppt,do_skeb,do_shum,lndp_type,n_var_lndp + namelist /gfs_physics_nml/do_sppt,do_skeb,do_shum,do_spp,lndp_type,n_var_lndp,n_var_spp write_this_tile=.false. ntile_out_str='0' nlunit=23 @@ -80,12 +84,21 @@ program standalone_stochy read(ntile_out_str,'(I1.1)') ntile_out open (unit=nlunit, file='input.nml', status='OLD') n_var_lndp=0 +n_var_spp =0 lndp_type=0 do_sppt=.false. do_shum=.false. do_skeb=.false. +do_spp =.false. read(nlunit,gfs_physics_nml) close(nlunit) + +!allocate spp and lnp arrays +allocate(lndp_var_list(n_var_lndp)) +allocate(lndp_prt_list(n_var_lndp)) +allocate(spp_var_list(n_var_spp)) +allocate(spp_prt_list(n_var_spp)) +allocate(spp_stddev_cutoff(n_var_spp)) ! define stuff pi=3.14159265359 undef=9.99e+20 @@ -178,11 +191,12 @@ program standalone_stochy root_pe=mpp_root_pe() allocate(input_nml_file(1)) input_nml_file='input.nml' -comm=MPI_COMM_WORLD +comm=mpi_comm_world call init_stochastic_physics(nlevs, blksz, dtp, sppt_amp, & input_nml_file, fn_nml, nlunit, xlon, xlat, do_sppt, do_shum, & do_skeb, lndp_type, n_var_lndp, use_zmtnblck, skeb_npass, & lndp_var_list, lndp_prt_list, & + n_var_spp, spp_var_list, spp_prt_list, spp_stddev_cutoff, do_spp, & ak, bk, nthreads, root_pe, comm, ierr) if (ierr .ne. 0) print *, 'ERROR init_stochastic_physics call' ! Draper - need proper error trapping here call get_outfile(fname) @@ -292,6 +306,7 @@ program standalone_stochy if (do_skeb)allocate(skebu_wts(nblks,blksz_1,nlevs)) if (do_skeb)allocate(skebv_wts(nblks,blksz_1,nlevs)) if (lndp_type > 0)allocate(sfc_wts(nblks,blksz_1,n_var_lndp)) +if (do_spp)allocate(spp_wts(nblks,blksz_1,nlevs,n_var_spp)) if (stochini) then istart=11 else @@ -301,7 +316,7 @@ program standalone_stochy do i=istart,21 ts=i/4.0 call run_stochastic_physics(nlevs, i-1, fhour, blksz, & - sppt_wts=sppt_wts, shum_wts=shum_wts, skebu_wts=skebu_wts, skebv_wts=skebv_wts, sfc_wts=sfc_wts, & + sppt_wts=sppt_wts, shum_wts=shum_wts, skebu_wts=skebu_wts, skebv_wts=skebv_wts, sfc_wts=sfc_wts, spp_wts=spp_wts, & nthreads=nthreads) if (me.EQ.0 .and. do_sppt) print*,'SPPT_WTS=',i,sppt_wts(1,1,2)