(PLATFORM-1077) Adding CI/CD capabilities and Unit Tests + Add if(allocated) checks to avoid segfaults #82

.cicd/Jenkinsfile

@@ -0,0 +1,59 @@
+pipeline {
+  agent none
+  environment {
+        GITHUB_TOKEN = credentials('GithubJenkinsNew')
+        AWS_ACCOUNT_ID = 211527314271
+    }
+  stages {
+  stage('Run Unit Tests') {
+        agent {
+          label 'hera' 
+        }
+      steps {
+          script {
+        cleanWs()
+        checkout scm
+        sh '''
+          cd unit_tests
+          echo "Running Unit Tests on ${NODE_NAME}"
+          ./compile_standalone.hera_gnu
+          cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT
+          srun --label -A epic -n 24 standalone_stochy.x
+          mkdir stochy_out
+          mv workg* stochy_out
+          cd stochy_out/
+          ls
+        '''
+      }
+    }
+  }
+ }
+   post {         
+         always {
+             node('built-in') {
+                sh ''' 
+                   GIT_OWNER=$(echo $CHANGE_URL | cut -d '/' -f4)
+                   GIT_REPO_NAME=$(echo $CHANGE_URL | cut -d '/' -f5 | cut -d '.' -f1)
+                   echo "Testing successfully completed...removing label(s)"
+          curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}"  https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}'
+               '''
+             }
+        }
+         success {     
+             node('built-in') {
+                echo 'This will run only if successful.'
+                sh ''' 
+                  aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_ACCOUNT_ID}:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'*  *succeeded*"}}'
+               '''     
+             }         
+         }             
+         failure {
+             node('built-in') { 
+                echo 'This will run only if the run was marked as unstable.'
+                sh '''      
+                  aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_ACCOUNT_ID}:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'*  *failed!*"}}'
+               '''        
+             }            
+         }                  
+     }
+}

docs/source/developers_guide.rst

@@ -12,3 +12,7 @@ If you intend to make modifications to the stochastic physics source code, there
 Full model tests
 """"""""""""""""
 The code updates are not expected to change existing results, so the full model regression tests need to be run.  All of the tests must pass, although only a sub-set of tests are needed to consider adding changes to the stochastic_physics repository: fv3_control, fv3_stochy, fv3_ccpp_control, and fv3_ccpp_stochy.  If the results are expected to change, then there needs to be scientific evidience that the change in results are what is expected.  
+
+Unit Test Pipeline
+"""""""""""""""""
+The Jenkins Pipeline for running the standalone unit test can be started automatically by adding the "run-unit-tests" Git label to the PR. Pass/Fail results can be viewed from the footer of the PR in the Git Actions. A Passing test run will show "All checks have passed". This section can be expanded and selected to show test log output within the Jenkins console. Jenkins access is required to view the console output and can be provided by the EPIC Platform team upon request.

stochastic_physics.F90

@@ -164,7 +164,7 @@ subroutine init_stochastic_physics(levs, blksz, dtp, sppt_amp, input_nml_file_in
       endif
    enddo
    if (sppt_sfclimit) then
-       do k=1,7
+       do k=1,levs
        vfact_sppt(k)=pbl_taper(k)
        enddo
    endif

unit_tests/compile_standalone.hera_gnu

@@ -5,8 +5,33 @@ DEBUG=YES
 source ./module-setup.sh
 module purge
 module use $( pwd -P )
-if [ $DEBUG == 'YES' ]; then
-   module load modules.stoch_gnu_dbg
+if [ $DEBUG == 'NO' ]; then
+module use /scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/modulefiles
+module use /scratch2/NCEPDEV/stmp1/role.epic/installs/openmpi/modulefiles
+module use /scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/fms-2024.01/install/modulefiles/Core
+module load stack-gcc/13.3.0
+module load stack-openmpi/4.1.6
+module load cmake/3.23.1
+module load nccmp/1.9.0.1
+module load jasper/2.0.32
+  module load zlib/1.2.13
+ module load libpng/1.6.37
+  module load hdf5/1.14.0
+  module load netcdf-c/4.9.2
+  module load netcdf-fortran/4.6.1
+  module load parallelio/2.5.10
+  module load esmf/8.6.0
+  module load fms/2024.01
+  module load bacio/2.4.1
+  module load crtm/2.4.0
+  module load g2/3.5.1
+  module load g2tmpl/1.13.0
+ module load ip/4.3.0
+  module load sp/2.5.0
+  module load w3emc/2.10.0
+  module load gftl-shared/1.6.1
+  module load mapl/2.40.3-esmf-8.6.0
+  module load scotch/7.0.4
 else
    module load modules.stoch_gnu
 fi

unit_tests/input.nml

@@ -0,0 +1,101 @@
+&amip_interp_nml
+       interp_oi_sst = .true.
+       use_ncep_sst = .true.
+       use_ncep_ice = .false.
+       no_anom_sst = .false.
+       data_set = 'reynolds_oi',
+       date_out_of_range = 'climo',
+/
+
+&atmos_model_nml
+       blocksize = 32
+       chksum_debug = .false.
+       dycore_only = .false.
+       fhout = 1
+       fhmax = 800
+       fhmaxhf = -1
+       ccpp_suite = 'FV3_GFS_v15p2_coupled'
+/
+
+&diag_manager_nml
+       prepend_date = .F.
+/
+
+&fms_io_nml
+       checksum_required   = .false.
+       max_files_r = 100,
+       max_files_w = 100,
+/
+
+&fms_nml
+       clock_grain = 'ROUTINE',
+       domains_stack_size = 40000000,
+       print_memory_usage = .false.
+/
+
+&fv_grid_nml
+       grid_file = 'INPUT/grid_spec.nc'
+/
+
+&fv_core_nml
+       layout   = 1,4
+       io_layout = 1,1
+       npx      = 97,
+       npy      = 97,
+       ntiles   = 6,
+       grid_type = -1
+
+/
+
+&gfs_physics_nml
+       do_spp         = .true.
+       do_sppt        = .true.
+       do_shum        = .true.
+       do_skeb        = .true.
+       lndp_type      = 2
+       n_var_lndp     = 2
+       n_var_spp      = 5
+
+/
+
+&nam_sppperts       
+    iseed_spp = 4,4,4,4,4
+    spp_lscale = 250000.0,250000.0,250000.0,250000.0,250000.0
+    spp_prt_list = 0.2,0.2,0.75,0.2,0.2
+    spp_sigtop1 = 0.1,0.1,0.1,0.1,0.1
+    spp_sigtop2 = 0.025,0.025,0.025,0.025,0.025
+    spp_stddev_cutoff = 1.5,1.5,2.5,1.5,1.5
+    spp_tau = 21600.0,21600.0,21600.0,21600.0,21600.0
+    spp_var_list = 'pbl','sfc','mp','rad','gwd'    
+/
+
+&nam_stochy
+  SKEBNORM=1,
+  SKEB_NPASS=30,
+  SKEB_VDOF=5,
+  SKEB=0.5,
+  SKEB_TAU=2.16E4,
+  SKEB_LSCALE=1000.E3,
+  SHUM=0.01,
+  SHUM_TAU=21600,
+  SHUM_LSCALE=500000,
+  SPPT=0.8,0.4,0.1
+  SPPT_TAU=21600,86400,216000
+  SPPT_LSCALE=500000,1000000,2000000
+  SPPT_LOGIT=.TRUE.,
+  SPPT_SFCLIMIT=.TRUE.,
+  ISEED_SHUM=1,
+  ISEED_SKEB=2,
+  ISEED_SPPT=3,
+  STOCHINI=.false.
+
+/
+&nam_sfcperts
+  lndp_type=2
+  lndp_var_list='vgf','smc'
+  lndp_prt_list=1.0,0.5
+  iseed_lndp=4,5
+  lndp_tau= 86400,
+  lndp_lscale = 250000,
+  lndp_model_type = 1
+/

unit_tests/run_standalone.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 #SBATCH -e err
 #SBATCH -o out
-#SBATCH --account=gsienkf
+#SBATCH --account=epic
 #SBATCH --qos=debug
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=40
@@ -25,7 +25,7 @@ fi
 
 # copy input directory
 if [ ! -d INPUT ]; then
-   cp -r /scratch2/BMC/gsienkf/Philip.Pegion/stochastic_physics_unit_tests/input_data INPUT
+   cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT
 fi
 mkdir -p RESTART
 
@@ -40,6 +40,6 @@ sed -i -e "s/RES/$RES/g" input.nml
 sed -i -e "s/_STOCHINI_/.false./g" input.nml
 export OMP_NUM_THREADS=2
 module list
-time srun --label -n 24 standalone_stochy.x 
+time srun --label -n 24 standalone_stochy.x
 mkdir stochy_out
 mv workg* stochy_out

unit_tests/standalone_stochy.F90

@@ -1,5 +1,5 @@
 program  standalone_stochy
-
+use mpi_f08
 use stochastic_physics,  only : init_stochastic_physics,run_stochastic_physics
 use get_stochy_pattern_mod,  only : write_stoch_restart_atm
 
@@ -16,6 +16,7 @@ program  standalone_stochy
 implicit none
 integer, parameter      :: nlevs=3
 integer, parameter :: max_n_var_lndp = 6
+integer, parameter :: max_n_var_spp  = 6
 integer                 :: ntasks,fid
 integer                 :: nthreads
 integer                 :: ncid,xt_dim_id,yt_dim_id,time_dim_id,xt_var_id,yt_var_id,time_var_id,var_id_lat,var_id_lon,var_id_tile
@@ -24,9 +25,11 @@ program  standalone_stochy
 integer                 :: zt_dim_id,zt_var_id
 character*2             :: strid
 
-character(len=3), dimension(max_n_var_lndp)         ::  lndp_var_list
-real(kind=kind_dbl_prec), dimension(max_n_var_lndp) ::  lndp_prt_list
-include 'mpif.h'
+character(len=3),         allocatable :: lndp_var_list(:)
+real(kind=kind_dbl_prec), allocatable :: lndp_prt_list(:)
+character(len=10),        allocatable :: spp_var_list(:)
+real(kind=kind_dbl_prec), allocatable :: spp_prt_list(:)
+real(kind=kind_dbl_prec), allocatable :: spp_stddev_cutoff(:)
 include 'netcdf.inc'
 real :: ak(nlevs+1),bk(nlevs+1)
 real(kind=4) :: ts,undef
@@ -42,7 +45,7 @@ program  standalone_stochy
 integer  :: i1,i2,j1,npts,istart,tpt
 character*80 :: fname
 character*1  :: ntile_out_str
-integer :: comm
+type(MPI_Comm) :: comm
 
 real(kind=4),allocatable,dimension(:,:) :: workg,tile_number
 real(kind=4),allocatable,dimension(:,:,:) :: workg3d
@@ -58,18 +61,19 @@ program  standalone_stochy
 real (kind=kind_phys),allocatable :: skebu_wts (:,:,:)
 real (kind=kind_phys),allocatable :: skebv_wts (:,:,:)
 real (kind=kind_phys),allocatable :: sfc_wts   (:,:,:)
+real (kind=kind_phys),allocatable :: spp_wts   (:,:,:,:)
 integer,allocatable :: blksz(:)
 integer              :: me              !< MPI rank designator
 integer              :: root_pe         !< MPI rank of root atmosphere processor
 real(kind=kind_phys) :: dtp             !< physics timestep in seconds
 real(kind=kind_phys) :: fhour           !< previous forecast hour
 real(kind=kind_phys) :: sppt_amp        !< amplitude of sppt (to go to cld scheme)
-logical  :: do_sppt,do_shum,do_skeb,use_zmtnblck
-integer  ::  skeb_npass,n_var_lndp, lndp_type
+logical  :: do_sppt,do_shum,do_skeb,do_spp,use_zmtnblck
+integer  ::  skeb_npass,n_var_lndp, lndp_type, n_var_spp
 character(len=65) :: fn_nml                   !< namelist filename
 character(len=256),allocatable :: input_nml_file(:) !< character string containing full namelist
 
-      namelist /gfs_physics_nml/do_sppt,do_skeb,do_shum,lndp_type,n_var_lndp
+      namelist /gfs_physics_nml/do_sppt,do_skeb,do_shum,do_spp,lndp_type,n_var_lndp,n_var_spp
 write_this_tile=.false.
 ntile_out_str='0'
 nlunit=23
@@ -80,12 +84,21 @@ program  standalone_stochy
 read(ntile_out_str,'(I1.1)') ntile_out
 open (unit=nlunit, file='input.nml', status='OLD')
 n_var_lndp=0
+n_var_spp =0
 lndp_type=0
 do_sppt=.false.
 do_shum=.false.
 do_skeb=.false.
+do_spp =.false.
 read(nlunit,gfs_physics_nml)
 close(nlunit)
+
+!allocate spp and lnp arrays
+allocate(lndp_var_list(n_var_lndp))
+allocate(lndp_prt_list(n_var_lndp))
+allocate(spp_var_list(n_var_spp))
+allocate(spp_prt_list(n_var_spp))
+allocate(spp_stddev_cutoff(n_var_spp))
 ! define stuff
 pi=3.14159265359
 undef=9.99e+20
@@ -178,11 +191,12 @@ program  standalone_stochy
 root_pe=mpp_root_pe()
 allocate(input_nml_file(1))
 input_nml_file='input.nml'
-comm=MPI_COMM_WORLD
+comm=mpi_comm_world
 call init_stochastic_physics(nlevs, blksz, dtp, sppt_amp,                         &
      input_nml_file, fn_nml, nlunit, xlon, xlat, do_sppt, do_shum,                &
      do_skeb, lndp_type, n_var_lndp, use_zmtnblck, skeb_npass, &
      lndp_var_list, lndp_prt_list,    &
+     n_var_spp, spp_var_list, spp_prt_list, spp_stddev_cutoff, do_spp, &
      ak, bk, nthreads, root_pe, comm, ierr)
 if (ierr .ne. 0) print *, 'ERROR init_stochastic_physics call' ! Draper - need proper error trapping here
 call get_outfile(fname)
@@ -292,6 +306,7 @@ program  standalone_stochy
 if (do_skeb)allocate(skebu_wts(nblks,blksz_1,nlevs))
 if (do_skeb)allocate(skebv_wts(nblks,blksz_1,nlevs))
 if (lndp_type > 0)allocate(sfc_wts(nblks,blksz_1,n_var_lndp))
+if (do_spp)allocate(spp_wts(nblks,blksz_1,nlevs,n_var_spp))
 if (stochini) then
    istart=11
 else
@@ -301,7 +316,7 @@ program  standalone_stochy
 do i=istart,21
    ts=i/4.0
    call run_stochastic_physics(nlevs, i-1, fhour, blksz, &
-                               sppt_wts=sppt_wts, shum_wts=shum_wts, skebu_wts=skebu_wts, skebv_wts=skebv_wts, sfc_wts=sfc_wts, &
+                               sppt_wts=sppt_wts, shum_wts=shum_wts, skebu_wts=skebu_wts, skebv_wts=skebv_wts, sfc_wts=sfc_wts, spp_wts=spp_wts, &
                                nthreads=nthreads)
 
    if (me.EQ.0 .and. do_sppt) print*,'SPPT_WTS=',i,sppt_wts(1,1,2)