(PLATFORM-1077) Adding CI/CD capabilities and Unit Tests + Add if(allocated) checks to avoid segfaults #82

.cicd/Jenkinsfile

@@ -0,0 +1,59 @@
+pipeline {
+  agent none
+  environment {
+        GITHUB_TOKEN = credentials('GithubJenkinsNew')
+        AWS_ACCOUNT_ID = credentials('AWS_PROD_ACCOUNT_ID')
+    }
+  stages {
+  stage('Run Unit Tests') {
+        agent {
+          label 'hera' 
+        }
+      steps {
+          script {
+        cleanWs()
+        checkout scm
+        sh '''
+          cd unit_tests
+          echo "Running Unit Tests on ${NODE_NAME}"
+          ./compile_standalone.hera_gnu
+          cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT
+          srun --label -A epic -n 24 standalone_stochy.x
+          mkdir stochy_out
+          mv workg* stochy_out
+          cd stochy_out/
+          ls
+        '''
+      }
+    }
+  }
+ }
+   post {         
+         always {
+             node('built-in') {
+                sh ''' 
+                   GIT_OWNER=$(echo $CHANGE_URL | cut -d '/' -f4)
+                   GIT_REPO_NAME=$(echo $CHANGE_URL | cut -d '/' -f5 | cut -d '.' -f1)
+                   echo "Testing successfully completed...removing label(s)"
+          curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}"  https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels -d '{"labels":["run-unit-tests"]}'
+               '''
+             }
+        }
+         success {     
+             node('built-in') {
+                echo 'This will run only if successful.'
+                sh ''' 
+                  aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_ACCOUNT_ID}:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'*  *succeeded*"}}'
+               '''     
+             }         
+         }             
+         failure {
+             node('built-in') { 
+                echo 'This will run only if the run was marked as unstable.'
+                sh '''      
+                  aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_ACCOUNT_ID}:Jenkins-CICD-Notifications" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'*  *failed!*"}}'
+               '''        
+             }            
+         }                  
+     }
+}

docs/source/developers_guide.rst

@@ -12,3 +12,7 @@ If you intend to make modifications to the stochastic physics source code, there
 Full model tests
 """"""""""""""""
 The code updates are not expected to change existing results, so the full model regression tests need to be run.  All of the tests must pass, although only a sub-set of tests are needed to consider adding changes to the stochastic_physics repository: fv3_control, fv3_stochy, fv3_ccpp_control, and fv3_ccpp_stochy.  If the results are expected to change, then there needs to be scientific evidience that the change in results are what is expected.  
+
+Unit Test Pipeline
+"""""""""""""""""
+The Jenkins Pipeline for running the standalone unit test can be started automatically by adding the "run-unit-tests" Git label to the PR. Pass/Fail results can be viewed from the footer of the PR in the Git Actions. A Passing test run will show "All checks have passed". This section can be expanded and selected to show test log output within the Jenkins console. Jenkins access is required to view the console output and can be provided by the EPIC Platform team upon request.

stochastic_physics.F90

@@ -164,9 +164,15 @@ subroutine init_stochastic_physics(levs, blksz, dtp, sppt_amp, input_nml_file_in
       endif
    enddo
    if (sppt_sfclimit) then
-       do k=1,7
-       vfact_sppt(k)=pbl_taper(k)
-       enddo
+      if (levs .le. 7) then
+         do k=1,levs
+            vfact_sppt(k)=pbl_taper(k)
+         enddo
+      else
+         do k=1,7
+            vfact_sppt(k)=pbl_taper(k)
+         enddo
+      endif
    endif
    if (is_rootpe()) then
       do k=1,levs
@@ -525,34 +531,21 @@ subroutine run_stochastic_physics_ocn(sppt_wts, skeb_wts, t_rp1, t_rp2)
 real(kind_dbl_prec), intent(inout) :: sppt_wts(:,:), t_rp1(:,:), t_rp2(:,:), skeb_wts(:,:)
 real(kind_dbl_prec), allocatable :: tmp_wts(:,:)
 
-if (pert_epbl .OR. do_ocnsppt) then
-   allocate(tmp_wts(gis_stochy_ocn%nx, gis_stochy_ocn%ny))
-   if (pert_epbl) then
-      call get_random_pattern_scalar(rpattern_epbl1, nepbl, gis_stochy_ocn, tmp_wts)
-      t_rp1(:,:) = 2.0 / (1.0 + exp(-1 * tmp_wts))
-      call get_random_pattern_scalar(rpattern_epbl2, nepbl, gis_stochy_ocn, tmp_wts)
-      t_rp2(:,:) = 2.0 / (1.0 + exp(-1 * tmp_wts))
-   else
-      t_rp1(:,:) = 1.0
-      t_rp2(:,:) = 1.0
-   endif
-   if (do_ocnsppt) then
-      call get_random_pattern_scalar(rpattern_ocnsppt, nocnsppt, gis_stochy_ocn, tmp_wts)
-      sppt_wts = 2.0 / (1.0 + exp(-1 * tmp_wts))
-   else
-      sppt_wts = 1.0
-   endif
-   deallocate(tmp_wts)
-else
-   sppt_wts(:,:) = 1.0
-   t_rp1(:,:) = 1.0
-   t_rp2(:,:) = 1.0
+allocate(tmp_wts(gis_stochy_ocn%nx, gis_stochy_ocn%ny))
+if (pert_epbl) then
+  call get_random_pattern_scalar(rpattern_epbl1, nepbl, gis_stochy_ocn, tmp_wts)
+  t_rp1(:,:) = 2.0 / (1.0 + exp(-1 * tmp_wts))
+  call get_random_pattern_scalar(rpattern_epbl2, nepbl, gis_stochy_ocn, tmp_wts)
+  t_rp2(:,:) = 2.0 / (1.0 + exp(-1 * tmp_wts))
+endif
+if (do_ocnsppt) then
+  call get_random_pattern_scalar(rpattern_ocnsppt, nocnsppt, gis_stochy_ocn, tmp_wts)
+  sppt_wts = 2.0 / (1.0 + exp(-1 * tmp_wts))
 endif
+deallocate(tmp_wts)
 
 if (do_ocnskeb) then
    call get_random_pattern_scalar(rpattern_ocnskeb, nocnskeb, gis_stochy_ocn_skeb, skeb_wts, normalize=.true.)
-else
-   skeb_wts(:,:) = 1.0
 endif
 
 end subroutine run_stochastic_physics_ocn

unit_tests/compile_standalone.hera_gnu

@@ -5,8 +5,33 @@ DEBUG=YES
 source ./module-setup.sh
 module purge
 module use $( pwd -P )
-if [ $DEBUG == 'YES' ]; then
-   module load modules.stoch_gnu_dbg
+if [ $DEBUG == 'NO' ]; then
+module use /scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/modulefiles
+module use /scratch2/NCEPDEV/stmp1/role.epic/installs/openmpi/modulefiles
+module use /scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/fms-2024.01/install/modulefiles/Core
+module load stack-gcc/13.3.0
+module load stack-openmpi/4.1.6
+module load cmake/3.23.1
+module load nccmp/1.9.0.1
+module load jasper/2.0.32
+  module load zlib/1.2.13
+ module load libpng/1.6.37
+  module load hdf5/1.14.0
+  module load netcdf-c/4.9.2
+  module load netcdf-fortran/4.6.1
+  module load parallelio/2.5.10
+  module load esmf/8.6.0
+  module load fms/2024.01
+  module load bacio/2.4.1
+  module load crtm/2.4.0
+  module load g2/3.5.1
+  module load g2tmpl/1.13.0
+ module load ip/4.3.0
+  module load sp/2.5.0
+  module load w3emc/2.10.0
+  module load gftl-shared/1.6.1
+  module load mapl/2.40.3-esmf-8.6.0
+  module load scotch/7.0.4
 else
    module load modules.stoch_gnu
 fi

unit_tests/input.nml

@@ -0,0 +1,101 @@
+&amip_interp_nml
+       interp_oi_sst = .true.
+       use_ncep_sst = .true.
+       use_ncep_ice = .false.
+       no_anom_sst = .false.
+       data_set = 'reynolds_oi',
+       date_out_of_range = 'climo',
+/
+
+&atmos_model_nml
+       blocksize = 32
+       chksum_debug = .false.
+       dycore_only = .false.
+       fhout = 1
+       fhmax = 800
+       fhmaxhf = -1
+       ccpp_suite = 'FV3_GFS_v15p2_coupled'
+/
+
+&diag_manager_nml
+       prepend_date = .F.
+/
+
+&fms_io_nml
+       checksum_required   = .false.
+       max_files_r = 100,
+       max_files_w = 100,
+/
+
+&fms_nml
+       clock_grain = 'ROUTINE',
+       domains_stack_size = 40000000,
+       print_memory_usage = .false.
+/
+
+&fv_grid_nml
+       grid_file = 'INPUT/grid_spec.nc'
+/
+
+&fv_core_nml
+       layout   = 1,4
+       io_layout = 1,1
+       npx      = 97,
+       npy      = 97,
+       ntiles   = 6,
+       grid_type = -1
+
+/
+
+&gfs_physics_nml
+       do_spp         = .true.
+       do_sppt        = .true.
+       do_shum        = .true.
+       do_skeb        = .true.
+       lndp_type      = 2
+       n_var_lndp     = 2
+       n_var_spp      = 5
+
+/
+
+&nam_sppperts       
+    iseed_spp = 4,4,4,4,4
+    spp_lscale = 250000.0,250000.0,250000.0,250000.0,250000.0
+    spp_prt_list = 0.2,0.2,0.75,0.2,0.2
+    spp_sigtop1 = 0.1,0.1,0.1,0.1,0.1
+    spp_sigtop2 = 0.025,0.025,0.025,0.025,0.025
+    spp_stddev_cutoff = 1.5,1.5,2.5,1.5,1.5
+    spp_tau = 21600.0,21600.0,21600.0,21600.0,21600.0
+    spp_var_list = 'pbl','sfc','mp','rad','gwd'    
+/
+
+&nam_stochy
+  SKEBNORM=1,
+  SKEB_NPASS=30,
+  SKEB_VDOF=5,
+  SKEB=0.5,
+  SKEB_TAU=2.16E4,
+  SKEB_LSCALE=1000.E3,
+  SHUM=0.01,
+  SHUM_TAU=21600,
+  SHUM_LSCALE=500000,
+  SPPT=0.8,0.4,0.1
+  SPPT_TAU=21600,86400,216000
+  SPPT_LSCALE=500000,1000000,2000000
+  SPPT_LOGIT=.TRUE.,
+  SPPT_SFCLIMIT=.TRUE.,
+  ISEED_SHUM=1,
+  ISEED_SKEB=2,
+  ISEED_SPPT=3,
+  STOCHINI=.false.
+
+/
+&nam_sfcperts
+  lndp_type=2
+  lndp_var_list='vgf','smc'
+  lndp_prt_list=1.0,0.5
+  iseed_lndp=4,5
+  lndp_tau= 86400,
+  lndp_lscale = 250000,
+  lndp_model_type = 1
+/

unit_tests/run_standalone.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 #SBATCH -e err
 #SBATCH -o out
-#SBATCH --account=gsienkf
+#SBATCH --account=epic
 #SBATCH --qos=debug
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=40
@@ -25,7 +25,7 @@ fi
 
 # copy input directory
 if [ ! -d INPUT ]; then
-   cp -r /scratch2/BMC/gsienkf/Philip.Pegion/stochastic_physics_unit_tests/input_data INPUT
+   cp -r /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20221101/FV3_input_data/INPUT INPUT
 fi
 mkdir -p RESTART
 
@@ -40,6 +40,6 @@ sed -i -e "s/RES/$RES/g" input.nml
 sed -i -e "s/_STOCHINI_/.false./g" input.nml
 export OMP_NUM_THREADS=2
 module list
-time srun --label -n 24 standalone_stochy.x 
+time srun --label -n 24 standalone_stochy.x
 mkdir stochy_out
 mv workg* stochy_out