diff --git a/CMakeLists.txt b/CMakeLists.txt index 39ebf33739..ca83984370 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,6 +155,9 @@ endif() ############################################################################### include(fms_files.cmake) add_library(fms ${fms_src_files}) +# stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs +set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0") + target_include_directories(fms PUBLIC $ $ $ diff --git a/FMS b/FMS index d34bb9fa11..f61416fef6 160000 --- a/FMS +++ b/FMS @@ -1 +1 @@ -Subproject commit d34bb9fa1107c5cc08a552eefd7b127a22c3a49d +Subproject commit f61416fef691d9ba39a40df1ce72aa574f54c390 diff --git a/fms_files.cmake b/fms_files.cmake index 0aade9f3d2..6756cd323d 100644 --- a/fms_files.cmake +++ b/fms_files.cmake @@ -2,7 +2,6 @@ list(APPEND fms_src_files FMS/amip_interp/amip_interp.F90 FMS/astronomy/astronomy.F90 FMS/axis_utils/axis_utils.F90 - FMS/axis_utils/axis_utils2.F90 FMS/block_control/block_control.F90 FMS/column_diagnostics/column_diagnostics.F90 FMS/constants/constants.F90 @@ -15,6 +14,7 @@ list(APPEND fms_src_files FMS/diag_manager/diag_data.F90 FMS/diag_manager/diag_grid.F90 FMS/diag_manager/diag_manager.F90 + FMS/diag_manager/diag_manifest.F90 FMS/diag_manager/diag_output.F90 FMS/diag_manager/diag_table.F90 FMS/diag_manager/diag_util.F90 @@ -27,16 +27,12 @@ list(APPEND fms_src_files FMS/drifters/quicksort.F90 FMS/exchange/stock_constants.F90 FMS/exchange/xgrid.F90 + FMS/fft/fft.F90 + FMS/fft/fft99.F90 FMS/field_manager/field_manager.F90 FMS/field_manager/fm_util.F90 FMS/fms/fms.F90 FMS/fms/fms_io.F90 - FMS/fms2_io/blackboxio.F90 - FMS/fms2_io/fms2_io.F90 - FMS/fms2_io/fms_io_utils.F90 - FMS/fms2_io/fms_netcdf_domain_io.F90 - FMS/fms2_io/fms_netcdf_unstructured_domain_io.F90 - FMS/fms2_io/netcdf_io.F90 FMS/horiz_interp/horiz_interp.F90 FMS/horiz_interp/horiz_interp_bicubic.F90 FMS/horiz_interp/horiz_interp_bilinear.F90 @@ -50,7 +46,6 @@ list(APPEND fms_src_files FMS/mosaic/gradient.F90 FMS/mosaic/grid.F90 FMS/mosaic/mosaic.F90 - FMS/mosaic/mosaic2.F90 FMS/mpp/mpp.F90 FMS/mpp/mpp_data.F90 FMS/mpp/mpp_domains.F90 @@ -65,9 +60,9 @@ list(APPEND fms_src_files FMS/random_numbers/random_numbers.F90 FMS/sat_vapor_pres/sat_vapor_pres.F90 FMS/sat_vapor_pres/sat_vapor_pres_k.F90 + FMS/station_data/station_data.F90 FMS/time_interp/time_interp.F90 FMS/time_interp/time_interp_external.F90 - FMS/time_interp/time_interp_external2.F90 FMS/time_manager/get_cal_time.F90 FMS/time_manager/time_manager.F90 FMS/topography/gaussian_topog.F90 @@ -75,12 +70,12 @@ list(APPEND fms_src_files FMS/tracer_manager/tracer_manager.F90 FMS/tridiagonal/tridiagonal.F90 + FMS/memutils/memuse.c FMS/mosaic/create_xgrid.c FMS/mosaic/gradient_c2l.c FMS/mosaic/interp.c FMS/mosaic/mosaic_util.c FMS/mosaic/read_mosaic.c - FMS/mpp/mpp_memuse.c FMS/mpp/nsclock.c FMS/mpp/threadloc.c ) diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log index 268ceb7479..784327adf1 100644 --- a/tests/RegressionTests_hera.gnu.log +++ b/tests/RegressionTests_hera.gnu.log @@ -1,9 +1,9 @@ -Wed Jul 29 21:28:54 UTC 2020 +Wed Aug 5 13:07:19 UTC 2020 Start Regression test -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_gfdlmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_gfdlmp_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_gfdlmp_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_gfdlmp_prod Checking test 001 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -16,7 +16,7 @@ Checking test 001 fv3_ccpp_gfdlmp results .... Comparing dynf000.nemsio .........OK Comparing dynf024.nemsio .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -50,8 +50,8 @@ Checking test 001 fv3_ccpp_gfdlmp results .... Test 001 fv3_ccpp_gfdlmp PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_gfs_v15p2_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_gfs_v15p2_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_gfs_v15p2_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_gfs_v15p2_prod Checking test 002 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -84,7 +84,7 @@ Checking test 002 fv3_ccpp_gfs_v15p2 results .... Comparing dynf024.tile5.nc .........OK Comparing dynf024.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -118,8 +118,8 @@ Checking test 002 fv3_ccpp_gfs_v15p2 results .... Test 002 fv3_ccpp_gfs_v15p2 PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_gfs_v16beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_gfs_v16beta_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_gfs_v16beta_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_gfs_v16beta_prod Checking test 003 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -152,7 +152,7 @@ Checking test 003 fv3_ccpp_gfs_v16beta results .... Comparing dynf024.tile5.nc .........OK Comparing dynf024.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -186,8 +186,8 @@ Checking test 003 fv3_ccpp_gfs_v16beta results .... Test 003 fv3_ccpp_gfs_v16beta PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_gfs_v16beta_flake_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_gfs_v16beta_flake_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_gfs_v16beta_flake_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_gfs_v16beta_flake_prod Checking test 004 fv3_ccpp_gfs_v16beta_flake results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -220,7 +220,7 @@ Checking test 004 fv3_ccpp_gfs_v16beta_flake results .... Comparing dynf024.tile5.nc .........OK Comparing dynf024.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -254,8 +254,8 @@ Checking test 004 fv3_ccpp_gfs_v16beta_flake results .... Test 004 fv3_ccpp_gfs_v16beta_flake PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_rrtmgp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_rrtmgp_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_rrtmgp_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_rrtmgp_prod Checking test 005 fv3_ccpp_rrtmgp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -288,7 +288,7 @@ Checking test 005 fv3_ccpp_rrtmgp results .... Comparing dynf024.tile5.nc .........OK Comparing dynf024.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -322,8 +322,8 @@ Checking test 005 fv3_ccpp_rrtmgp results .... Test 005 fv3_ccpp_rrtmgp PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_gsd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_gsd_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_gsd_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_gsd_prod Checking test 006 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -380,7 +380,7 @@ Checking test 006 fv3_ccpp_gsd results .... Comparing dynf048.tile5.nc .........OK Comparing dynf048.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -414,8 +414,8 @@ Checking test 006 fv3_ccpp_gsd results .... Test 006 fv3_ccpp_gsd PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_thompson_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_thompson_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_thompson_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_thompson_prod Checking test 007 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -448,7 +448,7 @@ Checking test 007 fv3_ccpp_thompson results .... Comparing dynf024.tile5.nc .........OK Comparing dynf024.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -482,8 +482,8 @@ Checking test 007 fv3_ccpp_thompson results .... Test 007 fv3_ccpp_thompson PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_thompson_no_aero_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_thompson_no_aero_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_thompson_no_aero_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_thompson_no_aero_prod Checking test 008 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -516,7 +516,7 @@ Checking test 008 fv3_ccpp_thompson_no_aero results .... Comparing dynf024.tile5.nc .........OK Comparing dynf024.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -550,8 +550,8 @@ Checking test 008 fv3_ccpp_thompson_no_aero results .... Test 008 fv3_ccpp_thompson_no_aero PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_rrfs_v1beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_rrfs_v1beta_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_rrfs_v1beta_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_rrfs_v1beta_prod Checking test 009 fv3_ccpp_rrfs_v1beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -608,7 +608,7 @@ Checking test 009 fv3_ccpp_rrfs_v1beta results .... Comparing dynf048.tile5.nc .........OK Comparing dynf048.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -642,14 +642,14 @@ Checking test 009 fv3_ccpp_rrfs_v1beta results .... Test 009 fv3_ccpp_rrfs_v1beta PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_control_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_control_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_control_debug_prod Checking test 010 fv3_ccpp_control_debug results .... Test 010 fv3_ccpp_control_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_gfs_v15p2_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_gfs_v15p2_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_gfs_v15p2_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_gfs_v15p2_debug_prod Checking test 011 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -682,7 +682,7 @@ Checking test 011 fv3_ccpp_gfs_v15p2_debug results .... Comparing dynf006.tile5.nc .........OK Comparing dynf006.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -716,8 +716,8 @@ Checking test 011 fv3_ccpp_gfs_v15p2_debug results .... Test 011 fv3_ccpp_gfs_v15p2_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_gfs_v16beta_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_gfs_v16beta_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_gfs_v16beta_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_gfs_v16beta_debug_prod Checking test 012 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -750,7 +750,7 @@ Checking test 012 fv3_ccpp_gfs_v16beta_debug results .... Comparing dynf006.tile5.nc .........OK Comparing dynf006.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -784,8 +784,8 @@ Checking test 012 fv3_ccpp_gfs_v16beta_debug results .... Test 012 fv3_ccpp_gfs_v16beta_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_rrtmgp_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_rrtmgp_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_rrtmgp_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_rrtmgp_debug_prod Checking test 013 fv3_ccpp_rrtmgp_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -818,7 +818,7 @@ Checking test 013 fv3_ccpp_rrtmgp_debug results .... Comparing dynf006.tile5.nc .........OK Comparing dynf006.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -852,8 +852,8 @@ Checking test 013 fv3_ccpp_rrtmgp_debug results .... Test 013 fv3_ccpp_rrtmgp_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/GNU/fv3_multigases_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_305591/fv3_ccpp_multigases_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/GNU/fv3_multigases_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_247246/fv3_ccpp_multigases_prod Checking test 014 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -886,7 +886,7 @@ Checking test 014 fv3_ccpp_multigases results .... Comparing dynf006.tile5.nc .........OK Comparing dynf006.tile6.nc .........OK Comparing RESTART/coupler.res .........OK - Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.nc ............SKIP for gnu compilers Comparing RESTART/fv_core.res.tile1.nc .........OK Comparing RESTART/fv_core.res.tile2.nc .........OK Comparing RESTART/fv_core.res.tile3.nc .........OK @@ -927,5 +927,5 @@ Test 014 fv3_ccpp_multigases PASS REGRESSION TEST WAS SUCCESSFUL -Wed Jul 29 21:58:04 UTC 2020 -Elapsed time: 00h:29m:12s. Have a nice day! +Wed Aug 5 13:38:21 UTC 2020 +Elapsed time: 00h:31m:03s. Have a nice day! diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log index bf7ba879da..ebb99fb12f 100644 --- a/tests/RegressionTests_hera.intel.log +++ b/tests/RegressionTests_hera.intel.log @@ -1,9 +1,9 @@ -Wed Jul 29 21:27:52 UTC 2020 +Wed Aug 5 12:33:45 UTC 2020 Start Regression test -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_control_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -70,8 +70,8 @@ Checking test 001 fv3_ccpp_control results .... Test 001 fv3_ccpp_control PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_decomp_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -138,8 +138,8 @@ Checking test 002 fv3_ccpp_decomp results .... Test 002 fv3_ccpp_decomp PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_2threads_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -206,8 +206,8 @@ Checking test 003 fv3_ccpp_2threads results .... Test 003 fv3_ccpp_2threads PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_restart_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_restart_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_restart_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -274,8 +274,8 @@ Checking test 004 fv3_ccpp_restart results .... Test 004 fv3_ccpp_restart PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_read_inc_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_read_inc_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_read_inc_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -342,8 +342,8 @@ Checking test 005 fv3_ccpp_read_inc results .... Test 005 fv3_ccpp_read_inc PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_wrtGauss_netcdf_esmf_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -390,8 +390,8 @@ Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_netcdf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_wrtGauss_netcdf_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_netcdf_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -438,8 +438,8 @@ Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Test 007 fv3_ccpp_wrtGauss_netcdf PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGlatlon_netcdf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_wrtGlatlon_netcdf_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGlatlon_netcdf_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -486,8 +486,8 @@ Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_nemsio_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_wrtGauss_nemsio_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_nemsio_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -534,8 +534,8 @@ Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Test 009 fv3_ccpp_wrtGauss_nemsio PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_wrtGauss_nemsio_c192_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_nemsio_c192_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -582,8 +582,8 @@ Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_stochy_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_stochy_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_stochy_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -650,8 +650,8 @@ Checking test 011 fv3_ccpp_stochy results .... Test 011 fv3_ccpp_stochy PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_iau_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_iau_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_iau_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -718,8 +718,8 @@ Checking test 012 fv3_ccpp_iau results .... Test 012 fv3_ccpp_iau PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_ca_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_ca_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_ca_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_ca_prod Checking test 013 fv3_ccpp_ca results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -786,8 +786,8 @@ Checking test 013 fv3_ccpp_ca results .... Test 013 fv3_ccpp_ca PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_lheatstrg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_lheatstrg_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_lheatstrg_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_lheatstrg_prod Checking test 014 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -834,8 +834,8 @@ Checking test 014 fv3_ccpp_lheatstrg results .... Test 014 fv3_ccpp_lheatstrg PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfdlmprad_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfdlmprad_prod Checking test 015 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -883,8 +883,8 @@ Checking test 015 fv3_ccpp_gfdlmprad results .... Test 015 fv3_ccpp_gfdlmprad PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_atmwav_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfdlmprad_atmwav_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_atmwav_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 016 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -932,8 +932,8 @@ Checking test 016 fv3_ccpp_gfdlmprad_atmwav results .... Test 016 fv3_ccpp_gfdlmprad_atmwav PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_nemsio_c768_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_wrtGauss_nemsio_c768_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_nemsio_c768_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_wrtGauss_nemsio_c768_prod Checking test 017 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -981,8 +981,8 @@ Checking test 017 fv3_ccpp_wrtGauss_nemsio_c768 results .... Test 017 fv3_ccpp_wrtGauss_nemsio_c768 PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_multigases_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_multigases_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_multigases_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_multigases_prod Checking test 018 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1055,8 +1055,8 @@ Checking test 018 fv3_ccpp_multigases results .... Test 018 fv3_ccpp_multigases PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_32bit_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_control_32bit_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_32bit_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_control_32bit_prod Checking test 019 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1123,8 +1123,8 @@ Checking test 019 fv3_ccpp_control_32bit results .... Test 019 fv3_ccpp_control_32bit PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_stretched_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_stretched_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_stretched_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_stretched_prod Checking test 020 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1179,8 +1179,8 @@ Checking test 020 fv3_ccpp_stretched results .... Test 020 fv3_ccpp_stretched PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_stretched_nest_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_stretched_nest_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_stretched_nest_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_stretched_nest_prod Checking test 021 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1188,21 +1188,21 @@ Checking test 021 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile4.nc .........OK Comparing atmos_4xdaily.tile5.nc .........OK Comparing atmos_4xdaily.tile6.nc .........OK - Comparing atmos_4xdaily.nest02.nc .........OK + Comparing atmos_4xdaily.nest02.tile7.nc .........OK Comparing fv3_history2d.tile1.nc .........OK Comparing fv3_history2d.tile2.nc .........OK Comparing fv3_history2d.tile3.nc .........OK Comparing fv3_history2d.tile4.nc .........OK Comparing fv3_history2d.tile5.nc .........OK Comparing fv3_history2d.tile6.nc .........OK - Comparing fv3_history2d.nest02.nc .........OK + Comparing fv3_history2d.nest02.tile7.nc .........OK Comparing fv3_history.tile1.nc .........OK Comparing fv3_history.tile2.nc .........OK Comparing fv3_history.tile3.nc .........OK Comparing fv3_history.tile4.nc .........OK Comparing fv3_history.tile5.nc .........OK Comparing fv3_history.tile6.nc .........OK - Comparing fv3_history.nest02.nc .........OK + Comparing fv3_history.nest02.tile7.nc .........OK Comparing RESTART/coupler.res .........OK Comparing RESTART/fv_core.res.nc .........OK Comparing RESTART/fv_core.res.nest02.nc .........OK @@ -1246,8 +1246,8 @@ Checking test 021 fv3_ccpp_stretched_nest results .... Test 021 fv3_ccpp_stretched_nest PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_regional_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_regional_control_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_regional_control_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_regional_control_prod Checking test 022 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1257,8 +1257,8 @@ Checking test 022 fv3_ccpp_regional_control results .... Test 022 fv3_ccpp_regional_control PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_regional_restart_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_regional_restart_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_regional_restart_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_regional_restart_prod Checking test 023 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1266,8 +1266,8 @@ Checking test 023 fv3_ccpp_regional_restart results .... Test 023 fv3_ccpp_regional_restart PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_regional_quilt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_regional_quilt_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_regional_quilt_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_regional_quilt_prod Checking test 024 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1277,8 +1277,8 @@ Checking test 024 fv3_ccpp_regional_quilt results .... Test 024 fv3_ccpp_regional_quilt PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_regional_c768_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_regional_c768_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_regional_c768_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_regional_c768_prod Checking test 025 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1288,20 +1288,20 @@ Checking test 025 fv3_ccpp_regional_c768 results .... Test 025 fv3_ccpp_regional_c768 PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_control_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_control_debug_prod Checking test 026 fv3_ccpp_control_debug results .... Test 026 fv3_ccpp_control_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_stretched_nest_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_stretched_nest_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_stretched_nest_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_stretched_nest_debug_prod Checking test 027 fv3_ccpp_stretched_nest_debug results .... Test 027 fv3_ccpp_stretched_nest_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfdlmp_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmp_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfdlmp_prod Checking test 028 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1348,8 +1348,8 @@ Checking test 028 fv3_ccpp_gfdlmp results .... Test 028 fv3_ccpp_gfdlmp PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_gwd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfdlmprad_gwd_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_gwd_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfdlmprad_gwd_prod Checking test 029 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1396,8 +1396,8 @@ Checking test 029 fv3_ccpp_gfdlmprad_gwd results .... Test 029 fv3_ccpp_gfdlmprad_gwd PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_noahmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfdlmprad_noahmp_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_noahmp_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 030 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1444,8 +1444,8 @@ Checking test 030 fv3_ccpp_gfdlmprad_noahmp results .... Test 030 fv3_ccpp_gfdlmprad_noahmp PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmp_hwrfsas_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfdlmp_hwrfsas_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmp_hwrfsas_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 031 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1492,8 +1492,8 @@ Checking test 031 fv3_ccpp_gfdlmp_hwrfsas results .... Test 031 fv3_ccpp_gfdlmp_hwrfsas PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_csawmg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_csawmg_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_csawmg_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_csawmg_prod Checking test 032 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1540,8 +1540,8 @@ Checking test 032 fv3_ccpp_csawmg results .... Test 032 fv3_ccpp_csawmg PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_satmedmf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_satmedmf_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_satmedmf_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_satmedmf_prod Checking test 033 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1588,8 +1588,8 @@ Checking test 033 fv3_ccpp_satmedmf results .... Test 033 fv3_ccpp_satmedmf PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_satmedmfq_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_satmedmfq_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_satmedmfq_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_satmedmfq_prod Checking test 034 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1636,8 +1636,8 @@ Checking test 034 fv3_ccpp_satmedmfq results .... Test 034 fv3_ccpp_satmedmfq PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmp_32bit_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfdlmp_32bit_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmp_32bit_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfdlmp_32bit_prod Checking test 035 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1684,8 +1684,8 @@ Checking test 035 fv3_ccpp_gfdlmp_32bit results .... Test 035 fv3_ccpp_gfdlmp_32bit PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_32bit_post_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfdlmprad_32bit_post_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_32bit_post_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 036 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1736,8 +1736,8 @@ Checking test 036 fv3_ccpp_gfdlmprad_32bit_post results .... Test 036 fv3_ccpp_gfdlmprad_32bit_post PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_cpt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_cpt_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_cpt_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_cpt_prod Checking test 037 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1790,8 +1790,8 @@ Checking test 037 fv3_ccpp_cpt results .... Test 037 fv3_ccpp_cpt PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gsd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gsd_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gsd_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gsd_prod Checking test 038 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1882,8 +1882,8 @@ Checking test 038 fv3_ccpp_gsd results .... Test 038 fv3_ccpp_gsd PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_rap_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_rap_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_rap_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_rap_prod Checking test 039 fv3_ccpp_rap results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1950,8 +1950,8 @@ Checking test 039 fv3_ccpp_rap results .... Test 039 fv3_ccpp_rap PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_hrrr_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_hrrr_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_hrrr_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_hrrr_prod Checking test 040 fv3_ccpp_hrrr results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2018,8 +2018,8 @@ Checking test 040 fv3_ccpp_hrrr results .... Test 040 fv3_ccpp_hrrr PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_thompson_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_thompson_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_thompson_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_thompson_prod Checking test 041 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2086,8 +2086,8 @@ Checking test 041 fv3_ccpp_thompson results .... Test 041 fv3_ccpp_thompson PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_thompson_no_aero_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_thompson_no_aero_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_thompson_no_aero_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_thompson_no_aero_prod Checking test 042 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2154,8 +2154,8 @@ Checking test 042 fv3_ccpp_thompson_no_aero results .... Test 042 fv3_ccpp_thompson_no_aero PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_rrfs_v1beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_rrfs_v1beta_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_rrfs_v1beta_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_rrfs_v1beta_prod Checking test 043 fv3_ccpp_rrfs_v1beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2246,8 +2246,8 @@ Checking test 043 fv3_ccpp_rrfs_v1beta results .... Test 043 fv3_ccpp_rrfs_v1beta PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v15p2_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfs_v15p2_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v15p2_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfs_v15p2_prod Checking test 044 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2314,8 +2314,8 @@ Checking test 044 fv3_ccpp_gfs_v15p2 results .... Test 044 fv3_ccpp_gfs_v15p2 PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v16beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfs_v16beta_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v16beta_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfs_v16beta_prod Checking test 045 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2382,8 +2382,8 @@ Checking test 045 fv3_ccpp_gfs_v16beta results .... Test 045 fv3_ccpp_gfs_v16beta PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_rrtmgp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_rrtmgp_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_rrtmgp_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_rrtmgp_prod Checking test 046 fv3_ccpp_rrtmgp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2450,8 +2450,8 @@ Checking test 046 fv3_ccpp_rrtmgp results .... Test 046 fv3_ccpp_rrtmgp PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v15p2_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfs_v15p2_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v15p2_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfs_v15p2_debug_prod Checking test 047 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2518,8 +2518,8 @@ Checking test 047 fv3_ccpp_gfs_v15p2_debug results .... Test 047 fv3_ccpp_gfs_v15p2_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v16beta_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfs_v16beta_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v16beta_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfs_v16beta_debug_prod Checking test 048 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2586,8 +2586,8 @@ Checking test 048 fv3_ccpp_gfs_v16beta_debug results .... Test 048 fv3_ccpp_gfs_v16beta_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_rrtmgp_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_rrtmgp_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_rrtmgp_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_rrtmgp_debug_prod Checking test 049 fv3_ccpp_rrtmgp_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2654,8 +2654,8 @@ Checking test 049 fv3_ccpp_rrtmgp_debug results .... Test 049 fv3_ccpp_rrtmgp_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gsd_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gsd_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gsd_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gsd_debug_prod Checking test 050 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2722,8 +2722,8 @@ Checking test 050 fv3_ccpp_gsd_debug results .... Test 050 fv3_ccpp_gsd_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gsd_diag3d_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gsd_diag3d_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gsd_diag3d_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gsd_diag3d_debug_prod Checking test 051 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2790,8 +2790,8 @@ Checking test 051 fv3_ccpp_gsd_diag3d_debug results .... Test 051 fv3_ccpp_gsd_diag3d_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_thompson_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_thompson_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_thompson_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_thompson_debug_prod Checking test 052 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2858,8 +2858,8 @@ Checking test 052 fv3_ccpp_thompson_debug results .... Test 052 fv3_ccpp_thompson_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_thompson_no_aero_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_thompson_no_aero_debug_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_thompson_no_aero_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_thompson_no_aero_debug_prod Checking test 053 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2926,8 +2926,8 @@ Checking test 053 fv3_ccpp_thompson_no_aero_debug results .... Test 053 fv3_ccpp_thompson_no_aero_debug PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfsv16_csawmg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfsv16_csawmg_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfsv16_csawmg_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfsv16_csawmg_prod Checking test 054 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2974,8 +2974,8 @@ Checking test 054 fv3_ccpp_gfsv16_csawmg results .... Test 054 fv3_ccpp_gfsv16_csawmg PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfsv16_csawmgt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfsv16_csawmgt_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfsv16_csawmgt_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfsv16_csawmgt_prod Checking test 055 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3022,8 +3022,8 @@ Checking test 055 fv3_ccpp_gfsv16_csawmgt results .... Test 055 fv3_ccpp_gfsv16_csawmgt PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gocart_clm_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gocart_clm_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gocart_clm_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gocart_clm_prod Checking test 056 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3070,8 +3070,8 @@ Checking test 056 fv3_ccpp_gocart_clm results .... Test 056 fv3_ccpp_gocart_clm PASS -baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v16beta_flake_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_267638/fv3_ccpp_gfs_v16beta_flake_prod +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v16beta_flake_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dusan.Jovic/FV3_RT/rt_120230/fv3_ccpp_gfs_v16beta_flake_prod Checking test 057 fv3_ccpp_gfs_v16beta_flake results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -3139,5 +3139,5 @@ Test 057 fv3_ccpp_gfs_v16beta_flake PASS REGRESSION TEST WAS SUCCESSFUL -Wed Jul 29 21:59:49 UTC 2020 -Elapsed time: 00h:31m:58s. Have a nice day! +Wed Aug 5 13:05:40 UTC 2020 +Elapsed time: 00h:31m:57s. Have a nice day! diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log index b4b180c722..41e0dbdf92 100644 --- a/tests/RegressionTests_orion.intel.log +++ b/tests/RegressionTests_orion.intel.log @@ -1,9 +1,9 @@ -Wed Jul 29 17:13:41 CDT 2020 +Tue Aug 4 12:25:41 CDT 2020 Start Regression test -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_control_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -70,8 +70,8 @@ Checking test 001 fv3_ccpp_control results .... Test 001 fv3_ccpp_control PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_decomp_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -138,8 +138,8 @@ Checking test 002 fv3_ccpp_decomp results .... Test 002 fv3_ccpp_decomp PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_2threads_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -206,8 +206,8 @@ Checking test 003 fv3_ccpp_2threads results .... Test 003 fv3_ccpp_2threads PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_restart_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_restart_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_restart_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -274,8 +274,8 @@ Checking test 004 fv3_ccpp_restart results .... Test 004 fv3_ccpp_restart PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_read_inc_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_read_inc_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_read_inc_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -342,8 +342,8 @@ Checking test 005 fv3_ccpp_read_inc results .... Test 005 fv3_ccpp_read_inc PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_wrtGauss_netcdf_esmf_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -390,8 +390,8 @@ Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_netcdf_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_wrtGauss_netcdf_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_netcdf_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -438,8 +438,8 @@ Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Test 007 fv3_ccpp_wrtGauss_netcdf PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_nemsio_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_wrtGauss_nemsio_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_nemsio_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_wrtGauss_nemsio_prod Checking test 008 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -486,8 +486,8 @@ Checking test 008 fv3_ccpp_wrtGauss_nemsio results .... Test 008 fv3_ccpp_wrtGauss_nemsio PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_wrtGauss_nemsio_c192_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_nemsio_c192_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -534,8 +534,8 @@ Checking test 009 fv3_ccpp_wrtGauss_nemsio_c192 results .... Test 009 fv3_ccpp_wrtGauss_nemsio_c192 PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_stochy_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_stochy_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_stochy_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_stochy_prod Checking test 010 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -602,8 +602,8 @@ Checking test 010 fv3_ccpp_stochy results .... Test 010 fv3_ccpp_stochy PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_iau_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_iau_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_iau_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_iau_prod Checking test 011 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -670,8 +670,8 @@ Checking test 011 fv3_ccpp_iau results .... Test 011 fv3_ccpp_iau PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_ca_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_ca_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_ca_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_ca_prod Checking test 012 fv3_ccpp_ca results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -738,8 +738,8 @@ Checking test 012 fv3_ccpp_ca results .... Test 012 fv3_ccpp_ca PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfdlmprad_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfdlmprad_prod Checking test 013 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -787,8 +787,8 @@ Checking test 013 fv3_ccpp_gfdlmprad results .... Test 013 fv3_ccpp_gfdlmprad PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_atmwav_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfdlmprad_atmwav_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_atmwav_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 014 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -836,8 +836,8 @@ Checking test 014 fv3_ccpp_gfdlmprad_atmwav results .... Test 014 fv3_ccpp_gfdlmprad_atmwav PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_wrtGauss_nemsio_c768_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_wrtGauss_nemsio_c768_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_wrtGauss_nemsio_c768_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_wrtGauss_nemsio_c768_prod Checking test 015 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -885,8 +885,8 @@ Checking test 015 fv3_ccpp_wrtGauss_nemsio_c768 results .... Test 015 fv3_ccpp_wrtGauss_nemsio_c768 PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_32bit_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_control_32bit_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_32bit_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_control_32bit_prod Checking test 016 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -953,8 +953,8 @@ Checking test 016 fv3_ccpp_control_32bit results .... Test 016 fv3_ccpp_control_32bit PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_stretched_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_stretched_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_stretched_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_stretched_prod Checking test 017 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1009,8 +1009,8 @@ Checking test 017 fv3_ccpp_stretched results .... Test 017 fv3_ccpp_stretched PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_stretched_nest_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_stretched_nest_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_stretched_nest_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_stretched_nest_prod Checking test 018 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1018,21 +1018,21 @@ Checking test 018 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile4.nc .........OK Comparing atmos_4xdaily.tile5.nc .........OK Comparing atmos_4xdaily.tile6.nc .........OK - Comparing atmos_4xdaily.nest02.nc .........OK + Comparing atmos_4xdaily.nest02.tile7.nc .........OK Comparing fv3_history2d.tile1.nc .........OK Comparing fv3_history2d.tile2.nc .........OK Comparing fv3_history2d.tile3.nc .........OK Comparing fv3_history2d.tile4.nc .........OK Comparing fv3_history2d.tile5.nc .........OK Comparing fv3_history2d.tile6.nc .........OK - Comparing fv3_history2d.nest02.nc .........OK + Comparing fv3_history2d.nest02.tile7.nc .........OK Comparing fv3_history.tile1.nc .........OK Comparing fv3_history.tile2.nc .........OK Comparing fv3_history.tile3.nc .........OK Comparing fv3_history.tile4.nc .........OK Comparing fv3_history.tile5.nc .........OK Comparing fv3_history.tile6.nc .........OK - Comparing fv3_history.nest02.nc .........OK + Comparing fv3_history.nest02.tile7.nc .........OK Comparing RESTART/coupler.res .........OK Comparing RESTART/fv_core.res.nc .........OK Comparing RESTART/fv_core.res.nest02.nc .........OK @@ -1076,8 +1076,8 @@ Checking test 018 fv3_ccpp_stretched_nest results .... Test 018 fv3_ccpp_stretched_nest PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_regional_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_regional_control_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_regional_control_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_regional_control_prod Checking test 019 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1087,8 +1087,8 @@ Checking test 019 fv3_ccpp_regional_control results .... Test 019 fv3_ccpp_regional_control PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_regional_restart_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_regional_restart_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_regional_restart_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_regional_restart_prod Checking test 020 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1096,8 +1096,8 @@ Checking test 020 fv3_ccpp_regional_restart results .... Test 020 fv3_ccpp_regional_restart PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_regional_quilt_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_regional_quilt_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_regional_quilt_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_regional_quilt_prod Checking test 021 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1107,8 +1107,8 @@ Checking test 021 fv3_ccpp_regional_quilt results .... Test 021 fv3_ccpp_regional_quilt PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_regional_c768_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_regional_c768_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_regional_c768_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_regional_c768_prod Checking test 022 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1118,20 +1118,20 @@ Checking test 022 fv3_ccpp_regional_c768 results .... Test 022 fv3_ccpp_regional_c768 PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_control_debug_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_control_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_control_debug_prod Checking test 023 fv3_ccpp_control_debug results .... Test 023 fv3_ccpp_control_debug PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_stretched_nest_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_stretched_nest_debug_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_stretched_nest_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_stretched_nest_debug_prod Checking test 024 fv3_ccpp_stretched_nest_debug results .... Test 024 fv3_ccpp_stretched_nest_debug PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmp_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfdlmp_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmp_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfdlmp_prod Checking test 025 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1178,8 +1178,8 @@ Checking test 025 fv3_ccpp_gfdlmp results .... Test 025 fv3_ccpp_gfdlmp PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_gwd_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfdlmprad_gwd_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_gwd_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfdlmprad_gwd_prod Checking test 026 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1226,8 +1226,8 @@ Checking test 026 fv3_ccpp_gfdlmprad_gwd results .... Test 026 fv3_ccpp_gfdlmprad_gwd PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_noahmp_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfdlmprad_noahmp_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_noahmp_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 027 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1274,8 +1274,8 @@ Checking test 027 fv3_ccpp_gfdlmprad_noahmp results .... Test 027 fv3_ccpp_gfdlmprad_noahmp PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_csawmg_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_csawmg_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_csawmg_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_csawmg_prod Checking test 028 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1322,8 +1322,8 @@ Checking test 028 fv3_ccpp_csawmg results .... Test 028 fv3_ccpp_csawmg PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_satmedmf_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_satmedmf_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_satmedmf_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_satmedmf_prod Checking test 029 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1370,8 +1370,8 @@ Checking test 029 fv3_ccpp_satmedmf results .... Test 029 fv3_ccpp_satmedmf PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmp_32bit_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfdlmp_32bit_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmp_32bit_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfdlmp_32bit_prod Checking test 030 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1418,8 +1418,8 @@ Checking test 030 fv3_ccpp_gfdlmp_32bit results .... Test 030 fv3_ccpp_gfdlmp_32bit PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfdlmprad_32bit_post_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfdlmprad_32bit_post_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfdlmprad_32bit_post_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 031 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1470,8 +1470,8 @@ Checking test 031 fv3_ccpp_gfdlmprad_32bit_post results .... Test 031 fv3_ccpp_gfdlmprad_32bit_post PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_cpt_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_cpt_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_cpt_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_cpt_prod Checking test 032 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1524,8 +1524,8 @@ Checking test 032 fv3_ccpp_cpt results .... Test 032 fv3_ccpp_cpt PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gsd_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gsd_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gsd_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gsd_prod Checking test 033 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1616,8 +1616,8 @@ Checking test 033 fv3_ccpp_gsd results .... Test 033 fv3_ccpp_gsd PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_thompson_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_thompson_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_thompson_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_thompson_prod Checking test 034 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1684,8 +1684,8 @@ Checking test 034 fv3_ccpp_thompson results .... Test 034 fv3_ccpp_thompson PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_thompson_no_aero_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_thompson_no_aero_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_thompson_no_aero_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_thompson_no_aero_prod Checking test 035 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1752,8 +1752,8 @@ Checking test 035 fv3_ccpp_thompson_no_aero results .... Test 035 fv3_ccpp_thompson_no_aero PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_rrfs_v1beta_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_rrfs_v1beta_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_rrfs_v1beta_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_rrfs_v1beta_prod Checking test 036 fv3_ccpp_rrfs_v1beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1844,8 +1844,8 @@ Checking test 036 fv3_ccpp_rrfs_v1beta results .... Test 036 fv3_ccpp_rrfs_v1beta PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v15p2_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfs_v15p2_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v15p2_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfs_v15p2_prod Checking test 037 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1912,8 +1912,8 @@ Checking test 037 fv3_ccpp_gfs_v15p2 results .... Test 037 fv3_ccpp_gfs_v15p2 PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v16beta_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfs_v16beta_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v16beta_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfs_v16beta_prod Checking test 038 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1980,8 +1980,8 @@ Checking test 038 fv3_ccpp_gfs_v16beta results .... Test 038 fv3_ccpp_gfs_v16beta PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_rrtmgp_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_rrtmgp_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_rrtmgp_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_rrtmgp_prod Checking test 039 fv3_ccpp_rrtmgp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2048,8 +2048,8 @@ Checking test 039 fv3_ccpp_rrtmgp results .... Test 039 fv3_ccpp_rrtmgp PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v15p2_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfs_v15p2_debug_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v15p2_debug_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfs_v15p2_debug_prod Checking test 040 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2116,8 +2116,8 @@ Checking test 040 fv3_ccpp_gfs_v15p2_debug results .... Test 040 fv3_ccpp_gfs_v15p2_debug PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v16beta_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfs_v16beta_debug_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v16beta_debug_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfs_v16beta_debug_prod Checking test 041 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2184,8 +2184,8 @@ Checking test 041 fv3_ccpp_gfs_v16beta_debug results .... Test 041 fv3_ccpp_gfs_v16beta_debug PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_rrtmgp_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_rrtmgp_debug_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_rrtmgp_debug_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_rrtmgp_debug_prod Checking test 042 fv3_ccpp_rrtmgp_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2252,8 +2252,8 @@ Checking test 042 fv3_ccpp_rrtmgp_debug results .... Test 042 fv3_ccpp_rrtmgp_debug PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gsd_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gsd_debug_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gsd_debug_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gsd_debug_prod Checking test 043 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2320,8 +2320,8 @@ Checking test 043 fv3_ccpp_gsd_debug results .... Test 043 fv3_ccpp_gsd_debug PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_thompson_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_thompson_debug_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_thompson_debug_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_thompson_debug_prod Checking test 044 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2388,8 +2388,8 @@ Checking test 044 fv3_ccpp_thompson_debug results .... Test 044 fv3_ccpp_thompson_debug PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_thompson_no_aero_debug_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_thompson_no_aero_debug_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_thompson_no_aero_debug_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_thompson_no_aero_debug_prod Checking test 045 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2456,8 +2456,8 @@ Checking test 045 fv3_ccpp_thompson_no_aero_debug results .... Test 045 fv3_ccpp_thompson_no_aero_debug PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfsv16_csawmg_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfsv16_csawmg_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfsv16_csawmg_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfsv16_csawmg_prod Checking test 046 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2504,8 +2504,8 @@ Checking test 046 fv3_ccpp_gfsv16_csawmg results .... Test 046 fv3_ccpp_gfsv16_csawmg PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfsv16_csawmgt_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfsv16_csawmgt_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfsv16_csawmgt_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfsv16_csawmgt_prod Checking test 047 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2552,8 +2552,8 @@ Checking test 047 fv3_ccpp_gfsv16_csawmgt results .... Test 047 fv3_ccpp_gfsv16_csawmgt PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gocart_clm_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gocart_clm_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gocart_clm_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gocart_clm_prod Checking test 048 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2600,8 +2600,8 @@ Checking test 048 fv3_ccpp_gocart_clm results .... Test 048 fv3_ccpp_gocart_clm PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_gfs_v16beta_flake_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_gfs_v16beta_flake_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_gfs_v16beta_flake_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_gfs_v16beta_flake_prod Checking test 049 fv3_ccpp_gfs_v16beta_flake results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2668,8 +2668,8 @@ Checking test 049 fv3_ccpp_gfs_v16beta_flake results .... Test 049 fv3_ccpp_gfs_v16beta_flake PASS -baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200724/INTEL/fv3_multigases_ccpp -working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_213116/fv3_ccpp_multigases_prod +baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200805/INTEL/fv3_multigases_ccpp +working dir = /work/noaa/stmp/djovic/stmp/djovic/FV3_RT/rt_251867/fv3_ccpp_multigases_prod Checking test 050 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2743,5 +2743,5 @@ Test 050 fv3_ccpp_multigases PASS REGRESSION TEST WAS SUCCESSFUL -Wed Jul 29 17:46:47 CDT 2020 -Elapsed time: 00h:33m:07s. Have a nice day! +Tue Aug 4 12:56:42 CDT 2020 +Elapsed time: 00h:31m:02s. Have a nice day! diff --git a/tests/RegressionTests_wcoss_cray.log b/tests/RegressionTests_wcoss_cray.log index f9df5ac897..ddc1317f48 100644 --- a/tests/RegressionTests_wcoss_cray.log +++ b/tests/RegressionTests_wcoss_cray.log @@ -1,9 +1,9 @@ -Wed Jul 29 22:16:19 UTC 2020 +Tue Aug 4 17:26:12 UTC 2020 Start Regression test -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_control_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -70,8 +70,8 @@ Checking test 001 fv3_ccpp_control results .... Test 001 fv3_ccpp_control PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_decomp_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -138,8 +138,8 @@ Checking test 002 fv3_ccpp_decomp results .... Test 002 fv3_ccpp_decomp PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_2threads_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -206,8 +206,8 @@ Checking test 003 fv3_ccpp_2threads results .... Test 003 fv3_ccpp_2threads PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_restart_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_restart_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_restart_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -274,8 +274,8 @@ Checking test 004 fv3_ccpp_restart results .... Test 004 fv3_ccpp_restart PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_read_inc_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_read_inc_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_read_inc_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -342,8 +342,8 @@ Checking test 005 fv3_ccpp_read_inc results .... Test 005 fv3_ccpp_read_inc PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_wrtGauss_netcdf_esmf_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGauss_netcdf_esmf_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -390,8 +390,8 @@ Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGauss_netcdf_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_wrtGauss_netcdf_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGauss_netcdf_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -438,8 +438,8 @@ Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Test 007 fv3_ccpp_wrtGauss_netcdf PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGlatlon_netcdf_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_wrtGlatlon_netcdf_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGlatlon_netcdf_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -486,8 +486,8 @@ Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGauss_nemsio_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_wrtGauss_nemsio_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGauss_nemsio_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -534,8 +534,8 @@ Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Test 009 fv3_ccpp_wrtGauss_nemsio PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_wrtGauss_nemsio_c192_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGauss_nemsio_c192_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -582,8 +582,8 @@ Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_stochy_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_stochy_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_stochy_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -650,8 +650,8 @@ Checking test 011 fv3_ccpp_stochy results .... Test 011 fv3_ccpp_stochy PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_iau_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_iau_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_iau_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -718,8 +718,8 @@ Checking test 012 fv3_ccpp_iau results .... Test 012 fv3_ccpp_iau PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_lheatstrg_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_lheatstrg_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_lheatstrg_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_lheatstrg_prod Checking test 013 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -766,8 +766,8 @@ Checking test 013 fv3_ccpp_lheatstrg results .... Test 013 fv3_ccpp_lheatstrg PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfdlmprad_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfdlmprad_prod Checking test 014 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -815,8 +815,8 @@ Checking test 014 fv3_ccpp_gfdlmprad results .... Test 014 fv3_ccpp_gfdlmprad PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_atmwav_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfdlmprad_atmwav_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_atmwav_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -864,8 +864,8 @@ Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Test 015 fv3_ccpp_gfdlmprad_atmwav PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_multigases_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_multigases_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_multigases_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_multigases_prod Checking test 016 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -938,8 +938,8 @@ Checking test 016 fv3_ccpp_multigases results .... Test 016 fv3_ccpp_multigases PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_32bit_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_control_32bit_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_32bit_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_control_32bit_prod Checking test 017 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1006,8 +1006,8 @@ Checking test 017 fv3_ccpp_control_32bit results .... Test 017 fv3_ccpp_control_32bit PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_stretched_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_stretched_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_stretched_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_stretched_prod Checking test 018 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1062,8 +1062,8 @@ Checking test 018 fv3_ccpp_stretched results .... Test 018 fv3_ccpp_stretched PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_stretched_nest_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_stretched_nest_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_stretched_nest_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_stretched_nest_prod Checking test 019 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1071,21 +1071,21 @@ Checking test 019 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile4.nc .........OK Comparing atmos_4xdaily.tile5.nc .........OK Comparing atmos_4xdaily.tile6.nc .........OK - Comparing atmos_4xdaily.nest02.nc .........OK + Comparing atmos_4xdaily.nest02.tile7.nc .........OK Comparing fv3_history2d.tile1.nc .........OK Comparing fv3_history2d.tile2.nc .........OK Comparing fv3_history2d.tile3.nc .........OK Comparing fv3_history2d.tile4.nc .........OK Comparing fv3_history2d.tile5.nc .........OK Comparing fv3_history2d.tile6.nc .........OK - Comparing fv3_history2d.nest02.nc .........OK + Comparing fv3_history2d.nest02.tile7.nc .........OK Comparing fv3_history.tile1.nc .........OK Comparing fv3_history.tile2.nc .........OK Comparing fv3_history.tile3.nc .........OK Comparing fv3_history.tile4.nc .........OK Comparing fv3_history.tile5.nc .........OK Comparing fv3_history.tile6.nc .........OK - Comparing fv3_history.nest02.nc .........OK + Comparing fv3_history.nest02.tile7.nc .........OK Comparing RESTART/coupler.res .........OK Comparing RESTART/fv_core.res.nc .........OK Comparing RESTART/fv_core.res.nest02.nc .........OK @@ -1129,8 +1129,8 @@ Checking test 019 fv3_ccpp_stretched_nest results .... Test 019 fv3_ccpp_stretched_nest PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_regional_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_regional_control_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_regional_control_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_regional_control_prod Checking test 020 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1140,8 +1140,8 @@ Checking test 020 fv3_ccpp_regional_control results .... Test 020 fv3_ccpp_regional_control PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_regional_restart_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_regional_restart_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_regional_restart_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_regional_restart_prod Checking test 021 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1149,8 +1149,8 @@ Checking test 021 fv3_ccpp_regional_restart results .... Test 021 fv3_ccpp_regional_restart PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_regional_quilt_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_regional_quilt_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_regional_quilt_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_regional_quilt_prod Checking test 022 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1160,20 +1160,20 @@ Checking test 022 fv3_ccpp_regional_quilt results .... Test 022 fv3_ccpp_regional_quilt PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_control_debug_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_control_debug_prod Checking test 023 fv3_ccpp_control_debug results .... Test 023 fv3_ccpp_control_debug PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_stretched_nest_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_stretched_nest_debug_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_stretched_nest_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_stretched_nest_debug_prod Checking test 024 fv3_ccpp_stretched_nest_debug results .... Test 024 fv3_ccpp_stretched_nest_debug PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmp_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfdlmp_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmp_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfdlmp_prod Checking test 025 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1220,8 +1220,8 @@ Checking test 025 fv3_ccpp_gfdlmp results .... Test 025 fv3_ccpp_gfdlmp PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_gwd_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfdlmprad_gwd_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_gwd_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfdlmprad_gwd_prod Checking test 026 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1268,8 +1268,8 @@ Checking test 026 fv3_ccpp_gfdlmprad_gwd results .... Test 026 fv3_ccpp_gfdlmprad_gwd PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_noahmp_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfdlmprad_noahmp_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_noahmp_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 027 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1316,8 +1316,8 @@ Checking test 027 fv3_ccpp_gfdlmprad_noahmp results .... Test 027 fv3_ccpp_gfdlmprad_noahmp PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmp_hwrfsas_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfdlmp_hwrfsas_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmp_hwrfsas_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 028 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1364,8 +1364,8 @@ Checking test 028 fv3_ccpp_gfdlmp_hwrfsas results .... Test 028 fv3_ccpp_gfdlmp_hwrfsas PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_csawmg_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_csawmg_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_csawmg_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_csawmg_prod Checking test 029 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1412,8 +1412,8 @@ Checking test 029 fv3_ccpp_csawmg results .... Test 029 fv3_ccpp_csawmg PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_satmedmf_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_satmedmf_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_satmedmf_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_satmedmf_prod Checking test 030 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1460,8 +1460,8 @@ Checking test 030 fv3_ccpp_satmedmf results .... Test 030 fv3_ccpp_satmedmf PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_satmedmfq_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_satmedmfq_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_satmedmfq_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_satmedmfq_prod Checking test 031 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1508,8 +1508,8 @@ Checking test 031 fv3_ccpp_satmedmfq results .... Test 031 fv3_ccpp_satmedmfq PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmp_32bit_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfdlmp_32bit_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmp_32bit_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfdlmp_32bit_prod Checking test 032 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1556,8 +1556,8 @@ Checking test 032 fv3_ccpp_gfdlmp_32bit results .... Test 032 fv3_ccpp_gfdlmp_32bit PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_32bit_post_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfdlmprad_32bit_post_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_32bit_post_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 033 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1608,8 +1608,8 @@ Checking test 033 fv3_ccpp_gfdlmprad_32bit_post results .... Test 033 fv3_ccpp_gfdlmprad_32bit_post PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_cpt_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_cpt_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_cpt_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_cpt_prod Checking test 034 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1662,8 +1662,8 @@ Checking test 034 fv3_ccpp_cpt results .... Test 034 fv3_ccpp_cpt PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gsd_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gsd_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gsd_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gsd_prod Checking test 035 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1754,8 +1754,8 @@ Checking test 035 fv3_ccpp_gsd results .... Test 035 fv3_ccpp_gsd PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_thompson_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_thompson_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_thompson_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_thompson_prod Checking test 036 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1822,8 +1822,8 @@ Checking test 036 fv3_ccpp_thompson results .... Test 036 fv3_ccpp_thompson PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_thompson_no_aero_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_thompson_no_aero_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_thompson_no_aero_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_thompson_no_aero_prod Checking test 037 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1890,8 +1890,8 @@ Checking test 037 fv3_ccpp_thompson_no_aero results .... Test 037 fv3_ccpp_thompson_no_aero PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_rrfs_v1beta_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_rrfs_v1beta_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_rrfs_v1beta_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_rrfs_v1beta_prod Checking test 038 fv3_ccpp_rrfs_v1beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1982,8 +1982,8 @@ Checking test 038 fv3_ccpp_rrfs_v1beta results .... Test 038 fv3_ccpp_rrfs_v1beta PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gsd_debug_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gsd_debug_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gsd_debug_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gsd_debug_prod Checking test 039 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2050,8 +2050,8 @@ Checking test 039 fv3_ccpp_gsd_debug results .... Test 039 fv3_ccpp_gsd_debug PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gsd_diag3d_debug_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gsd_diag3d_debug_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gsd_diag3d_debug_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gsd_diag3d_debug_prod Checking test 040 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2118,8 +2118,8 @@ Checking test 040 fv3_ccpp_gsd_diag3d_debug results .... Test 040 fv3_ccpp_gsd_diag3d_debug PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_thompson_debug_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_thompson_debug_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_thompson_debug_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_thompson_debug_prod Checking test 041 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2186,8 +2186,8 @@ Checking test 041 fv3_ccpp_thompson_debug results .... Test 041 fv3_ccpp_thompson_debug PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_thompson_no_aero_debug_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_thompson_no_aero_debug_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_thompson_no_aero_debug_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_thompson_no_aero_debug_prod Checking test 042 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2254,8 +2254,8 @@ Checking test 042 fv3_ccpp_thompson_no_aero_debug results .... Test 042 fv3_ccpp_thompson_no_aero_debug PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfsv16_csawmg_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfsv16_csawmg_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfsv16_csawmg_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfsv16_csawmg_prod Checking test 043 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2302,8 +2302,8 @@ Checking test 043 fv3_ccpp_gfsv16_csawmg results .... Test 043 fv3_ccpp_gfsv16_csawmg PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfsv16_csawmgt_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfsv16_csawmgt_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfsv16_csawmgt_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfsv16_csawmgt_prod Checking test 044 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2350,8 +2350,8 @@ Checking test 044 fv3_ccpp_gfsv16_csawmgt results .... Test 044 fv3_ccpp_gfsv16_csawmgt PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gocart_clm_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gocart_clm_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gocart_clm_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gocart_clm_prod Checking test 045 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2398,8 +2398,8 @@ Checking test 045 fv3_ccpp_gocart_clm results .... Test 045 fv3_ccpp_gocart_clm PASS -baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfs_v16beta_flake_ccpp -working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_39112/fv3_ccpp_gfs_v16beta_flake_prod +baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfs_v16beta_flake_ccpp +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_17593/fv3_ccpp_gfs_v16beta_flake_prod Checking test 046 fv3_ccpp_gfs_v16beta_flake results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2467,5 +2467,5 @@ Test 046 fv3_ccpp_gfs_v16beta_flake PASS REGRESSION TEST WAS SUCCESSFUL -Wed Jul 29 22:57:43 UTC 2020 -Elapsed time: 00h:41m:26s. Have a nice day! +Tue Aug 4 18:13:18 UTC 2020 +Elapsed time: 00h:47m:07s. Have a nice day! diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log index 5513adb5d9..ab2bd002b8 100644 --- a/tests/RegressionTests_wcoss_dell_p3.log +++ b/tests/RegressionTests_wcoss_dell_p3.log @@ -1,9 +1,9 @@ -Wed Jul 29 21:23:35 UTC 2020 +Tue Aug 4 17:26:12 UTC 2020 Start Regression test -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_control_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -70,8 +70,8 @@ Checking test 001 fv3_ccpp_control results .... Test 001 fv3_ccpp_control PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_decomp_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -138,8 +138,8 @@ Checking test 002 fv3_ccpp_decomp results .... Test 002 fv3_ccpp_decomp PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_2threads_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -206,8 +206,8 @@ Checking test 003 fv3_ccpp_2threads results .... Test 003 fv3_ccpp_2threads PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_restart_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_restart_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_restart_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -274,8 +274,8 @@ Checking test 004 fv3_ccpp_restart results .... Test 004 fv3_ccpp_restart PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_read_inc_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_read_inc_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_read_inc_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -342,8 +342,8 @@ Checking test 005 fv3_ccpp_read_inc results .... Test 005 fv3_ccpp_read_inc PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_wrtGauss_netcdf_esmf_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGauss_netcdf_esmf_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -390,8 +390,8 @@ Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGauss_netcdf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_wrtGauss_netcdf_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGauss_netcdf_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -438,8 +438,8 @@ Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Test 007 fv3_ccpp_wrtGauss_netcdf PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGlatlon_netcdf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_wrtGlatlon_netcdf_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGlatlon_netcdf_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -486,8 +486,8 @@ Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGauss_nemsio_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_wrtGauss_nemsio_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGauss_nemsio_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -534,8 +534,8 @@ Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Test 009 fv3_ccpp_wrtGauss_nemsio PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_wrtGauss_nemsio_c192_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_wrtGauss_nemsio_c192_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -582,8 +582,8 @@ Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_stochy_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_stochy_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_stochy_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -650,8 +650,8 @@ Checking test 011 fv3_ccpp_stochy results .... Test 011 fv3_ccpp_stochy PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_iau_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_iau_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_iau_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -718,8 +718,8 @@ Checking test 012 fv3_ccpp_iau results .... Test 012 fv3_ccpp_iau PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_lheatstrg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_lheatstrg_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_lheatstrg_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_lheatstrg_prod Checking test 013 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -766,8 +766,8 @@ Checking test 013 fv3_ccpp_lheatstrg results .... Test 013 fv3_ccpp_lheatstrg PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfdlmprad_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfdlmprad_prod Checking test 014 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -815,8 +815,8 @@ Checking test 014 fv3_ccpp_gfdlmprad results .... Test 014 fv3_ccpp_gfdlmprad PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_atmwav_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfdlmprad_atmwav_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_atmwav_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -864,8 +864,8 @@ Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Test 015 fv3_ccpp_gfdlmprad_atmwav PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_multigases_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_multigases_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_multigases_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_multigases_prod Checking test 016 fv3_ccpp_multigases results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -938,8 +938,8 @@ Checking test 016 fv3_ccpp_multigases results .... Test 016 fv3_ccpp_multigases PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_32bit_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_control_32bit_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_32bit_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_control_32bit_prod Checking test 017 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1006,8 +1006,8 @@ Checking test 017 fv3_ccpp_control_32bit results .... Test 017 fv3_ccpp_control_32bit PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_stretched_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_stretched_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_stretched_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_stretched_prod Checking test 018 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1062,8 +1062,8 @@ Checking test 018 fv3_ccpp_stretched results .... Test 018 fv3_ccpp_stretched PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_stretched_nest_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_stretched_nest_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_stretched_nest_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_stretched_nest_prod Checking test 019 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1071,21 +1071,21 @@ Checking test 019 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile4.nc .........OK Comparing atmos_4xdaily.tile5.nc .........OK Comparing atmos_4xdaily.tile6.nc .........OK - Comparing atmos_4xdaily.nest02.nc .........OK + Comparing atmos_4xdaily.nest02.tile7.nc .........OK Comparing fv3_history2d.tile1.nc .........OK Comparing fv3_history2d.tile2.nc .........OK Comparing fv3_history2d.tile3.nc .........OK Comparing fv3_history2d.tile4.nc .........OK Comparing fv3_history2d.tile5.nc .........OK Comparing fv3_history2d.tile6.nc .........OK - Comparing fv3_history2d.nest02.nc .........OK + Comparing fv3_history2d.nest02.tile7.nc .........OK Comparing fv3_history.tile1.nc .........OK Comparing fv3_history.tile2.nc .........OK Comparing fv3_history.tile3.nc .........OK Comparing fv3_history.tile4.nc .........OK Comparing fv3_history.tile5.nc .........OK Comparing fv3_history.tile6.nc .........OK - Comparing fv3_history.nest02.nc .........OK + Comparing fv3_history.nest02.tile7.nc .........OK Comparing RESTART/coupler.res .........OK Comparing RESTART/fv_core.res.nc .........OK Comparing RESTART/fv_core.res.nest02.nc .........OK @@ -1129,8 +1129,8 @@ Checking test 019 fv3_ccpp_stretched_nest results .... Test 019 fv3_ccpp_stretched_nest PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_regional_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_regional_control_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_regional_control_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_regional_control_prod Checking test 020 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1140,8 +1140,8 @@ Checking test 020 fv3_ccpp_regional_control results .... Test 020 fv3_ccpp_regional_control PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_regional_restart_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_regional_restart_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_regional_restart_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_regional_restart_prod Checking test 021 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1149,8 +1149,8 @@ Checking test 021 fv3_ccpp_regional_restart results .... Test 021 fv3_ccpp_regional_restart PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_regional_quilt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_regional_quilt_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_regional_quilt_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_regional_quilt_prod Checking test 022 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1160,8 +1160,8 @@ Checking test 022 fv3_ccpp_regional_quilt results .... Test 022 fv3_ccpp_regional_quilt PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_regional_c768_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_regional_c768_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_regional_c768_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_regional_c768_prod Checking test 023 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1171,20 +1171,20 @@ Checking test 023 fv3_ccpp_regional_c768 results .... Test 023 fv3_ccpp_regional_c768 PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_control_debug_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_control_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_control_debug_prod Checking test 024 fv3_ccpp_control_debug results .... Test 024 fv3_ccpp_control_debug PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_stretched_nest_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_stretched_nest_debug_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_stretched_nest_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_stretched_nest_debug_prod Checking test 025 fv3_ccpp_stretched_nest_debug results .... Test 025 fv3_ccpp_stretched_nest_debug PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmp_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfdlmp_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmp_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfdlmp_prod Checking test 026 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1231,8 +1231,8 @@ Checking test 026 fv3_ccpp_gfdlmp results .... Test 026 fv3_ccpp_gfdlmp PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_gwd_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfdlmprad_gwd_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_gwd_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfdlmprad_gwd_prod Checking test 027 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1279,8 +1279,8 @@ Checking test 027 fv3_ccpp_gfdlmprad_gwd results .... Test 027 fv3_ccpp_gfdlmprad_gwd PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_noahmp_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfdlmprad_noahmp_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_noahmp_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 028 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1327,8 +1327,8 @@ Checking test 028 fv3_ccpp_gfdlmprad_noahmp results .... Test 028 fv3_ccpp_gfdlmprad_noahmp PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmp_hwrfsas_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfdlmp_hwrfsas_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmp_hwrfsas_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 029 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1375,8 +1375,8 @@ Checking test 029 fv3_ccpp_gfdlmp_hwrfsas results .... Test 029 fv3_ccpp_gfdlmp_hwrfsas PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_csawmg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_csawmg_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_csawmg_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_csawmg_prod Checking test 030 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1423,8 +1423,8 @@ Checking test 030 fv3_ccpp_csawmg results .... Test 030 fv3_ccpp_csawmg PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_satmedmf_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_satmedmf_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_satmedmf_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_satmedmf_prod Checking test 031 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1471,8 +1471,8 @@ Checking test 031 fv3_ccpp_satmedmf results .... Test 031 fv3_ccpp_satmedmf PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_satmedmfq_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_satmedmfq_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_satmedmfq_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_satmedmfq_prod Checking test 032 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1519,8 +1519,8 @@ Checking test 032 fv3_ccpp_satmedmfq results .... Test 032 fv3_ccpp_satmedmfq PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmp_32bit_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfdlmp_32bit_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmp_32bit_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfdlmp_32bit_prod Checking test 033 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1567,8 +1567,8 @@ Checking test 033 fv3_ccpp_gfdlmp_32bit results .... Test 033 fv3_ccpp_gfdlmp_32bit PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfdlmprad_32bit_post_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfdlmprad_32bit_post_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfdlmprad_32bit_post_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 034 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1619,8 +1619,8 @@ Checking test 034 fv3_ccpp_gfdlmprad_32bit_post results .... Test 034 fv3_ccpp_gfdlmprad_32bit_post PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_cpt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_cpt_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_cpt_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_cpt_prod Checking test 035 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1673,8 +1673,8 @@ Checking test 035 fv3_ccpp_cpt results .... Test 035 fv3_ccpp_cpt PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gsd_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gsd_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gsd_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gsd_prod Checking test 036 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1765,8 +1765,8 @@ Checking test 036 fv3_ccpp_gsd results .... Test 036 fv3_ccpp_gsd PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_thompson_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_thompson_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_thompson_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_thompson_prod Checking test 037 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1833,8 +1833,8 @@ Checking test 037 fv3_ccpp_thompson results .... Test 037 fv3_ccpp_thompson PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_thompson_no_aero_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_thompson_no_aero_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_thompson_no_aero_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_thompson_no_aero_prod Checking test 038 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1901,8 +1901,8 @@ Checking test 038 fv3_ccpp_thompson_no_aero results .... Test 038 fv3_ccpp_thompson_no_aero PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_rrfs_v1beta_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_rrfs_v1beta_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_rrfs_v1beta_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_rrfs_v1beta_prod Checking test 039 fv3_ccpp_rrfs_v1beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1993,8 +1993,8 @@ Checking test 039 fv3_ccpp_rrfs_v1beta results .... Test 039 fv3_ccpp_rrfs_v1beta PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gsd_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gsd_debug_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gsd_debug_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gsd_debug_prod Checking test 040 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2061,8 +2061,8 @@ Checking test 040 fv3_ccpp_gsd_debug results .... Test 040 fv3_ccpp_gsd_debug PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gsd_diag3d_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gsd_diag3d_debug_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gsd_diag3d_debug_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gsd_diag3d_debug_prod Checking test 041 fv3_ccpp_gsd_diag3d_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2129,8 +2129,8 @@ Checking test 041 fv3_ccpp_gsd_diag3d_debug results .... Test 041 fv3_ccpp_gsd_diag3d_debug PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_thompson_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_thompson_debug_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_thompson_debug_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_thompson_debug_prod Checking test 042 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2197,8 +2197,8 @@ Checking test 042 fv3_ccpp_thompson_debug results .... Test 042 fv3_ccpp_thompson_debug PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_thompson_no_aero_debug_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_thompson_no_aero_debug_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_thompson_no_aero_debug_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_thompson_no_aero_debug_prod Checking test 043 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2265,8 +2265,8 @@ Checking test 043 fv3_ccpp_thompson_no_aero_debug results .... Test 043 fv3_ccpp_thompson_no_aero_debug PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfsv16_csawmg_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfsv16_csawmg_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfsv16_csawmg_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfsv16_csawmg_prod Checking test 044 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2313,8 +2313,8 @@ Checking test 044 fv3_ccpp_gfsv16_csawmg results .... Test 044 fv3_ccpp_gfsv16_csawmg PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfsv16_csawmgt_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfsv16_csawmgt_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfsv16_csawmgt_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfsv16_csawmgt_prod Checking test 045 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2361,8 +2361,8 @@ Checking test 045 fv3_ccpp_gfsv16_csawmgt results .... Test 045 fv3_ccpp_gfsv16_csawmgt PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gocart_clm_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gocart_clm_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gocart_clm_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gocart_clm_prod Checking test 046 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2409,8 +2409,8 @@ Checking test 046 fv3_ccpp_gocart_clm results .... Test 046 fv3_ccpp_gocart_clm PASS -baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200724/fv3_gfs_v16beta_flake_ccpp -working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_154309/fv3_ccpp_gfs_v16beta_flake_prod +baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200805/fv3_gfs_v16beta_flake_ccpp +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_109920/fv3_ccpp_gfs_v16beta_flake_prod Checking test 047 fv3_ccpp_gfs_v16beta_flake results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2478,5 +2478,5 @@ Test 047 fv3_ccpp_gfs_v16beta_flake PASS REGRESSION TEST WAS SUCCESSFUL -Wed Jul 29 22:26:28 UTC 2020 -Elapsed time: 01h:02m:57s. Have a nice day! +Tue Aug 4 18:32:04 UTC 2020 +Elapsed time: 01h:05m:55s. Have a nice day! diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 8cbcd80b69..4bbc40255f 100755 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -7,13 +7,7 @@ # ############################################################################### -if [ $MACHINE_ID = wcoss ]; then - - TASKS_dflt=144 ; TPN_dflt=16 ; INPES_dflt=3 ; JNPES_dflt=8 - TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4 - TASKS_strnest=96 ; TPN_strnest=12 ; INPES_strnest=2 ; JNPES_strnest=4 - -elif [ $MACHINE_ID = wcoss_cray ]; then +if [ $MACHINE_ID = wcoss_cray ]; then TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8 TASKS_thrd=84 ; TPN_thrd=12 ; INPES_thrd=3 ; JNPES_thrd=4 diff --git a/tests/head.h b/tests/head.h index 00c7d1d1fc..bcae862cc5 100644 --- a/tests/head.h +++ b/tests/head.h @@ -23,10 +23,28 @@ ecflow_client --init=$$ # Define a error handler -ERROR() { +handle_error() { set +e # Clear -e flag, so we don't fail wait # wait for background process to stop + ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT} + not_running=$? + if [[ $not_running -eq 0 ]]; then + export ECF_TIMEOUT=5 + ecflow_client --abort=error # Notify ecFlow that something went wrong, using 'trap' as the reason + fi + sleep 5 + trap 0 # Remove the trap + exit 0 # End the script +} + +# Define a signal handler +handle_signal() { + set +e # Clear -e flag, so we don't fail + jobs -l + [[ -z "$(jobs -p)" ]] || kill $(jobs -p) + wait # wait for background process to stop + ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT} not_running=$? if [[ $not_running -eq 0 ]]; then @@ -40,9 +58,9 @@ ERROR() { # Trap any calls to exit and errors caught by the -e flag -trap ERROR 0 +trap handle_error 0 # Trap any signal that may cause the script to fail -trap '{ echo "$0 Killed by a signal"; ERROR ; }' 1 2 3 4 5 6 7 8 10 12 13 15 +trap '{ echo "$0 Killed by a signal"; handle_signal ; }' 1 2 3 4 5 6 7 8 10 12 13 15 ### head.h end diff --git a/tests/rt.sh b/tests/rt.sh index 128499cade..a69c679077 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -58,6 +58,7 @@ rt_single() { rt_trap() { [[ ${ROCOTO:-false} == true ]] && rocoto_kill + [[ ${ECFLOW:-false} == true ]] && ecflow_kill cleanup } @@ -71,7 +72,7 @@ cleanup() { trap '{ echo "rt.sh interrupted"; rt_trap ; }' INT trap '{ echo "rt.sh quit"; rt_trap ; }' QUIT trap '{ echo "rt.sh terminated"; rt_trap ; }' TERM -trap '{ echo "rt.sh error on line $LINENO"; rt_trap ; }' ERR +trap '{ echo "rt.sh error on line $LINENO"; cleanup ; }' ERR trap '{ echo "rt.sh finished"; cleanup ; }' EXIT # PATHRT - Path to regression tests directory @@ -186,7 +187,6 @@ elif [[ $MACHINE_ID = gaea.* ]]; then STMP=/lustre/f2/scratch PTMP=/lustre/f2/scratch - # default scheduler on Gaea SCHEDULER=slurm cp fv3_conf/fv3_slurm.IN_gaea fv3_conf/fv3_slurm.IN @@ -236,7 +236,7 @@ elif [[ $MACHINE_ID = orion.* ]]; then # Re-instantiate COMPILER in case it gets deleted by module purge COMPILER=${NEMS_COMPILER:-intel} - module load rocoto/1.3.1 + module load contrib rocoto/1.3.1 ROCOTORUN=$(which rocotorun) ROCOTOSTAT=$(which rocotostat) ROCOTOCOMPLETE=$(which rocotocomplete) @@ -286,7 +286,6 @@ elif [[ $MACHINE_ID = jet.* ]]; then STMP=$dprefix/RT_BASELINE PTMP=$dprefix/RT_RUNDIRS - # default scheduler on Jet SCHEDULER=slurm cp fv3_conf/fv3_slurm.IN_jet fv3_conf/fv3_slurm.IN @@ -418,9 +417,9 @@ if [[ $SINGLE_NAME != '' ]]; then fi if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]]; then - RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200724/${COMPILER^^}} + RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200805/${COMPILER^^}} else - RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200724} + RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-20200805} fi shift $((OPTIND-1)) @@ -432,7 +431,7 @@ if [[ $CREATE_BASELINE == true ]]; then # rm -rf "${NEW_BASELINE}" mkdir -p "${NEW_BASELINE}" - echo "copy baseline inputs form: ${RTPWD}" + echo "copy baseline inputs from: ${RTPWD}" echo " to: ${NEW_BASELINE}" rsync -a "${RTPWD}"/FV3_* "${NEW_BASELINE}"/ @@ -544,7 +543,8 @@ suite ${ECFLOW_SUITE} edit ECF_INCLUDE '${ECFLOW_RUN}' edit ECF_KILL_CMD kill -15 %ECF_RID% > %ECF_JOB%.kill 2>&1 edit ECF_TRIES 1 - label rundir_root '${RUNDIR_ROOT}' + label src_dir '${PATHTR}' + label run_dir '${RUNDIR_ROOT}' limit max_builds 10 limit max_jobs 30 EOF diff --git a/tests/rt_fv3.sh b/tests/rt_fv3.sh deleted file mode 100755 index e7f629144b..0000000000 --- a/tests/rt_fv3.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -set -eux - -source rt_utils.sh -source atparse.bash - -mkdir -p ${RUNDIR} -cd $RUNDIR - -############################################################################### -# Make configure and run files -############################################################################### - -# FV3 executable: -cp ${PATHRT}/$FV3X fv3.exe - -# modulefile for FV3 prerequisites: -cp ${PATHRT}/modules.fv3_${COMPILE_NR} modules.fv3 - -# Get the shell file that loads the "module" command and purges modules: -cp ${PATHRT}/../NEMS/src/conf/module-setup.sh.inc module-setup.sh -cp ${PATHTR}/parm/post_itag itag -cp ${PATHTR}/parm/postxconfig-NT.txt postxconfig-NT.txt -cp ${PATHTR}/parm/postxconfig-NT_FH00.txt postxconfig-NT_FH00.txt -cp ${PATHTR}/parm/params_grib2_tbl_new params_grib2_tbl_new - -SRCD="${PATHTR}" -RUND="${RUNDIR}" - -atparse < ${PATHRT}/fv3_conf/${FV3_RUN:-fv3_run.IN} > fv3_run - -atparse < ${PATHTR}/parm/${INPUT_NML:-input.nml.IN} > input.nml - -atparse < ${PATHTR}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure - -atparse < ${PATHTR}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure - -if [[ "Q${INPUT_NEST02_NML:-}" != Q ]] ; then - atparse < ${PATHTR}/parm/${INPUT_NEST02_NML} > input_nest02.nml -fi - -# Set up the run directory -source ./fv3_run - -if [[ $SCHEDULER = 'pbs' ]]; then - NODES=$(( TASKS / TPN )) - if (( NODES * TPN < TASKS )); then - NODES=$(( NODES + 1 )) - fi - atparse < $PATHRT/fv3_conf/fv3_qsub.IN > job_card -elif [[ $SCHEDULER = 'slurm' ]]; then - NODES=$(( TASKS / TPN )) - if (( NODES * TPN < TASKS )); then - NODES=$(( NODES + 1 )) - fi - atparse < $PATHRT/fv3_conf/fv3_slurm.IN > job_card -elif [[ $SCHEDULER = 'lsf' ]]; then - if (( TASKS < TPN )); then - TPN=${TASKS} - fi - NODES=$(( TASKS / TPN )) - if (( NODES * TPN < TASKS )); then - NODES=$(( NODES + 1 )) - fi - atparse < $PATHRT/fv3_conf/fv3_bsub.IN > job_card -fi - -atparse < ${PATHTR}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure - -################################################################################ -# Submit test -################################################################################ - -if [[ $ROCOTO = 'false' ]]; then - submit_and_wait job_card -else - chmod u+x job_card - ./job_card -fi - -check_results - -################################################################################ -# End test -################################################################################ - -exit 0 diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh index 4829568836..acb5c6cfc0 100755 --- a/tests/rt_utils.sh +++ b/tests/rt_utils.sh @@ -7,6 +7,26 @@ fi UNIT_TEST=${UNIT_TEST:-false} +qsub_id=0 +slurm_id=0 +bsub_id=0 + +interrupt_job() { + set -x + if [[ $SCHEDULER = 'pbs' ]]; then + echo "run_util.sh: interrupt_job qsub_id = ${qsub_id}" + qdel ${qsub_id} + elif [[ $SCHEDULER = 'slurm' ]]; then + echo "run_util.sh: interrupt_job slurm_id = ${slurm_id}" + scancel ${slurm_id} + elif [[ $SCHEDULER = 'lsf' ]]; then + echo "run_util.sh: interrupt_job bsub_id = ${bsub_id}" + bkill ${bsub_id} + else + echo "run_util.sh: interrupt_job unknown SCHEDULER $SCHEDULER" + fi +} + submit_and_wait() { [[ -z $1 ]] && exit 1 @@ -28,7 +48,6 @@ submit_and_wait() { else re='^([0-9]+\.[a-zA-Z0-9]+)$' fi - qsub_id=0 [[ "${qsubout}" =~ $re ]] && qsub_id=${BASH_REMATCH[1]} if [[ ${MACHINE_ID} = cheyenne.* ]]; then qsub_id="${qsub_id%.chadm*}" @@ -37,13 +56,11 @@ submit_and_wait() { elif [[ $SCHEDULER = 'slurm' ]]; then slurmout=$( sbatch $job_card ) re='Submitted batch job ([0-9]+)' - slurm_id=0 [[ "${slurmout}" =~ $re ]] && slurm_id=${BASH_REMATCH[1]} echo "Job id ${slurm_id}" elif [[ $SCHEDULER = 'lsf' ]]; then bsubout=$( bsub < $job_card ) re='Job <([0-9]+)> is submitted to queue <(.+)>.' - bsub_id=0 [[ "${bsubout}" =~ $re ]] && bsub_id=${BASH_REMATCH[1]} echo "Job id ${bsub_id}" else @@ -57,6 +74,7 @@ submit_and_wait() { until [[ $job_running -eq 1 ]] do echo "TEST ${TEST_NR} ${TEST_NAME} is waiting to enter the queue" + [[ ${ECFLOW:-false} == true ]] && ecflow_client --label=job_status "waiting to enter the queue" if [[ $SCHEDULER = 'pbs' ]]; then if [[ ${MACHINE_ID} = cheyenne.* ]]; then job_running=$( qstat ${qsub_id} | grep ${qsub_id} | wc -l) @@ -87,6 +105,11 @@ submit_and_wait() { echo "Unknown SCHEDULER $SCHEDULER" exit 1 fi + echo "TEST ${TEST_NR} ${TEST_NAME} is submitted " + if [[ ${ECFLOW:-false} == true ]]; then + ecflow_client --label=job_id "${jobid}" + ecflow_client --label=job_status "submitted" + fi # wait for the job to finish and compare results job_running=1 @@ -94,8 +117,6 @@ submit_and_wait() { until [[ $job_running -eq 0 ]] do - sleep 60 & wait $! - if [[ $SCHEDULER = 'pbs' ]]; then if [[ ${MACHINE_ID} = cheyenne.* ]]; then job_running=$( qstat ${qsub_id} | grep ${qsub_id} | wc -l) @@ -118,10 +139,15 @@ submit_and_wait() { else status=$( qstat -u ${USER} -n | grep ${JBNME} | awk '{print $10}' ); status=${status:--} fi - if [[ $status = 'Q' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is waiting in a queue, Status: $status jobid ${jobid}" - elif [[ $status = 'H' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is held in a queue, Status: $status" - elif [[ $status = 'R' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is running, Status: $status" + if [[ $status = 'Q' ]]; then + status_label='waiting in a queue' + elif [[ $status = 'H' ]]; then + status_label='held in a queue' + elif [[ $status = 'R' ]]; then + status_label='running' elif [[ $status = 'E' ]] || [[ $status = 'C' ]]; then + status_label='finished' + test_status='DONE' if [[ ${MACHINE_ID} = cheyenne.* ]]; then exit_status=$( qstat ${jobid} -x -f | grep Exit_status | awk '{print $3}') else @@ -129,62 +155,53 @@ submit_and_wait() { exit_status=$( qstat ${jobid} -f | grep exit_status | awk '{print $3}') fi if [[ $exit_status != 0 ]]; then - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" >> ${REGRESSIONTEST_LOG} - echo >> ${REGRESSIONTEST_LOG} - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" - echo test_status='FAIL' - break fi - echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" - job_running=0 - elif [[ $status = 'C' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" ; job_running=0 - else echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" + else + status_label='finished' fi elif [[ $SCHEDULER = 'slurm' ]]; then status=$( squeue -u ${USER} -j ${slurm_id} 2>/dev/null | grep ${slurm_id} | awk '{print $5}' ); status=${status:--} - if [[ $status = 'R' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is running, Status: $status" - elif [[ $status = 'PD' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is pending, Status: $status" - elif [[ $status = 'F' ]]; then - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" >> ${REGRESSIONTEST_LOG} - echo >> ${REGRESSIONTEST_LOG} - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" - echo - echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is failed, Status: $status" - job_running=0 - elif [[ $status = 'C' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" ; job_running=0 + if [[ $status = 'R' ]]; then + status_label='running' + elif [[ $status = 'PD' ]]; then + status_label='pending' + elif [[ $status = 'F' ]]; then + status_label='failed' + test_status='FAIL' + elif [[ $status = 'C' ]]; then + status_label='finished' + test_status='DONE' else echo "Slurm unknown status ${status}. Check sacct ..." - sacct -n -j ${slurm_id} --format=JobID,state,Jobname - state=$( sacct -n -j ${slurm_id} --format=JobID,state,Jobname | grep ${slurm_id} | awk '{print $2}' ) - echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is ${state}" + sacct -n -j ${slurm_id} --format=JobID,state%20,Jobname%20 + status_label=$( sacct -n -j ${slurm_id} --format=JobID,state%20,Jobname%20 | grep "^${slurm_id}" | grep ${JBNME} | awk '{print $2}' ) fi elif [[ $SCHEDULER = 'lsf' ]]; then status=$( bjobs ${bsub_id} 2>/dev/null | grep ${bsub_id} | awk '{print $3}' ); status=${status:--} - if [[ $status = 'PEND' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is waiting in a queue, Status: $status" - elif [[ $status = 'RUN' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is running, Status: $status" + if [[ $status = 'PEND' ]]; then + status_label='pending' + elif [[ $status = 'RUN' ]]; then + status_label='running' + elif [[ $status = 'DONE' ]]; then + status_label='finished' + test_status='DONE' elif [[ $status = 'EXIT' ]]; then - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" >> ${REGRESSIONTEST_LOG} - echo;echo;echo >> ${REGRESSIONTEST_LOG} - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" - echo;echo;echo + status_label='failed' test_status='FAIL' - break else - echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" + echo "bsub unknown status ${status}" + status_label='finished' + test_status='DONE' exit_status=$( bjobs ${bsub_id} 2>/dev/null | grep ${bsub_id} | awk '{print $3}' ); status=${status:--} if [[ $exit_status = 'EXIT' ]]; then - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" >> ${REGRESSIONTEST_LOG} - echo;echo;echo >> ${REGRESSIONTEST_LOG} - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" - echo;echo;echo + status_label='failed' test_status='FAIL' fi - break fi else @@ -192,12 +209,24 @@ submit_and_wait() { exit 1 fi + + echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is ${status_label}, status: $status jobid ${jobid}" + [[ ${ECFLOW:-false} == true ]] && ecflow_client --label=job_status "$status_label" + + if [[ $test_status = 'FAIL' || $test_status = 'DONE' ]]; then + break + fi + (( n=n+1 )) + sleep 60 & wait $! done if [[ $test_status = 'FAIL' ]]; then if [[ ${UNIT_TEST} == false ]]; then - echo "${TEST_NAME} ${TEST_NR}" >> $PATHRT/fail_test + echo "${TEST_NAME} ${TEST_NR} failed" >> $PATHRT/fail_test + echo "Test ${TEST_NR} ${TEST_NAME} FAIL" >> ${REGRESSIONTEST_LOG} + echo;echo;echo >> ${REGRESSIONTEST_LOG} + echo "Test ${TEST_NR} ${TEST_NAME} FAIL" else echo ${TEST_NR} $TEST_NAME >> $PATHRT/fail_unit_test fi @@ -237,7 +266,7 @@ check_results() { if [[ ${CREATE_BASELINE} = false ]]; then # - # --- regression test comparison ---- + # --- regression test comparison # for i in ${LIST_FILES} ; do printf %s " Comparing " $i " ....." >> ${REGRESSIONTEST_LOG} @@ -255,21 +284,13 @@ check_results() { echo ".......MISSING baseline" test_status='FAIL' - elif [[ ( $COMPILER == "gnu" || $COMPILER == "pgi" ) && $i == "RESTART/fv_core.res.nc" ]] ; then + elif [[ $COMPILER == "gnu" && $i == "RESTART/fv_core.res.nc" ]] ; then # Although identical in ncdiff, RESTART/fv_core.res.nc differs in byte 469, line 3, # for the fv3_control_32bit test between each run (without changing the source code) - # for GNU and PGI compilers - skip comparison. - echo ".......SKIP for gnu/pgi compilers" >> ${REGRESSIONTEST_LOG} - echo ".......SKIP for gnu/pgi compilers" - - elif [[ $COMPILER == "pgi" && ( $i == "RESTART/fv_BC_sw.res.nest02.nc" || $i == "RESTART/fv_BC_ne.res.nest02.nc" ) ]] ; then - - # Although identical in ncdiff, RESTART/fv_BC_sw.res.nest02.nc differs in byte 6897, line 17 - # (similar for fv_BC_ne.res.nest02.nc) for the fv3_stretched_nest test between each run - # (without changing the source code) for the PGI compiler - skip comparison. - echo ".......SKIP for pgi compiler" >> ${REGRESSIONTEST_LOG} - echo ".......SKIP for pgi compiler" + # for GNU compilers - skip comparison. + echo ".......SKIP for gnu compilers" >> ${REGRESSIONTEST_LOG} + echo ".......SKIP for gnu compilers" else @@ -292,19 +313,23 @@ check_results() { # # --- create baselines # - echo;echo;echo "Moving set ${TEST_NR} ${TEST_NAME} files ...." + echo;echo "Moving baseline ${TEST_NR} ${TEST_NAME} files ...." + echo;echo "Moving baseline ${TEST_NR} ${TEST_NAME} files ...." >> ${REGRESSIONTEST_LOG} if [[ ! -d ${NEW_BASELINE}/${CNTL_DIR}/RESTART ]] ; then echo " mkdir -p ${NEW_BASELINE}/${CNTL_DIR}/RESTART" >> ${REGRESSIONTEST_LOG} mkdir -p ${NEW_BASELINE}/${CNTL_DIR}/RESTART fi for i in ${LIST_FILES} ; do + printf %s " Moving " $i " ....." printf %s " Moving " $i " ....." >> ${REGRESSIONTEST_LOG} if [[ -f ${RUNDIR}/$i ]] ; then cp ${RUNDIR}/${i} ${NEW_BASELINE}/${CNTL_DIR}/${i} + echo ".... OK" + echo ".... OK" >> ${REGRESSIONTEST_LOG} else - echo "Missing " ${RUNDIR}/$i " output file" - echo;echo " Set ${TEST_NR} ${TEST_NAME} failed" + echo ".... missing " ${RUNDIR}/$i + echo ".... missing " ${RUNDIR}/$i >> ${REGRESSIONTEST_LOG} test_status='FAIL' fi done @@ -318,7 +343,7 @@ check_results() { if [[ $test_status = 'FAIL' ]]; then if [[ ${UNIT_TEST} == false ]]; then - echo $TEST_NAME >> $PATHRT/fail_test + echo "${TEST_NAME} ${TEST_NR} failed in check_result" >> $PATHRT/fail_test else echo ${TEST_NR} $TEST_NAME >> $PATHRT/fail_unit_test fi @@ -331,6 +356,7 @@ check_results() { eval "$set_x" } + kill_job() { [[ -z $1 ]] && exit 1 @@ -346,7 +372,6 @@ kill_job() { fi } - rocoto_create_compile_task() { new_compile=true @@ -402,7 +427,6 @@ rocoto_create_compile_task() { EOF } - rocoto_create_run_task() { if [[ $DEP_RUN != '' ]]; then @@ -441,7 +465,6 @@ EOF } - rocoto_kill() { for jobid in $( $ROCOTOSTAT -w $ROCOTO_XML -d $ROCOTO_DB | grep 197001010000 | grep -E 'QUEUED|RUNNING' | awk -F" " '{print $3}' ); do kill_job ${jobid} @@ -454,18 +477,19 @@ rocoto_run() { while [[ $state != "Done" ]] do $ROCOTORUN -v 10 -w $ROCOTO_XML -d $ROCOTO_DB - sleep 10 & wait $! + sleep 10 state=$($ROCOTOSTAT -w $ROCOTO_XML -d $ROCOTO_DB -s | grep 197001010000 | awk -F" " '{print $2}') dead_compile=$($ROCOTOSTAT -w $ROCOTO_XML -d $ROCOTO_DB | grep compile_ | grep DEAD | head -1 | awk -F" " '{print $2}') if [[ ! -z ${dead_compile} ]]; then echo "y" | ${ROCOTOCOMPLETE} -w $ROCOTO_XML -d $ROCOTO_DB -m ${dead_compile}_tasks ${ROCOTOCOMPLETE} -w $ROCOTO_XML -d $ROCOTO_DB -t ${dead_compile} fi - sleep 20 & wait $! + sleep 20 done } + ecflow_create_compile_task() { new_compile=true @@ -473,11 +497,14 @@ ecflow_create_compile_task() { cat << EOF > ${ECFLOW_RUN}/${ECFLOW_SUITE}/compile_${COMPILE_NR}.ecf %include -$PATHRT/run_compile.sh ${PATHRT} ${RUNDIR_ROOT} "${NEMS_VER}" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1 +$PATHRT/run_compile.sh ${PATHRT} ${RUNDIR_ROOT} "${NEMS_VER}" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1 & %include EOF echo " task compile_${COMPILE_NR}" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def + echo " label build_options '${NEMS_VER}'" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def + echo " label job_id ''" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def + echo " label job_status ''" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def echo " inlimit max_builds" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def # serialize WW3 builds. FIXME if [[ ${NEMS_VER^^} =~ "WW3=Y" && ${COMPILE_PREV_WW3_NR} != '' ]]; then @@ -489,11 +516,13 @@ ecflow_create_run_task() { cat << EOF > ${ECFLOW_RUN}/${ECFLOW_SUITE}/${TEST_NAME}${RT_SUFFIX}.ecf %include -$PATHRT/run_test.sh ${PATHRT} ${RUNDIR_ROOT} ${TEST_NAME} ${TEST_NR} ${COMPILE_NR} > ${LOG_DIR}/run_${TEST_NR}_${TEST_NAME}${RT_SUFFIX}.log 2>&1 +$PATHRT/run_test.sh ${PATHRT} ${RUNDIR_ROOT} ${TEST_NAME} ${TEST_NR} ${COMPILE_NR} > ${LOG_DIR}/run_${TEST_NR}_${TEST_NAME}${RT_SUFFIX}.log 2>&1 & %include EOF echo " task ${TEST_NAME}${RT_SUFFIX}" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def + echo " label job_id ''" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def + echo " label job_status ''" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def echo " inlimit max_jobs" >> ${ECFLOW_RUN}/${ECFLOW_SUITE}.def if [[ $DEP_RUN != '' ]]; then if [[ ${UNIT_TEST} == false ]]; then @@ -548,15 +577,15 @@ ecflow_run() { ecflow_kill() { [[ ${ECFLOW_RUNNING:-false} == true ]] || return set +e - wait + ecflow_client --suspend /${ECFLOW_SUITE} ecflow_client --kill /${ECFLOW_SUITE} - sleep 10 + sleep 20 + ecflow_client --delete=force yes /${ECFLOW_SUITE} } ecflow_stop() { [[ ${ECFLOW_RUNNING:-false} == true ]] || return set +e - wait SUITES=$( ecflow_client --get | grep "^suite" ) echo "SUITES=${SUITES}" if [ -z "${SUITES}" ]; then diff --git a/tests/run_compile.sh b/tests/run_compile.sh index 53819be257..7ee1db6ea5 100755 --- a/tests/run_compile.sh +++ b/tests/run_compile.sh @@ -1,8 +1,19 @@ #!/bin/bash set -eux +echo "PID=$$" SECONDS=0 +trap 'echo "run_compile.sh interrupted PID=$$"; cleanup' INT +trap 'echo "run_compile.sh terminated PID=$$"; cleanup' TERM + +cleanup() { + [[ $ROCOTO = 'false' ]] && interrupt_job + trap 0 + exit +} + + if [[ $# != 4 ]]; then echo "Usage: $0 PATHRT RUNDIR_ROOT MAKE_OPT COMPILE_NR" exit 1 @@ -10,8 +21,8 @@ fi export PATHRT=$1 export RUNDIR_ROOT=$2 -export MAKE_OPT=${3} -export COMPILE_NR=${4} +export MAKE_OPT=$3 +export COMPILE_NR=$4 cd ${PATHRT} diff --git a/tests/run_test.sh b/tests/run_test.sh index 76a607c68d..7525e1b652 100755 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -1,14 +1,25 @@ #!/bin/bash set -eux +echo "PID=$$" +SECONDS=0 + +trap '[ "$?" -eq 0 ] || write_fail_test' EXIT +trap 'echo "run_test.sh interrupted PID=$$"; cleanup' INT +trap 'echo "run_test.sh terminated PID=$$"; cleanup' TERM + +cleanup() { + [[ $ROCOTO = 'false' ]] && interrupt_job + trap 0 + exit +} + write_fail_test() { if [[ ${UNIT_TEST} == true ]]; then echo ${TEST_NR} $TEST_NAME >> $PATHRT/fail_unit_test fi } -SECONDS=0 - if [[ $# != 5 ]]; then echo "Usage: $0 PATHRT RUNDIR_ROOT TEST_NAME TEST_NR COMPILE_NR" exit 1 @@ -34,10 +45,7 @@ export INPUT_DIR=${CNTL_DIR} export RUNDIR=${RUNDIR_ROOT}/${TEST_NAME}${RT_SUFFIX} export CNTL_DIR=${CNTL_DIR}${BL_SUFFIX} -JBNME=$(basename $RUNDIR_ROOT)_${TEST_NR} -export JBNME - -export FV3X=fv3_${COMPILE_NR}.exe +export JBNME=$(basename $RUNDIR_ROOT)_${TEST_NR} UNIT_TEST=${UNIT_TEST:-false} if [[ ${UNIT_TEST} == false ]]; then @@ -47,12 +55,90 @@ else fi export REGRESSIONTEST_LOG -# Submit the actual test run script echo "Test ${TEST_NR} ${TEST_NAME} ${TEST_DESCR}" -trap 'echo "run_test.sh: Test ${TEST_NAME} killed"; kill $(jobs -p); wait; trap 0; exit' 1 2 3 4 5 6 7 8 10 12 13 15 -trap '[ "$?" -eq 0 ] || write_fail_test' EXIT -trap 'echo "run_test.sh: Test ${TEST_NAME} error"; echo "${TEST_NAME}" >> ${PATHRT}/fail_test; trap 0; exit' ERR -./${RUN_SCRIPT} > ${RUNDIR_ROOT}/${TEST_NAME}${RT_SUFFIX}.log 2>&1 + +source rt_utils.sh +source atparse.bash + +mkdir -p ${RUNDIR} +cd $RUNDIR + +############################################################################### +# Make configure and run files +############################################################################### + +# FV3 executable: +cp ${PATHRT}/fv3_${COMPILE_NR}.exe fv3.exe + +# modulefile for FV3 prerequisites: +cp ${PATHRT}/modules.fv3_${COMPILE_NR} modules.fv3 + +# Get the shell file that loads the "module" command and purges modules: +cp ${PATHRT}/../NEMS/src/conf/module-setup.sh.inc module-setup.sh +cp ${PATHTR}/parm/post_itag itag +cp ${PATHTR}/parm/postxconfig-NT.txt postxconfig-NT.txt +cp ${PATHTR}/parm/postxconfig-NT_FH00.txt postxconfig-NT_FH00.txt +cp ${PATHTR}/parm/params_grib2_tbl_new params_grib2_tbl_new + +SRCD="${PATHTR}" +RUND="${RUNDIR}" + +atparse < ${PATHRT}/fv3_conf/${FV3_RUN:-fv3_run.IN} > fv3_run + +atparse < ${PATHTR}/parm/${INPUT_NML:-input.nml.IN} > input.nml + +atparse < ${PATHTR}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure + +atparse < ${PATHTR}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure + +if [[ "Q${INPUT_NEST02_NML:-}" != Q ]] ; then + atparse < ${PATHTR}/parm/${INPUT_NEST02_NML} > input_nest02.nml +fi + +# Set up the run directory +source ./fv3_run + +if [[ $SCHEDULER = 'pbs' ]]; then + NODES=$(( TASKS / TPN )) + if (( NODES * TPN < TASKS )); then + NODES=$(( NODES + 1 )) + fi + atparse < $PATHRT/fv3_conf/fv3_qsub.IN > job_card +elif [[ $SCHEDULER = 'slurm' ]]; then + NODES=$(( TASKS / TPN )) + if (( NODES * TPN < TASKS )); then + NODES=$(( NODES + 1 )) + fi + atparse < $PATHRT/fv3_conf/fv3_slurm.IN > job_card +elif [[ $SCHEDULER = 'lsf' ]]; then + if (( TASKS < TPN )); then + TPN=${TASKS} + fi + NODES=$(( TASKS / TPN )) + if (( NODES * TPN < TASKS )); then + NODES=$(( NODES + 1 )) + fi + atparse < $PATHRT/fv3_conf/fv3_bsub.IN > job_card +fi + +atparse < ${PATHTR}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure + +################################################################################ +# Submit test job +################################################################################ + +if [[ $ROCOTO = 'false' ]]; then + submit_and_wait job_card +else + chmod u+x job_card + ./job_card +fi + +check_results + +################################################################################ +# End test +################################################################################ elapsed=$SECONDS echo "Elapsed time $elapsed seconds. Test ${TEST_NAME}" diff --git a/tests/tail.h b/tests/tail.h index 57fde5041f..480941a8f1 100644 --- a/tests/tail.h +++ b/tests/tail.h @@ -1,6 +1,6 @@ ### tail.h start -wait # wait for background process to stop +wait $! # wait for background process to stop ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT} not_running=$? diff --git a/tests/tests/fv3_ccpp_stretched_nest b/tests/tests/fv3_ccpp_stretched_nest index 3e5f78b8b2..b19b0ad181 100644 --- a/tests/tests/fv3_ccpp_stretched_nest +++ b/tests/tests/fv3_ccpp_stretched_nest @@ -13,21 +13,21 @@ export LIST_FILES=" atmos_4xdaily.tile1.nc \ atmos_4xdaily.tile4.nc \ atmos_4xdaily.tile5.nc \ atmos_4xdaily.tile6.nc \ - atmos_4xdaily.nest02.nc \ + atmos_4xdaily.nest02.tile7.nc \ fv3_history2d.tile1.nc \ fv3_history2d.tile2.nc \ fv3_history2d.tile3.nc \ fv3_history2d.tile4.nc \ fv3_history2d.tile5.nc \ fv3_history2d.tile6.nc \ - fv3_history2d.nest02.nc \ + fv3_history2d.nest02.tile7.nc \ fv3_history.tile1.nc \ fv3_history.tile2.nc \ fv3_history.tile3.nc \ fv3_history.tile4.nc \ fv3_history.tile5.nc \ fv3_history.tile6.nc \ - fv3_history.nest02.nc \ + fv3_history.nest02.tile7.nc \ RESTART/coupler.res \ RESTART/fv_core.res.nc \ RESTART/fv_core.res.nest02.nc \ diff --git a/tests/tests/fv3_ccpp_stretched_nest_quilt b/tests/tests/fv3_ccpp_stretched_nest_quilt index ad3e21ee5b..10cf13091f 100644 --- a/tests/tests/fv3_ccpp_stretched_nest_quilt +++ b/tests/tests/fv3_ccpp_stretched_nest_quilt @@ -13,7 +13,7 @@ export LIST_FILES=" atmos_4xdaily.tile1.nc \ atmos_4xdaily.tile4.nc \ atmos_4xdaily.tile5.nc \ atmos_4xdaily.tile6.nc \ - atmos_4xdaily.nest02.nc \ + atmos_4xdaily.nest02.tile7.nc \ dynf000.nc \ dynf001.nc \ dynf006.nc \