Download unit test data as part of the CMake build

.github/workflows/debug-docs-test_coverage.yml

@@ -103,10 +103,7 @@ jobs:
 
     - name: test
       run: |
-        cd ufs_utils/build/tests
-        wget -i ./chgres_cube/data/files.txt -P ./chgres_cube/data
-        wget -i ./sfc_climo_gen/data/files.txt -P ./sfc_climo_gen/data
-        cd ../
+        cd ufs_utils/build
         export LSAN_OPTIONS=suppressions=LSanSuppress.supp
         ctest --rerun-failed --output-on-failure
         export PATH="/home/runner/.local/bin:$PATH"

.github/workflows/intel.yml

@@ -166,8 +166,5 @@ jobs:
 
     - name: test
       run: |
-        cd ufs_utils/build/tests
-        wget -i ./chgres_cube/data/files.txt -P ./chgres_cube/data
-        wget -i ./sfc_climo_gen/data/files.txt -P ./sfc_climo_gen/data
-        cd ../
+        cd ufs_utils/build
         ctest --rerun-failed --output-on-failure

.github/workflows/linux-mac-nceplibs-mpi.yml

@@ -221,10 +221,7 @@ jobs:
 
     - name: test
       run: |
-        cd ufs_utils/build/tests
-        wget -i ./chgres_cube/data/files.txt -P ./chgres_cube/data
-        wget -i ./sfc_climo_gen/data/files.txt -P ./sfc_climo_gen/data
-        cd ../
+        cd ufs_utils/build
         # Oversubscribe for OpenMPI to run more processes than CPUs
         export OMPI_MCA_rmaps_base_oversubscribe=1
         ctest --rerun-failed --output-on-failure

.github/workflows/netcdf-versions.yml

@@ -154,9 +154,6 @@ jobs:
 
     - name: test
       run: |
-        cd ufs_utils/build/tests
-        wget -i ./chgres_cube/data/files.txt -P ./chgres_cube/data
-        wget -i ./sfc_climo_gen/data/files.txt -P ./sfc_climo_gen/data
-        cd ../
+        cd ufs_utils/build
         export LSAN_OPTIONS=suppressions=LSanSuppress.supp
         ctest --rerun-failed --output-on-failure

CMakeLists.txt

@@ -23,8 +23,6 @@ if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
       CACHE STRING "Choose the type of build." FORCE)
   set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
                                                "MinSizeRel" "RelWithDebInfo")
-  message(STATUS "Set BUILD_TESTING to YES and build unit testing package under tests")
-  set(BUILD_TESTING "YES")
 endif()
 
 # Set compiler flags.
@@ -92,6 +90,6 @@ add_subdirectory(sorc)
 # Run unit tests.
 include(CTest)
 if(BUILD_TESTING)
+  message(STATUS "Set BUILD_TESTING to YES and build unit testing package under tests")
   add_subdirectory(tests)
 endif()
-

build_all.sh

@@ -26,8 +26,18 @@ else
  set -x
 fi
 
-CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON"
-#CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DENABLE_DOCS=ON"
+# The unit test data download is part of the build system. Not all machines can
+# access the EMC ftp site, so turn off the build (-DBUILD_TESTING=OFF) of the units tests accordingly.
+# Those with access to the EMC ftp site are: Orion, Hera, WCOSS-Cray, WCOSS-Dell.
+
+if [[ "$target" == "hera" || "$target" == "orion" || "$target" == "wcoss_cray" || "$target" == "wcoss_dell_p3" ]]; then
+  CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=OFF"
+   #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=ON"
+   #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DENABLE_DOCS=ON -DBUILD_TESTING=ON"
+else
+  CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DBUILD_TESTING=OFF"
+   #CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=../ -DEMC_EXEC_DIR=ON -DENABLE_DOCS=ON -DBUILD_TESTING=OFF"
+fi
 
 rm -fr ./build
 mkdir ./build && cd ./build
@@ -37,4 +47,7 @@ cmake .. ${CMAKE_FLAGS}
 make -j 8 VERBOSE=1
 make install
 
+#make test
+#ctest -I 4,5
+
 exit

tests/CMakeLists.txt

@@ -3,7 +3,28 @@
 #
 # Ed Hartnett 2/11/21
 
+# This function is used to download unit test data.
+# It takes two arguments, the URL and the file to
+# be downloaded.
+
+function(PULL_DATA THE_URL THE_FILE)
+  if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}")
+    file(DOWNLOAD
+      ${THE_URL}/${THE_FILE}
+      ${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}
+      SHOW_PROGRESS
+      STATUS status
+      INACTIVITY_TIMEOUT 30
+      )
+    list(GET status 0 status_num)
+    if(NOT status_num EQUAL 0 OR NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/data/${THE_FILE}")
+      message(FATAL_ERROR "Could not download ${THE_FILE}")
+    endif()
+  endif()
+endfunction()
+
 # Add the test subdirecotries.
+# fvcom test only works for Intel. Comment out for now.
 #add_subdirectory(fvcom_tools)
 add_subdirectory(filter_topo)
 add_subdirectory(chgres_cube)

tests/chgres_cube/CMakeLists.txt

@@ -3,7 +3,19 @@
 #
 # George Gayno, Lin Gan, Ed Hartnett, Larissa Reames
 
-# Include cmake to allow parallel I/O tests.
+set(CHGRES_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/chgres_cube")
+
+set(V16SFC_FILE "gfs.v16.sfc.history.nc")
+set(V16SFC_FILE2 "gfs.v16.sfc2.history.nc")
+set(V16ATM_FILE "gfs.v16.atm.history.nc")
+set(V14SFC_FILE "gfs.v14.sfc.history.nemsio")
+set(V15SFC_FILE "gfs.v15.sfc.history.nemsio")
+set(GFS_GRIB_FILE "gfs.t00z.pgrb2.0p50.f000")
+
+foreach(THE_FILE IN LISTS V16SFC_FILE V16SFC_FILE2 V16ATM_FILE V14SFC_FILE V15SFC_FILE GFS_GRIB_FILE) 
+  PULL_DATA(${CHGRES_URL} ${THE_FILE})
+endforeach()
+
 include (LibMPI)
 
 if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")
@@ -16,8 +28,6 @@ include_directories(${PROJECT_SOURCE_DIR})
 
 # Copy necessary test files from the source data directory to the
 # build data directory.
-execute_process( COMMAND ${CMAKE_COMMAND} -E copy
-    ${CMAKE_CURRENT_SOURCE_DIR}/data/files.txt ${CMAKE_CURRENT_BINARY_DIR}/data/files.txt)
 execute_process( COMMAND ${CMAKE_COMMAND} -E copy
     ${CMAKE_CURRENT_SOURCE_DIR}/data/config_fv3_tiled.nml ${CMAKE_CURRENT_BINARY_DIR}/data/config_fv3_tiled.nml)
 execute_process( COMMAND ${CMAKE_COMMAND} -E copy

tests/fvcom_tools/CMakeLists.txt

@@ -1,8 +1,17 @@
-# This is the cmake build file for the tests directory of the
+# This is the cmake build file for the fvcom_tools test directory of the
 # UFS_UTILS project.
 #
 # David Wright
 
+set(FVCOM_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/fvcom_tools")
+
+set(FVCOM_FILE "fvcom_unittest.nc")
+set(SFCDATA_FILE "sfcdata_unittest.nc")
+
+foreach(THE_FILE IN LISTS FVCOM_FILE SFCDATA_FILE)
+  PULL_DATA(${FVCOM_URL} ${THE_FILE})
+endforeach()
+
 if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8 -assume byterecl")
 elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
@@ -13,14 +22,9 @@ include_directories(${PROJECT_SOURCE_DIR})
 
 # Copy necessary test files from the source data directory to the
 # build data directory.
-execute_process( COMMAND ${CMAKE_COMMAND} -E copy
-    ${CMAKE_CURRENT_SOURCE_DIR}/data/sfcdata_unittest.nc ${CMAKE_CURRENT_BINARY_DIR}/sfcdata_unittest.nc)
-execute_process( COMMAND ${CMAKE_COMMAND} -E copy
-    ${CMAKE_CURRENT_SOURCE_DIR}/data/fvcom_unittest.nc ${CMAKE_CURRENT_BINARY_DIR}/fvcom_unittest.nc)
 execute_process( COMMAND ${CMAKE_COMMAND} -E copy
     ${CMAKE_CURRENT_SOURCE_DIR}/LSanSuppress.supp ${CMAKE_CURRENT_BINARY_DIR}/LSanSuppress.supp)
 
 add_executable(ftst_readfvcomnetcdf ftst_readfvcomnetcdf.F90)
 add_test(NAME fvcom_tools-ftst_readfvcomnetcdf COMMAND ftst_readfvcomnetcdf)
 target_link_libraries(ftst_readfvcomnetcdf fvcom_tools_lib)
-

tests/fvcom_tools/ftst_readfvcomnetcdf.F90

@@ -74,8 +74,8 @@ program readfvcomnetcdf
 
  print*,"Starting test of fvcom_tools."
 !Set default file names, cold start, and time str
- fv3file = 'sfcdata_unittest.nc'
- fvcomfile = 'fvcom_unittest.nc'
+ fv3file = './data/sfcdata_unittest.nc'
+ fvcomfile = './data/fvcom_unittest.nc'
  wcstart = 'cold'
  inputFVCOMselStr = '3333-44-55T66:77:88.000000'
  t1 = 1

tests/sfc_climo_gen/CMakeLists.txt

@@ -3,6 +3,20 @@
 #
 # George Gayno, Lin Gan, Ed Hartnett, Larissa Reames
 
+set(SFCGEN_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/ufs_utils/unit_tests/sfc_climo_gen")
+set(MOSAIC_FILE "C450_mosaic.nc")
+set(GRID_FILE "C450_grid.tile7.nc")
+set(ORO_FILE "C450_oro_data.tile7.nc")
+
+foreach(THE_FILE IN LISTS MOSAIC_FILE GRID_FILE ORO_FILE)
+  PULL_DATA(${SFCGEN_URL} ${THE_FILE}) 
+endforeach()
+
+set(SFCGEN2_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/UFS/GFS/fix/fix_sfc_climo")
+set(VEG_FILE "vegetation_type.viirs.igbp.0.1.nc")
+
+PULL_DATA(${SFCGEN2_URL} ${VEG_FILE})
+
 # Include cmake to allow parallel I/O tests.
 include (LibMPI)
 
@@ -16,8 +30,6 @@ include_directories(${PROJECT_SOURCE_DIR})
 
 # Copy necessary test files from the source data directory to the
 # build data directory.
-execute_process( COMMAND ${CMAKE_COMMAND} -E copy
-    ${CMAKE_CURRENT_SOURCE_DIR}/data/files.txt ${CMAKE_CURRENT_BINARY_DIR}/data/files.txt)
 execute_process( COMMAND ${CMAKE_COMMAND} -E copy
     ${CMAKE_CURRENT_SOURCE_DIR}/LSanSuppress.supp ${CMAKE_CURRENT_BINARY_DIR}/LSanSuppress.supp)