Update datetime handling to use dateTime with correct type and fill behavior and simplify environment setup

README.md

@@ -38,6 +38,7 @@ The `obs2ioda-v3` executable will reside in the `bin` directory within the build
 ### Running the Obs2Ioda Test Suite
 #### Running the Unit Test Suite
 1. **Run the unit test suite** using `ctest`. To see detailed output and the list of tests being executed, add the `--verbose` flag:
+1. **Run the unit test suite** using `ctest` from the `obs2ioda` build directory. To see detailed output and the list of tests being executed, add the `--verbose` flag:
    ```bash
    ctest --verbose
    ```

cmake/Obs2Ioda_Test_Functions.cmake

@@ -27,3 +27,31 @@ function(add_cxx_ctest name sources include_directories library_dependencies)
             COMMAND ${name} --gtest_filter=*
     )
 endfunction()
+
+# Adds a Fortran test executable and registers it as a CTest.
+#
+# This function creates an executable target for a Fortran test, sets up
+# library dependencies, and registers the test with CTest.
+#
+# Arguments:
+#   name                 - Name of the test executable.
+#   sources              - List of source files for the test.
+#   library_dependencies - Libraries that the test executable should link against.
+#
+# The function does the following:
+#   1. Creates an executable target with the given name and sources.
+#   3. Links the target against the specified libraries.
+#   4. Registers the executable as a CTest.
+#
+# Example usage:
+#   add_fortran_ctest(my_test "test.f90" "<library_dependencies>")
+#
+function(add_fortran_ctest name sources library_dependencies)
+    add_executable(${name} ${sources})
+    set_target_properties(${name} PROPERTIES LINKER_LANGUAGE Fortran)
+    target_link_libraries(${name} PUBLIC ${library_dependencies})
+    add_test(
+            NAME ${name}
+            COMMAND ${name}
+    )
+endfunction()

env-setup/environment.yml

@@ -0,0 +1,8 @@
+name: obs2ioda
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11  # You can change this version if needed
+  - netCDF4
+  - pytest
+  - requests

env-setup/gnu_derecho.csh

@@ -0,0 +1,32 @@
+#!/bin/csh
+
+# Load modules
+module purge
+module load ncarenv/23.09
+module load gcc/12.2.0 
+module load netcdf/4.9.2
+module load cmake
+module load conda/latest
+
+# Check if conda is available
+if ( ! $?CONDA_EXE ) then
+    echo "Error: conda not found. Check if the 'conda/latest' module loaded correctly."
+    exit 1
+endif
+
+# Check if obs2ioda conda environment exists
+setenv HAS_ENV `conda info --envs | awk '{print $1}' | grep -x "obs2ioda"`
+
+if ( "$HAS_ENV" == "" ) then
+    echo "Creating conda environment 'obs2ioda'..."
+    conda env create -f environment.yml
+endif
+
+# Initialize conda if needed (optional if not already initialized in .cshrc)
+if ( $?CONDA_SHLVL == 0 ) then
+    source `conda info --base`/etc/profile.d/conda.csh
+endif
+
+# Activate the environment
+conda activate obs2ioda
+

env-setup/gnu_derecho.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Load modules
+module --force purge
+module load ncarenv/23.09
+module load gcc/12.2.0
+module load netcdf/4.9.2
+module load cmake
+module load conda/latest 
+
+# Check if conda is available
+if ! command -v conda &> /dev/null; then
+    echo "Error: conda not found. Check if the 'anaconda' module loaded correctly."
+    exit 1
+fi
+
+# Check if obs2ioda conda environment exists
+if ! conda info --envs | awk '{print $1}' | grep -qx "obs2ioda"; then
+    echo "Creating conda environment 'obs2ioda'..."
+    conda env create -f environment.yml
+fi
+
+# Activate the environment
+conda activate obs2ioda
+

env-setup/intel_derecho.csh

@@ -0,0 +1,33 @@
+#!/bin/csh
+
+# Load modules
+module purge
+module load ncarenv/23.09
+module load intel-classic/2023.2.1
+module load netcdf/4.9.2
+module load cmake
+module load conda/latest
+
+# Check if conda is available
+if ( ! $?CONDA_EXE ) then
+    echo "Error: conda not found. Check if the 'conda/latest' module loaded correctly."
+    exit 1
+endif
+
+# Check if obs2ioda conda environment exists
+setenv HAS_ENV `conda info --envs | awk '{print $1}' | grep -x "obs2ioda"`
+
+if ( "$HAS_ENV" == "" ) then
+    echo "Creating conda environment 'obs2ioda'..."
+    conda env create -f environment.yml
+endif
+
+# Initialize conda if needed (optional if not already initialized in .cshrc)
+if ( $?CONDA_SHLVL == 0 ) then
+    source `conda info --base`/etc/profile.d/conda.csh
+endif
+
+# Activate the environment
+conda activate obs2ioda
+
+

env-setup/intel_derecho.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Load modules
+module --force purge
+module load ncarenv/23.09
+module load intel-classic/2023.2.1
+module load netcdf/4.9.2
+module load cmake
+module load conda/latest 
+
+# Check if conda is available
+if ! command -v conda &> /dev/null; then
+    echo "Error: conda not found. Check if the 'anaconda' module loaded correctly."
+    exit 1
+fi
+
+# Check if obs2ioda conda environment exists
+if ! conda info --envs | awk '{print $1}' | grep -qx "obs2ioda"; then
+    echo "Creating conda environment 'obs2ioda'..."
+    conda env create -f environment.yml
+fi
+
+# Activate the environment
+conda activate obs2ioda
+

src/goes_abi_converter.f90

@@ -17,7 +17,7 @@ program Goes_ReBroadcast_converter
 !        /
 
    use define_mod, only:  missing_r
-   use goes_abi_converter_mod, only: write_iodav3_netcdf
+   use goes_abi_converter_mod, only: write_iodav3_netcdf, set_goes_abi_out_fname
 
    implicit none
    include 'netcdf.inc'
@@ -29,7 +29,8 @@ program Goes_ReBroadcast_converter
    integer, parameter  :: i_long   = selected_int_kind(8)   ! long integer
    integer, parameter  :: i_kind   = i_long                 ! default integer
    integer, parameter  :: r_kind   = r_single               ! default real
-   character(len=14), parameter :: BCM_id = 'CG_ABI-L2-ACMC'
+   !   character(len=14), parameter :: BCM_id = 'CG_ABI-L2-ACMC'
+   character(len=14), parameter :: BCM_id = 'OR_ABI-L2-ACMF'
 
    integer(i_kind), parameter :: nband      = 10  ! IR bands 7-16
    integer(i_kind) :: band_start = 7
@@ -342,7 +343,7 @@ program Goes_ReBroadcast_converter
 
    if ( write_iodav3 ) then
       do it = 1, ntime
-         out_fname = trim(data_id)//'_'//sat_id//'_'//time_start(it)//'.nc4'
+         call set_goes_abi_out_fname(out_fname, trim(sat_id), time_start(it))
          write(0,*) 'Writing ', trim(out_fname)
          if ( allocated(rdata(it)%cm) ) then
             call output_iodav3(trim(out_fname), time_start(it), nx, ny, nband, got_latlon, &
@@ -984,8 +985,7 @@ subroutine output_iodav3(fname, time_start, nx, ny, nband, got_latlon, lat, lon,
             end if
 
             iloc = iloc + 1
-            write(unit=datetime(iloc), fmt='(i4,a,i2.2,a,i2.2,a,i2.2,a,i2.2,a,i2.2,a)')  &
-                  iyear, '-', imonth, '-', iday, 'T', ihour, ':', imin, ':', isec, 'Z'
+            call get_julian_time(iyear, imonth, iday, ihour, imin, isec, gstime, datetime(iloc))
 
             ! Super-ob BT for this channel
             do k = 1, nband

src/goes_abi_converter_mod.f90

@@ -44,6 +44,43 @@ subroutine transpose_and_flatten(mat, flat_mat_trans)
         flat_mat_trans = reshape(transpose(mat), [m*n])
     end subroutine transpose_and_flatten
 
+    ! @brief Sets the output filename for GOES ABI data based on satellite ID and time.
+    !
+    ! This subroutine generates the output filename for GOES ABI data by combining the
+    ! satellite ID and the start time. The satellite ID is converted to lowercase,
+    ! and the start time is formatted into the filename as "yyyyMMddhh_mm". The filename
+    ! follows the format:
+    !
+    !   abi_<sat_id_lower>_obs_<time_str>.h5
+    !
+    ! @param fname (inout) The output filename. It will be updated with the generated
+    !        filename based on the satellite ID and time.
+    ! @param sat_id (in) The satellite ID (e.g., "G16") to be used in the filename.
+    ! @param time_start (in) The start time in ISO 8601 format (e.g., "2018-04-15T00:00:41.9Z").
+    !
+    ! @note The satellite ID is converted to lowercase and only the year, month, day,
+    !       and hour from the `time_start` string are used in the generated filename.
+    !       The minute value is extracted from `time_start` and included in the filename.
+    subroutine set_goes_abi_out_fname(fname, sat_id, time_start)
+        use utils_mod, only : to_lower
+        implicit none
+        character(len=*), intent(inout) :: fname
+        character(len=*), intent(in) :: sat_id, time_start
+        character(len=:), allocatable :: sat_id_lower
+        integer :: iyear, imonth, iday, ihour, imin
+        character(len=32) :: time_str
+
+        ! Set the output filename based on satellite ID and time string
+        read(time_start( 1: 4), '(i4)') iyear
+        read(time_start( 6: 7), '(i2)') imonth
+        read(time_start( 9:10), '(i2)') iday
+        read(time_start(12:13), '(i2)') ihour
+        write(time_str, '(i4.4, i2.2, i2.2, i2.2)') iyear, imonth, iday, ihour
+        time_str = trim(time_str) // '_' // time_start(15:16)
+        sat_id_lower = to_lower(trim(sat_id))
+        fname = 'abi_' // trim(sat_id_lower) // '_obs_' // trim(time_str)  // '.h5'
+    end subroutine set_goes_abi_out_fname
+
     ! write_iodav3_netcdf:
     !   Writes GOES-ABI observation data into a NetCDF file formatted for IODA-v3.
     !
@@ -125,9 +162,12 @@ subroutine write_iodav3_netcdf(fname, nlocs, nchans, missing_r, missing_i, &
         call check(netcdfAddVar(ncid, 'solar_zenith_angle', NF90_REAL, 1, ['nlocs'], 'MetaData', fillValue=missing_r))
         call check(netcdfAddVar(ncid, 'sensor_zenith_angle', NF90_REAL, 1, ['nlocs'], 'MetaData', fillValue=missing_r))
         call check(netcdfAddVar(ncid, 'sensor_view_angle', NF90_REAL, 1, ['nlocs'], 'MetaData', fillValue=missing_r))
-        call check(netcdfAddVar(ncid, 'datetime', NF90_INT64, 1, ['nlocs'], 'MetaData', fillValue=missing_i))
+        call check(netcdfAddVar(ncid, 'dateTime', NF90_INT64, 1, ['nlocs'], 'MetaData'))
+        call check(netcdfPutAtt(ncid, "units", "seconds since 1970-01-01T00:00:00Z", 'dateTime', 'MetaData'))
         call check(netcdfAddVar(ncid, 'sensor_channel', NF90_INT, 1, ['nchans'], 'MetaData', fillValue=missing_i))
 
+        call check(netcdfPutVar(ncid, 'nchans', (/7,8,9,10,11,12,13,14,15,16/)))
+
         call transpose_and_flatten(bt_out, rtmp1d)
         call check(netcdfPutVar(ncid, 'brightness_temperature', rtmp1d, 'ObsValue'))
         call transpose_and_flatten(err_out, rtmp1d)
@@ -144,7 +184,7 @@ subroutine write_iodav3_netcdf(fname, nlocs, nchans, missing_r, missing_i, &
         call check(netcdfPutVar(ncid, 'sensor_zenith_angle', sat_zen_out, 'MetaData'))
         call check(netcdfPutVar(ncid, 'sensor_view_angle', sat_zen_out, 'MetaData'))
         call check(netcdfPutVar(ncid, 'sensor_channel', (/7,8,9,10,11,12,13,14,15,16/), 'MetaData'))
-        call check(netcdfPutVar(ncid, 'datetime', datetime, 'MetaData'))
+        call check(netcdfPutVar(ncid, 'dateTime', datetime, 'MetaData'))
         call check(netcdfClose(ncid))
         deallocate(rtmp1d)
     end subroutine write_iodav3_netcdf

src/utils_mod.f90

@@ -9,6 +9,7 @@ module utils_mod
 public :: da_advance_time
 public :: get_julian_time
 public :: da_get_time_slots
+public :: to_lower
 
 contains
 subroutine da_advance_time (date_in, dtime, date_out)
@@ -338,4 +339,34 @@ subroutine da_get_time_slots(nt,tmin,tmax,time_slots)
 
 end subroutine da_get_time_slots
 
+   ! @brief Converts a string to lowercase.
+   !
+   ! This function takes a string as input and converts all uppercase
+   ! alphabetical characters to their corresponding lowercase characters.
+   ! Non-alphabetical characters are left unchanged.
+   !
+   ! @param s The input string to be converted.
+   !          It is passed as an argument with intent(in) and remains unchanged.
+   !
+   ! @return The string in lowercase. The function returns a new string
+   !         with the same length as the input, but with all uppercase
+   !         characters converted to lowercase.
+   !
+   ! @note This function does not handle locale-specific conversions and
+   !       assumes that the characters in the string are ASCII.
+   !
+   function to_lower(s)
+      character(len=*), intent(in) :: s  ! The input string to be converted.
+      character(len=len(s)) :: to_lower  ! The output string in lowercase.
+      integer :: i  ! Loop variable.
+
+      to_lower = s  ! Initialize output string as input string.
+      do i = 1, len(s)
+         if (iachar(to_lower(i:i)) >= iachar('A') .and. iachar(to_lower(i:i)) <= iachar('Z')) then
+            ! Convert uppercase letters to lowercase.
+            to_lower(i:i) = achar(iachar(to_lower(i:i)) - iachar('A') + iachar('a'))
+         end if
+      end do
+   end function to_lower
+
 end module utils_mod

test/fortran/CMakeLists.txt

@@ -4,26 +4,39 @@ set(test_transpose_and_flatten_SOURCES
 set(test_transpose_and_flatten_LIBRARY_DEPENDENCIES
         v3
 )
-add_executable(test_transpose_and_flatten ${test_transpose_and_flatten_SOURCES})
-target_link_libraries(test_transpose_and_flatten
-        PRIVATE
+add_fortran_ctest(test_transpose_and_flatten
+        ${test_transpose_and_flatten_SOURCES}
         ${test_transpose_and_flatten_LIBRARY_DEPENDENCIES}
 )
-add_test(NAME test_transpose_and_flatten
-        COMMAND test_transpose_and_flatten
-)
 
 set(test_get_julian_time_SOURCES
         get_julian_time.test.f90
 )
 set(test_get_julian_time_LIBRARY_DEPENDENCIES
         v3
 )
-add_executable(test_get_julian_time ${test_get_julian_time_SOURCES})
-target_link_libraries(test_get_julian_time
-        PRIVATE
+add_fortran_ctest(test_get_julian_time
+        ${test_get_julian_time_SOURCES}
         ${test_get_julian_time_LIBRARY_DEPENDENCIES}
 )
-add_test(NAME test_get_julian_time
-        COMMAND test_get_julian_time
+set(test_to_lower_SOURCES
+        to_lower.test.f90
+)
+set(test_to_lower_LIBRARY_DEPENDENCIES
+        v3
+)
+add_fortran_ctest(test_to_lower
+        ${test_to_lower_SOURCES}
+        ${test_to_lower_LIBRARY_DEPENDENCIES}
+)
+
+set(test_set_goes_abi_out_fname_SOURCES
+        set_goes_abi_out_fname.test.f90
+)
+set(test_set_goes_abi_out_fname_LIBRARY_DEPENDENCIES
+        v3
+)
+add_fortran_ctest(test_set_goes_abi_out_fname
+        ${test_set_goes_abi_out_fname_SOURCES}
+        ${test_set_goes_abi_out_fname_LIBRARY_DEPENDENCIES}
 )