Initial steps towards making ObsForge run with rocoto

.gitignore

@@ -35,6 +35,7 @@
 bin/
 build*/
 install*/
+__pycache__*/
 
 # Ignore the following bundle repositories
 /ecbuild
@@ -49,3 +50,6 @@ install*/
 /sorc/obsforge-utils
 !/bundle/CMakeLists.txt
 /bundle/*
+
+# rocoto
+*.db

bundle/CMakeLists.txt

@@ -44,6 +44,7 @@ option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF )
 option( ENABLE_QG_MODEL "Build QG toy model" OFF )
 
 # EMC BUFR-query library
+set( SKIP_DOWNLOAD_TEST_DATA OFF )
 ecbuild_bundle( PROJECT bufr-query SOURCE "../sorc/bufr-query" )
 
 # Core JEDI repositories

jobs/JOBSFORGE_GLOBAL_AOD_DUMP

@@ -0,0 +1,45 @@
+#! /usr/bin/env bash
+
+source "${HOMEobsforge}/ush/preamble.sh"
+source "${HOMEobsforge}/ush/jjob_header.sh"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${DUMPAODPY:-${HOMEobsforge}/scripts/exobsforge_global_aod_dump.py}
+${EXSCRIPT}
+status=$?
+if [[ ${status} -ne 0 ]]; then
+    exit "${status}"
+fi
+
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit
+if [[ "${KEEPDATA}" == "NO" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

jobs/rocoto/aoddump.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEobsforge}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEobsforge}/ush/load_obsforge_modules.sh"
+status=$?
+if [[ ${status} -ne 0 ]]; then
+    exit "${status}"
+fi
+
+export job="aoddump"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEobsforge}/jobs/JOBSFORGE_GLOBAL_AOD_DUMP"
+status=$?
+exit "${status}"

parm/config.yaml

@@ -0,0 +1,4 @@
+obsforge:
+  assim_freq: 6
+aoddump:
+  platforms: ['npp', 'n20', 'n21']

parm/obsforge_test_hera.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0"?>
+<!DOCTYPE workflow
+[
+	<!--
+	PROGRAM
+		Main workflow manager for ObsForge
+
+	-->
+	<!ENTITY PSLOT "obsforge">
+	<!ENTITY ROTDIR "/scratch2/NCEPDEV/stmp1/Cory.R.Martin/obsforge/COMROOT/obsforge">
+	<!ENTITY JOBS_DIR "/scratch2/NCEPDEV/stmp1/Cory.R.Martin/mar2025/obsforge/jobs/rocoto">
+	<!ENTITY MAXTRIES "2">
+]>
+
+<workflow realtime="F" scheduler="slurm" cyclethrottle="1" taskthrottle="25">
+
+	<log verbosity="10"><cyclestr>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/obsforge/COMROOT/obsforge/logs/@Y@m@d@H.log</cyclestr></log>
+
+	<!-- Define the cycles -->
+	<cycledef group="gdas">202503160000 202503170000 06:00:00</cycledef>
+	<cycledef group="gfs">202503160000 202503170000 06:00:00</cycledef>
+
+<task name="gfs_aod_dump" cycledefs="gfs" maxtries="&MAXTRIES;">
+
+	<command>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/mar2025/obsforge/jobs/rocoto/aoddump.sh</command>
+
+	<jobname><cyclestr>obsforge_gfs_aod_dump_@H</cyclestr></jobname>
+	<account>da-cpu</account>
+	<queue>batch</queue>
+	<partition>hera</partition>
+	<walltime>00:30:00</walltime>
+	<nodes>1:ppn=1:tpp=1</nodes>
+	<memory>96GB</memory>
+	<native>--export=NONE</native>
+
+	<join><cyclestr>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/obsforge/COMROOT/obsforge/logs/@Y@m@d@H/gfs_aod_dump_prep.log</cyclestr></join>
+
+	<envar><name>RUN_ENVIR</name><value>emc</value></envar>
+	<envar><name>HOMEobsforge</name><value>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/mar2025/obsforge/</value></envar>
+	<envar><name>NET</name><value>gfs</value></envar>
+	<envar><name>RUN</name><value>gfs</value></envar>
+	<envar><name>CDATE</name><value><cyclestr>@Y@m@d@H</cyclestr></value></envar>
+	<envar><name>PDY</name><value><cyclestr>@Y@m@d</cyclestr></value></envar>
+	<envar><name>cyc</name><value><cyclestr>@H</cyclestr></value></envar>
+	<envar><name>KEEPDATA</name><value>NO</value></envar>
+	<envar><name>COMROOT</name><value>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/com</value></envar>
+	<envar><name>DCOMROOT</name><value>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/dcom</value></envar>
+	<envar><name>DATAROOT</name><value>/scratch1/NCEPDEV/stmp2/Cory.R.Martin/RUNDIRS/obsforge/gfs.<cyclestr>@Y@m@d@H</cyclestr></value></envar>
+
+	<!--<dependency>
+		<datadep><cyclestr>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/obsforge/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.updated.status.tm00.bufr_d</cyclestr></datadep>
+	</dependency>
+	-->
+
+</task>
+
+<task name="gdas_aod_dump" cycledefs="gdas" maxtries="&MAXTRIES;">
+
+	<command>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/mar2025/obsforge/jobs/rocoto/aoddump.sh</command>
+
+	<jobname><cyclestr>obsforge_gdas_aod_dump_@H</cyclestr></jobname>
+	<account>da-cpu</account>
+	<queue>batch</queue>
+	<partition>hera</partition>
+	<walltime>00:30:00</walltime>
+	<nodes>1:ppn=1:tpp=1</nodes>
+	<memory>96GB</memory>
+	<native>--export=NONE</native>
+
+	<join><cyclestr>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/obsforge/COMROOT/obsforge/logs/@Y@m@d@H/gdas_aod_dump_prep.log</cyclestr></join>
+
+	<envar><name>RUN_ENVIR</name><value>emc</value></envar>
+	<envar><name>HOMEobsforge</name><value>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/mar2025/obsforge/</value></envar>
+	<envar><name>NET</name><value>gfs</value></envar>
+	<envar><name>RUN</name><value>gdas</value></envar>
+	<envar><name>CDATE</name><value><cyclestr>@Y@m@d@H</cyclestr></value></envar>
+	<envar><name>PDY</name><value><cyclestr>@Y@m@d</cyclestr></value></envar>
+	<envar><name>cyc</name><value><cyclestr>@H</cyclestr></value></envar>
+	<envar><name>KEEPDATA</name><value>NO</value></envar>
+	<envar><name>COMROOT</name><value>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/com</value></envar>
+	<envar><name>DCOMROOT</name><value>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/dcom</value></envar>
+	<envar><name>DATAROOT</name><value>/scratch1/NCEPDEV/stmp2/Cory.R.Martin/RUNDIRS/obsforge/gdas.<cyclestr>@Y@m@d@H</cyclestr></value></envar>
+
+	<!--<dependency>
+		<datadep><cyclestr>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/obsforge/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.updated.status.tm00.bufr_d</cyclestr></datadep>
+	</dependency>
+	-->
+
+</task>
+
+</workflow>

scripts/exobsforge_global_aod_dump.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# exobsforge_global_aod_dump.py
+# This script will collect and preprocess
+# aerosol optical depth observations for
+# global aerosol assimilation 
+import os
+
+from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, parse_j2yaml
+from pyobsforge.task.aero_prepobs import AerosolObsPrep
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config_env = cast_strdict_as_dtypedict(os.environ)
+    # Take configuration from YAML file to augment/append config dict
+    config_yaml = parse_j2yaml(os.path.join(config_env['HOMEobsforge'], 'parm', 'config.yaml'), config_env)
+    # Combine configs together
+    config = AttrDict(**config_env, **config_yaml['obsforge'])
+    config = AttrDict(**config, **config_yaml['aoddump'])
+
+    AeroObs = AerosolObsPrep(config)
+    AeroObs.initialize()
+    AeroObs.execute()
+    AeroObs.finalize()

test/CMakeLists.txt

@@ -301,44 +301,46 @@ foreach(file ${test_aux_files})
         ${CMAKE_CURRENT_BINARY_DIR}/${file} )
 endforeach(file)
 
-# ===================================================================
-# Download the test data ./build/obsForge/test/ if we don't have it
-# ===================================================================
-
-set(DOWNLOAD_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/obsforge")
-set(FILE_COLLECTION "obsforge-0.0.0.tgz")
-#set(FILE_COLLECTION "obsforge-0.0.1.tgz")
-
-# Get the test data if we don't have it.
-if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION}")
-    message(STATUS "Downloading: ${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION}")
-    file(DOWNLOAD
-        ${DOWNLOAD_URL}/${FILE_COLLECTION}
-        ${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION}
-        SHOW_PROGRESS
-        STATUS download_status
-        INACTIVITY_TIMEOUT 30
-    )
-
-  list(GET download_status 0 download_status_num)
+if(NOT SKIP_DOWNLOAD_TEST_DATA)
+  # ===================================================================
+  # Download the test data ./build/obsForge/test/ if we don't have it
+  # ===================================================================
+
+  set(DOWNLOAD_URL "https://ftp.emc.ncep.noaa.gov/static_files/public/obsforge")
+  set(FILE_COLLECTION "obsforge-0.0.0.tgz")
+  #set(FILE_COLLECTION "obsforge-0.0.1.tgz")
+
+  # Get the test data if we don't have it.
+  if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION}")
+      message(STATUS "Downloading: ${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION}")
+      file(DOWNLOAD
+          ${DOWNLOAD_URL}/${FILE_COLLECTION}
+          ${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION}
+          SHOW_PROGRESS
+          STATUS download_status
+          INACTIVITY_TIMEOUT 30
+      )
+
+    list(GET download_status 0 download_status_num)
+
+    if(NOT download_status_num EQUAL 0 OR NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION})
+      # Remove empty file if download doesn't complete
+      file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${BUFR_TAR})
+      message(STATUS "Could not download test files, not building tests")
+      return()
+    endif()
 
-  if(NOT download_status_num EQUAL 0 OR NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION})
-    # Remove empty file if download doesn't complete
-    file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/${BUFR_TAR})
-    message(STATUS "Could not download test files, not building tests")
-    return()
+    # file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${FILE_COLLECTION} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+    add_custom_target(get_obsforge_test_data ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION})
+    add_custom_command(
+      TARGET get_obsforge_test_data
+      POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} tar xzf ${FILE_COLLECTION}
+      COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} rm -rf testdata
+      COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} rm -rf testoutput
+      COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} mv remote_data/testdata testdata
+      COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} mv remote_data/testoutput testoutput
+      COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} rm -rf remote_data)
   endif()
-
-  # file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${FILE_COLLECTION} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-
-  add_custom_target(get_obsforge_test_data ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${FILE_COLLECTION})
-  add_custom_command(
-    TARGET get_obsforge_test_data
-    POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} tar xzf ${FILE_COLLECTION}
-    COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} rm -rf testdata
-    COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} rm -rf testoutput
-    COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} mv remote_data/testdata testdata
-    COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} mv remote_data/testoutput testoutput
-    COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_BINARY_DIR} rm -rf remote_data)
 endif()