Add marine job and make it work on HPC

.github/workflows/build.yaml

@@ -42,12 +42,27 @@ jobs:
       - name: Build inside Docker container
         run: |
           docker run --rm -v ${{ github.workspace }}:/workspace -w /workspace jcsda/docker-gnu-openmpi-dev:latest /bin/bash -c "
-            echo '===== Inside Docker container ${PWD}' &&
+
+            echo '===== Preparing for pytest =====' &&
+            apt-get update &&
+            apt-get install -y libsqlite3-dev &&
+            cd /workspace &&
+            python3 -m venv ~/obsforge &&
+            source ~/obsforge/bin/activate &&
+            pip install -r requirements.txt &&
+
+            echo '===== Building C++ code =====' &&
             mkdir -p build &&
             cd build &&
-            ls /workspace/bundle &&
-            ls /workspace/sorc &&
-            cmake /workspace/bundle/ &&
-            make -j$(nproc) &&
-            cd obsforge-utils && ctest -j$(nproc) --output-on-failure"
+            cmake /workspace/bundle &&
+            make -j\$(nproc) &&
+
+            echo '===== Running C++ tests (ctest) =====' &&
+            cd obsforge-utils &&
+            ctest -j\$(nproc) --output-on-failure &&
+
+            echo '===== Running pytest =====' &&
+            cd /workspace &&
+            pytest scripts/tests --disable-warnings -v"
+
 

.github/workflows/pytest.yaml

@@ -39,3 +39,4 @@ jobs:
         run: |
           source obsdb/bin/activate
           pytest ush/python/pyobsforge/tests/ --disable-warnings -v
+          pytest scripts/tests/ --disable-warnings -v

.gitignore

@@ -53,3 +53,6 @@ __pycache__*/
 
 # rocoto
 *.db
+
+# Ignore pytest working directory
+scripts/tests/tests_output

README.md

@@ -1,2 +1,42 @@
 # obsForge
 Forging next generation of observation processing leveraging JEDI
+
+# Clone and Build
+```
+git clone --recursive --jobs 2 https://github.com/NOAA-EMC/obsForge.git
+cd obsForge
+./build.sh
+```
+
+# Tests
+Load the modules if you have not yet,
+```
+module use modulefiles
+module load obsforge/{hpc}.{compiler}
+```
+
+Testing the bufr to ioda converters:
+```
+cd build/obsForge
+ctest
+```
+
+Testing the non-bufr to ioda converters:
+```
+cd build
+ctest -R test_obsforge_util
+```
+
+
+
+# Workflow usage
+```console
+source ush/of_setup.sh
+setup_xml.py --config config.yaml  --template obsforge_rocoto_template.xml.j2 --output obsforge.xml
+```
+
+load rocoto
+```
+module use /apps/ops/test/nco/modulefiles/core
+module load rocoto
+```

jobs/JOBSFORGE_GLOBAL_MARINE_DUMP

@@ -0,0 +1,45 @@
+#! /usr/bin/env bash
+
+source "${HOMEobsforge}/ush/preamble.sh"
+source "${HOMEobsforge}/ush/jjob_header.sh"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${DUMPMARINEPY:-${HOMEobsforge}/scripts/exobsforge_global_marine_dump.py}
+${EXSCRIPT}
+status=$?
+if [[ ${status} -ne 0 ]]; then
+    exit "${status}"
+fi
+
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit
+if [[ "${KEEPDATA}" == "NO" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

jobs/rocoto/marinedump.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEobsforge}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEobsforge}/ush/load_obsforge_modules.sh"
+status=$?
+if [[ ${status} -ne 0 ]]; then
+    exit "${status}"
+fi
+
+export job="marinedump"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEobsforge}/jobs/JOBSFORGE_GLOBAL_MARINE_DUMP"
+status=$?
+exit "${status}"

parm/config.yaml

@@ -1,19 +1,43 @@
 obsforge:
-  PSLOT: obsforge
-  HOMEobsforge: /scratch2/NCEPDEV/stmp1/Cory.R.Martin/mar2025/obsforge/
-  SDATE: 202503160000
-  EDATE: 202503170000
-  COMROOT: /scratch2/NCEPDEV/stmp1/Cory.R.Martin/obsforge/COMROOT/
-  DCOMROOT: /scratch2/NCEPDEV/stmp1/Cory.R.Martin/dcom
-  DATAROOT: /scratch1/NCEPDEV/stmp2/Cory.R.Martin/RUNDIRS
+  PSLOT: realtimeobs_testing
+  HOMEobsforge: /work2/noaa/da/gvernier/prs/obsForge
+  SDATE: 202503141800
+  EDATE: 202503150000
+  COMROOT: /work2/noaa/da/gvernier/prs/obsForge/realtimeobs_testing/COMROOT
+  DCOMROOT: /work2/noaa/da/common/lfs/h1/ops/prod/dcom
+  DATAROOT: /work2/noaa/da/gvernier/prs/obsForge/realtimeobs_testing/RUNDIRS
   SCHEDULER: slurm
   ACCOUNT: da-cpu
-  QUEUE: batch
-  PARTITION: hera
+  QUEUE: debug
+  PARTITION: hercules
   KEEPDATA: NO
   assim_freq: 6
+
 aoddump:
   platforms: ['npp', 'n20', 'n21']
   WALLTIME_AOD_DUMP: '00:30:00'
   TASK_GEOM_AOD_DUMP: '1:ppn=1:tpp=1'
   MEMORY_AOD_DUMP: 96GB
+
+marinedump:
+  providers:
+    ghrsst:
+      list:
+        - sst_viirs_n21_l3u
+        - sst_viirs_n20_l3u
+        - sst_viirs_npp_l3u
+        - sst_avhrrf_ma_l3u
+        - sst_avhrrf_mb_l3u
+        - sst_avhrrf_mc_l3u
+        - sst_ahi_h08_l3c
+        - sst_abi_g17_l3c
+        - sst_abi_g16_l3c
+      qc config:
+        min: -2
+        max: 45
+        stride: 15
+        min number of obs: 10
+
+  WALLTIME_MARINE_DUMP: '00:30:00'
+  TASK_GEOM_MARINE_DUMP: '1:ppn=40:tpp=2'
+  MEMORY_MARINE_DUMP: 96GB

parm/nc2ioda/nc2ioda.yaml.j2

@@ -0,0 +1,16 @@
+provider: {{ provider }}
+window begin: {{ window_begin | to_isotime }}
+window end: {{ window_end | to_isotime }}
+{% if binning_stride is defined%}
+binning:
+  stride: {{ binning_stride }}
+  min number of obs: {{ binning_min_number_of_obs }}
+{% endif %}
+bounds:
+  min: {{ bounds_min }}
+  max: {{ bounds_max }}
+output file: {{ output_file }}
+{% if ocean_basin is defined %}
+ocean basin: {{ ocean_basin }}
+{% endif %}
+input files: {{ input_files }}

parm/obsforge_rocoto_template.xml.j2

@@ -20,6 +20,11 @@
 	<cycledef group="gdas">{{ SDATE }} {{ EDATE }} 06:00:00</cycledef>
 	<cycledef group="gfs">{{ SDATE }} {{ EDATE }} 06:00:00</cycledef>
 
+<!--
+	============================================
+	Task: gfs_aod_dump
+	============================================
+-->
 <task name="gfs_aod_dump" cycledefs="gfs" maxtries="&MAXTRIES;">
 
 	<command>{{ HOMEobsforge }}/jobs/rocoto/aoddump.sh</command>
@@ -54,6 +59,11 @@
 
 </task>
 
+<!--
+	============================================
+	Task: gdas_aod_dump
+	============================================
+-->
 <task name="gdas_aod_dump" cycledefs="gdas" maxtries="&MAXTRIES;">
 
 	<command>{{ HOMEobsforge }}/jobs/rocoto/aoddump.sh</command>
@@ -88,4 +98,43 @@
 
 </task>
 
+<!--
+	============================================
+	Task: gfs_marine_dump
+	============================================
+-->
+<task name="gfs_marine_dump" cycledefs="gfs" maxtries="&MAXTRIES;">
+
+	<command>{{ HOMEobsforge }}/jobs/rocoto/marinedump.sh</command>
+
+	<jobname><cyclestr>obsforge_gfs_marine_dump_@H</cyclestr></jobname>
+	<account>{{ ACCOUNT }}</account>
+	<queue>{{ QUEUE }}</queue>
+	<partition>{{ PARTITION }}</partition>
+	<walltime>{{ WALLTIME_MARINE_DUMP }}</walltime>
+	<nodes>{{ TASK_GEOM_MARINE_DUMP }}</nodes>
+	<memory>{{ MEMORY_MARINE_DUMP }}</memory>
+	<native>--export=NONE</native>
+
+	<join><cyclestr>{{ COMROOT }}/{{ PSLOT }}/logs/@Y@m@d@H/gfs_marine_dump_prep.log</cyclestr></join>
+
+	<envar><name>RUN_ENVIR</name><value>emc</value></envar>
+	<envar><name>HOMEobsforge</name><value>{{ HOMEobsforge }}</value></envar>
+	<envar><name>NET</name><value>gfs</value></envar>
+	<envar><name>RUN</name><value>gfs</value></envar>
+	<envar><name>CDATE</name><value><cyclestr>@Y@m@d@H</cyclestr></value></envar>
+	<envar><name>PDY</name><value><cyclestr>@Y@m@d</cyclestr></value></envar>
+	<envar><name>cyc</name><value><cyclestr>@H</cyclestr></value></envar>
+	<envar><name>KEEPDATA</name><value>{{ KEEPDATA }}</value></envar>
+	<envar><name>COMROOT</name><value>{{ COMROOT }}</value></envar>
+	<envar><name>DCOMROOT</name><value>{{ DCOMROOT }}</value></envar>
+	<envar><name>DATAROOT</name><value>{{ DATAROOT }}/{{ PSLOT }}/gfs.<cyclestr>@Y@m@d@H</cyclestr></value></envar>
+
+	<!--<dependency>
+		<datadep><cyclestr>/scratch2/NCEPDEV/stmp1/Cory.R.Martin/obsforge/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.updated.status.tm00.bufr_d</cyclestr></datadep>
+	</dependency>
+	-->
+
+</task>
+
 </workflow>

requirements.txt

@@ -15,19 +15,19 @@ mccabe==0.7.0
 numpy
 packaging==24.2
 panda==0.3.1
-pandas==2.2.3
+pandas
 pillow==11.1.0
 pluggy==1.5.0
 pycodestyle==2.12.1
 pyflakes==3.2.0
 pyparsing==3.2.1
 pytest==8.3.5
-python-dateutil==2.9.0.post0
-pytz==2025.1
+python-dateutil
+pytz
 PyYAML==6.0.2
 requests==2.32.3
 six==1.17.0
 tomli==2.2.1
 tzdata==2025.1
-urllib3==2.3.0
+urllib3
 pysqlite3

scripts/exobsforge_global_marine_dump.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# exobsforge_global_marine_dump.py
+# This script will collect and preprocess
+# the ocean and seaice observations for
+# global marine assimilation
+import os
+
+from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, parse_j2yaml
+from pyobsforge.task.marine_prepobs import MarineObsPrep
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config_env = cast_strdict_as_dtypedict(os.environ)
+
+    # Take configuration from YAML file to augment/append config dict
+    config_yaml = parse_j2yaml(os.path.join(config_env['HOMEobsforge'], 'parm', 'config.yaml'), config_env)
+    # Extract obsforge specific configuration
+    obsforge_dict = {}
+    for key, value in config_yaml['obsforge'].items():
+        if key not in config_env.keys():
+            obsforge_dict[key] = value
+
+    # Combine configs together
+    config = AttrDict(**config_env, **obsforge_dict)
+    config = AttrDict(**config, **config_yaml['marinedump'])
+
+    marineObs = MarineObsPrep(config)
+    marineObs.initialize()
+    marineObs.execute()
+    marineObs.finalize()

scripts/tests/conftest.py

@@ -0,0 +1,35 @@
+# tests/conftest.py
+import os
+import pytest
+
+home_obsforge = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
+test_dir = os.path.join(home_obsforge, 'scripts', 'tests', 'tests_output')
+run_dir = os.path.join(test_dir, 'RUNDIRS', 'obsforge')
+comroot = os.path.join(test_dir, 'COMROOT')
+dcomroot = os.path.join(test_dir, 'dcom')
+dataroot = os.path.join(test_dir, 'RUNDIRS')
+
+
+@pytest.fixture(scope="session", autouse=True)
+def set_env_vars():
+    os.environ["HOMEobsforge"] = home_obsforge
+    pythonpath = os.environ.get("PYTHONPATH", "")
+    sorc_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../sorc/wxflow/src"))
+    os.environ["PYTHONPATH"] = f"{pythonpath}:{sorc_path}"
+    os.environ["PYTHONPATH"] += f":{os.path.abspath(os.path.join(os.path.dirname(__file__), '../../ush/python'))}"
+    os.environ["CONFIGYAML"] = os.path.abspath(os.path.join(os.path.dirname(__file__), "config.yaml"))
+    os.environ["DATA"] = run_dir
+    os.environ["COMROOT"] = comroot
+    os.environ["DCOMROOT"] = dcomroot
+    os.environ["DATAROOT"] = dataroot
+
+
+@pytest.fixture(autouse=True, scope="session")
+def isolate_test_output():
+    test_dir = os.path.join(home_obsforge, 'scripts', 'tests', 'tests_output')
+    run_dir = os.path.join(test_dir, 'RUNDIRS', 'obsforge')
+    os.environ["DATA"] = run_dir
+    os.makedirs(test_dir, exist_ok=True)
+    os.makedirs(os.path.join(run_dir), exist_ok=True)
+
+    os.chdir(os.path.join(run_dir))