Added seaice and sea surface salinity database

parm/config.hera.yaml

@@ -0,0 +1,80 @@
+obsforge:
+  PSLOT: obsforge
+  HOMEobsforge: /scratch1/NCEPDEV/da/Mindo.Choi/test/obsForge
+  SDATE: 202503141800
+  EDATE: 202503150000
+  COMROOT: /scratch1/NCEPDEV/da/Mindo.Choi/test/test_obsForge/COMROOT
+  DCOMROOT: /scratch1/NCEPDEV/da/common/realtime_sample/lfs/h1/ops/prod/dcom
+  DATAROOT: /scratch1/NCEPDEV/da/Mindo.Choi/test/test_obsForge/RUNDIRS
+  SCHEDULER: slurm
+  ACCOUNT: da-cpu
+  QUEUE: debug
+  PARTITION: hera
+  KEEPDATA: NO
+  assim_freq: 6
+
+aoddump:
+  provider: VIIRSAOD
+  platforms: ['npp', 'n20', 'n21']
+  thinning_threshold: 0
+  channel: 4
+  preqc: 2
+  WALLTIME_AOD_DUMP: '00:10:00'
+  TASK_GEOM_AOD_DUMP: '1:ppn=1:tpp=1'
+  MEMORY_AOD_DUMP: 96GB
+
+marinedump:
+  providers:
+    ghrsst:
+      list:
+        - sst_viirs_n21_l3u
+        - sst_viirs_n20_l3u
+        - sst_viirs_npp_l3u
+        - sst_avhrrf_ma_l3u
+        - sst_avhrrf_mb_l3u
+        - sst_avhrrf_mc_l3u
+        - sst_ahi_h08_l3c
+        - sst_abi_g17_l3c
+        - sst_abi_g16_l3c
+      qc config:
+        min: -2.0
+        max: 45.0
+        stride: 15
+        min number of obs: 10
+    rads:
+      list:
+        - rads_adt_3a
+        - rads_adt_3b
+        - rads_adt_6a
+        - rads_adt_c2
+        - rads_adt_j2
+        - rads_adt_j3
+        - rads_adt_sa
+        - rads_adt_sw
+      qc config:
+        min: -2.0
+        max: 3.0
+        error ratio: 1.0
+    nesdis_amsr2:
+      list:
+        - icec_amsr2_north
+        - icec_amsr2_south
+      qc config:
+        min: 0.0
+        max: 1.0
+    smap:
+      list:
+        - sss_smap_l2
+      qc config:
+        min: 0.1
+        max: 40.0
+    smos:
+      list:
+        - sss_smos_l2
+      qc config:
+        min: 0.1
+        max: 40.0
+
+  WALLTIME_MARINE_DUMP: '00:10:00'
+  TASK_GEOM_MARINE_DUMP: '1:ppn=20:tpp=2'
+  MEMORY_MARINE_DUMP: 32GB

parm/config.hercules.yaml

@@ -0,0 +1,80 @@
+obsforge:
+  PSLOT: obsforge
+  HOMEobsforge: /scratch1/NCEPDEV/da/Mindo.Choi/test/obsForge
+  SDATE: 202503141800
+  EDATE: 202503150000
+  COMROOT: /scratch1/NCEPDEV/da/Mindo.Choi/test/test_obsForge/COMROOT
+  DCOMROOT: /scratch1/NCEPDEV/da/common/realtime_sample/lfs/h1/ops/prod/dcom
+  DATAROOT: /scratch1/NCEPDEV/da/Mindo.Choi/test/test_obsForge/RUNDIRS
+  SCHEDULER: slurm
+  ACCOUNT: da-cpu
+  QUEUE: debug
+  PARTITION: hera
+  KEEPDATA: NO
+  assim_freq: 6
+
+aoddump:
+  provider: VIIRSAOD
+  platforms: ['npp', 'n20', 'n21']
+  thinning_threshold: 0
+  channel: 4
+  preqc: 2
+  WALLTIME_AOD_DUMP: '00:10:00'
+  TASK_GEOM_AOD_DUMP: '1:ppn=1:tpp=1'
+  MEMORY_AOD_DUMP: 96GB
+
+marinedump:
+  providers:
+    ghrsst:
+      list:
+        - sst_viirs_n21_l3u
+        - sst_viirs_n20_l3u
+        - sst_viirs_npp_l3u
+        - sst_avhrrf_ma_l3u
+        - sst_avhrrf_mb_l3u
+        - sst_avhrrf_mc_l3u
+        - sst_ahi_h08_l3c
+        - sst_abi_g17_l3c
+        - sst_abi_g16_l3c
+      qc config:
+        min: -2.0
+        max: 45.0
+        stride: 15
+        min number of obs: 10
+    rads:
+      list:
+        - rads_adt_3a
+        - rads_adt_3b
+        - rads_adt_6a
+        - rads_adt_c2
+        - rads_adt_j2
+        - rads_adt_j3
+        - rads_adt_sa
+        - rads_adt_sw
+      qc config:
+        min: -2.0
+        max: 3.0
+        error ratio: 1.0
+    nesdis_amsr2:
+      list:
+        - icec_amsr2_north
+        - icec_amsr2_south
+      qc config:
+        min: 0.0
+        max: 1.0
+    smap:
+      list:
+        - sss_smap_l2
+      qc config:
+        min: 0.1
+        max: 40.0
+    smos:
+      list:
+        - sss_smos_l2
+      qc config:
+        min: 0.1
+        max: 40.0
+
+  WALLTIME_MARINE_DUMP: '00:10:00'
+  TASK_GEOM_MARINE_DUMP: '1:ppn=20:tpp=2'
+  MEMORY_MARINE_DUMP: 32GB

parm/config.orion.yaml

@@ -1,15 +1,15 @@
 obsforge:
   PSLOT: obsforge
-  HOMEobsforge: /work/noaa/da/gvernier/runs/obsForge
+  HOMEobsforge: /scratch1/NCEPDEV/da/Mindo.Choi/test/obsForge
   SDATE: 202503141800
   EDATE: 202503150000
-  COMROOT: /work/noaa/da/gvernier/runs/test_obsforge/COMROOT
-  DCOMROOT: /work2/noaa/da/common/lfs/h1/ops/prod/dcom
-  DATAROOT: /work/noaa/da/gvernier/runs/test_obsforge/RUNDIRS
+  COMROOT: /scratch1/NCEPDEV/da/Mindo.Choi/test/test_obsForge/COMROOT
+  DCOMROOT: /scratch1/NCEPDEV/da/common/realtime_sample/lfs/h1/ops/prod/dcom
+  DATAROOT: /scratch1/NCEPDEV/da/Mindo.Choi/test/test_obsForge/RUNDIRS
   SCHEDULER: slurm
   ACCOUNT: da-cpu
   QUEUE: debug
-  PARTITION: orion
+  PARTITION: hera
   KEEPDATA: NO
   assim_freq: 6
 
@@ -55,6 +55,25 @@ marinedump:
         min: -2.0
         max: 3.0
         error ratio: 1.0
+    nesdis_amsr2:
+      list:
+        - icec_amsr2_north
+        - icec_amsr2_south
+      qc config:
+        min: 0.0
+        max: 1.0
+    smap:
+      list:
+        - sss_smap_l2
+      qc config:
+        min: 0.1
+        max: 40.0
+    smos:
+      list:
+        - sss_smos_l2
+      qc config:
+        min: 0.1
+        max: 40.0
 
   WALLTIME_MARINE_DUMP: '00:10:00'
   TASK_GEOM_MARINE_DUMP: '1:ppn=20:tpp=2'

parm/config.yaml

@@ -1 +1 @@
-config.orion.yaml
+config.hera.yaml

ush/python/pyobsforge/obsdb/nesdis_amsr2_db.py

@@ -0,0 +1,100 @@
+import os
+import glob
+from datetime import datetime
+from pyobsforge.obsdb import BaseDatabase
+
+
+class NesdisAmsr2Database(BaseDatabase):
+    """Class to manage an observation file database for data assimilation."""
+
+    def __init__(self, db_name="nesdis_amsr2.db",
+                 dcom_dir="/lfs/h1/ops/prod/dcom/",
+                 obs_dir="seaice/pda"):
+        base_dir = os.path.join(dcom_dir, '*', obs_dir)
+        super().__init__(db_name, base_dir)
+
+    def create_database(self):
+        """
+        Create the SQLite database and observation files table.
+
+        This method initializes the database with a table named `obs_files` to store metadata
+        about observation files. The table contains the following columns:
+
+        - `id`: A unique identifier for each record (auto-incremented primary key).
+        - `filename`: The full path to the observation file (must be unique).
+        - `obs_time`: The timestamp of the observation, extracted from the filename.
+        - `receipt_time`: The timestamp when the file was added to the `dcom` directory.
+        - `instrument`: The instrument used to collect the observation (e.g., AMSR2).
+        - `satellite`: The satellite from which the observation was collected (e.g., GW1).
+        - `obs_type`: The type of observation (e.g., SEAICE)
+
+        The table is created if it does not already exist.
+        """
+        query = """
+        CREATE TABLE IF NOT EXISTS obs_files (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            filename TEXT UNIQUE,
+            obs_time TIMESTAMP,
+            receipt_time TIMESTAMP,
+            instrument TEXT,
+            satellite TEXT,
+            obs_type TEXT
+        )
+        """
+        self.execute_query(query)
+
+    def parse_filename(self, filename):
+        """Extract metadata from filenames matching the AMSR2-SEAICE pattern."""
+        # Make sure the filename matches the expected pattern
+        # Pattern: AMSR2-SEAICE-NH_v2r2_GW1_s202503140032240_e202503140211220_c202503140245560.nc
+        parts = os.path.basename(filename).split('_')
+
+        # Pre-check: Must be an AMSR2-SEAICE file
+        if not parts[0].startswith("AMSR2-SEAICE"):
+            print(f"[DEBUG] Skipping non AMSR2-SEAICE file: {filename}")
+            return None
+
+        try:
+            # Extract hemisphere from the first hyphen-separated segment
+            name_parts = parts[0].split('-')
+            instrument = name_parts[0]
+            hemisphere = name_parts[2].lower()
+
+            # Determine obs_type
+            if hemisphere == "nh":
+                obs_type = "icec_amsr2_north"
+            elif hemisphere == "sh":
+                obs_type = "icec_amsr2_south"
+            else:
+                print(f"[DEBUG] Unrecognized hemisphere in filename: {filename}")
+                return None
+
+            satellite = parts[2]
+            obs_time = datetime.strptime(parts[3][1:16], "%Y%m%d%H%M%S%f")
+            receipt_time = datetime.fromtimestamp(os.path.getctime(filename))
+            return filename, obs_time, receipt_time, instrument, satellite, obs_type
+
+        except Exception as e:
+            print(f"[DEBUG] Error parsing filename {filename}: {e}")
+            return None
+
+    def ingest_files(self):
+        """Scan the directory for new observation files and insert them into the database."""
+        obs_files = glob.glob(os.path.join(self.base_dir, "*.nc"))
+
+        # Counter for successful ingestions
+        ingested_count = 0
+
+        for file in obs_files:
+            parsed_data = self.parse_filename(file)
+            if parsed_data:
+                query = """
+                    INSERT INTO obs_files (filename, obs_time, receipt_time, instrument, satellite, obs_type)
+                    VALUES (?, ?, ?, ?, ?, ?)
+                """
+                try:
+                    self.insert_record(query, parsed_data)
+                    ingested_count += 1
+                except Exception as e:
+                    print(f"[DEBUG] Failed to insert record for {file}: {e}")
+        print(f"################################ Successfully ingested {ingested_count} files into the database.")

ush/python/pyobsforge/obsdb/rads_db.py

@@ -9,7 +9,7 @@ class RADSDatabase(BaseDatabase):
 
     def __init__(self, db_name="rads.db",
                  dcom_dir="/lfs/h1/ops/prod/dcom/",
-                 obs_dir="sst"):
+                 obs_dir="wgrdbul/adt"):
         base_dir = os.path.join(dcom_dir, '*', obs_dir)
         super().__init__(db_name, base_dir)