From ddb3b9cbe249343d4d79d7c112d3c270dc903885 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 20 Mar 2025 11:52:07 -0400 Subject: [PATCH 01/15] wip but works --- ush/python/pygfs/obsdb/da_obs_db.py | 154 ++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 ush/python/pygfs/obsdb/da_obs_db.py diff --git a/ush/python/pygfs/obsdb/da_obs_db.py b/ush/python/pygfs/obsdb/da_obs_db.py new file mode 100644 index 00000000..b649b4b3 --- /dev/null +++ b/ush/python/pygfs/obsdb/da_obs_db.py @@ -0,0 +1,154 @@ +import sqlite3 +import os +import re +import glob +import time +from datetime import datetime, timedelta +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +class ObsDatabase: + """Class to manage an observation file database for data assimilation.""" + + def __init__(self, db_path="obs_files.db", base_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/"): + self.db_path = db_path + self.base_dir = base_dir + self.create_database() + + def create_database(self): + """Create the SQLite database and table if it doesn't exist.""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + + cursor.execute(""" + CREATE TABLE IF NOT EXISTS obs_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + filename TEXT UNIQUE, + obs_time TIMESTAMP, + ingest_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + obs_dir TEXT, + instrument TEXT, + satellite TEXT, + obs_type TEXT + ) + """) + + conn.commit() + conn.close() + + def ingest_files(self, date_str, obs_dir): + """Scan a specific directory for new observation files and insert them into the database.""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + + # Define the directory path based on date and obs_dir + obs_path = os.path.join(self.base_dir, date_str, obs_dir) + #if not os.path.exists(obs_path): + # print(f"Warning: Directory {obs_path} does not exist.") + # conn.close() + # return + + # Regex pattern to extract timestamps, instrument, satellite, and obs_type from filenames + pattern = re.compile(r"(\d{14})-OSPO-L3U_GHRSST-(\w+)-(\w+)_(\w+)-ACSPO.*\.nc") + + # Get list of NetCDF files + obs_files = glob.glob(os.path.join(obs_path, "*.nc")) + + for file in obs_files: + filename = os.path.basename(file) + match = pattern.match(filename) + if match: + obs_time = datetime.strptime(match.group(1), "%Y%m%d%H%M%S") + obs_type = match.group(2) # Example: SSTsubskin + instrument = match.group(3) # Example: AVHRRF + satellite = match.group(4) # Example: MB + + try: + cursor.execute(""" + INSERT INTO obs_files (filename, obs_time, obs_dir, instrument, satellite, obs_type) + VALUES (?, ?, ?, ?, ?, ?) + """, (filename, obs_time, obs_dir, instrument, satellite, obs_type)) + except sqlite3.IntegrityError: + pass # Skip duplicates + + conn.commit() + conn.close() + + def get_valid_files(self, da_cycle, window_hours=3, instrument=None, satellite=None, obs_type=None): + """Retrieve observation files within a DA window, filtered by instrument, satellite, and observation type.""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + + da_time = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + window = timedelta(hours=window_hours) + + query = """ + SELECT filename FROM obs_files + WHERE obs_time BETWEEN ? AND ? + """ + params = [da_time - window, da_time + window] + + if instrument: + query += " AND instrument = ?" + params.append(instrument) + if satellite: + query += " AND satellite = ?" + params.append(satellite) + if obs_type: + query += " AND obs_type = ?" + params.append(obs_type) + + cursor.execute(query, tuple(params)) + valid_files = [row[0] for row in cursor.fetchall()] + conn.close() + + return valid_files + + def watch_directory(self, date_str, obs_dir): + """Monitor the directory and ingest new files in real-time.""" + class FileHandler(FileSystemEventHandler): + """Handles new file creation events.""" + def on_created(event_handler, event): + if event.is_directory: + return + if event.src_path.endswith(".nc"): + self.ingest_files(date_str, obs_dir) + + obs_path = os.path.join(self.base_dir, date_str, obs_dir) + event_handler = FileHandler() + observer = Observer() + observer.schedule(event_handler, obs_path, recursive=False) + observer.start() + + try: + while True: + time.sleep(5) # Adjust polling interval as needed + except KeyboardInterrupt: + observer.stop() + observer.join() + + +# Example Usage +if __name__ == "__main__": + db = ObsDatabase() + + # Define a specific date and observation directory + date_str = "*" # "20250316" + obs_dir = "sst" + + # Ingest existing files + db.ingest_files(date_str, obs_dir) + + # Query files for a given DA cycle + da_cycle = "20250316000000" # Example DA cycle time + valid_files = db.get_valid_files(da_cycle, instrument="VIIRS", satellite="N20", obs_type="SSTsubskin") + print("Valid observation files:", valid_files) + + print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") + for valid_file in valid_files: + resolved_date_str = datetime.strptime(valid_file.split('-')[0], "%Y%m%d%H%M%S").strftime("%Y%m%d") + print(os.path.join(db.base_dir, resolved_date_str, obs_dir, valid_file)) + + + # Start monitoring the directory for new files (Run in a separate process if needed) + # db.watch_directory(date_str, obs_dir) From b13d827f6a499718eb35a9e503c17f20e6a0a63e Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 20 Mar 2025 20:48:39 -0400 Subject: [PATCH 02/15] jrr_aod and ghrsst --- ush/python/pygfs/obsdb/da_obs_db.py | 154 --------------------------- ush/python/pygfs/obsdb/ghrsst_db.py | 67 ++++++++++++ ush/python/pygfs/obsdb/jrr_aod_db.py | 65 +++++++++++ ush/python/pygfs/obsdb/obsdb.py | 103 ++++++++++++++++++ 4 files changed, 235 insertions(+), 154 deletions(-) delete mode 100644 ush/python/pygfs/obsdb/da_obs_db.py create mode 100644 ush/python/pygfs/obsdb/ghrsst_db.py create mode 100644 ush/python/pygfs/obsdb/jrr_aod_db.py create mode 100644 ush/python/pygfs/obsdb/obsdb.py diff --git a/ush/python/pygfs/obsdb/da_obs_db.py b/ush/python/pygfs/obsdb/da_obs_db.py deleted file mode 100644 index b649b4b3..00000000 --- a/ush/python/pygfs/obsdb/da_obs_db.py +++ /dev/null @@ -1,154 +0,0 @@ -import sqlite3 -import os -import re -import glob -import time -from datetime import datetime, timedelta -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - -class ObsDatabase: - """Class to manage an observation file database for data assimilation.""" - - def __init__(self, db_path="obs_files.db", base_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/"): - self.db_path = db_path - self.base_dir = base_dir - self.create_database() - - def create_database(self): - """Create the SQLite database and table if it doesn't exist.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - cursor.execute(""" - CREATE TABLE IF NOT EXISTS obs_files ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - filename TEXT UNIQUE, - obs_time TIMESTAMP, - ingest_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - obs_dir TEXT, - instrument TEXT, - satellite TEXT, - obs_type TEXT - ) - """) - - conn.commit() - conn.close() - - def ingest_files(self, date_str, obs_dir): - """Scan a specific directory for new observation files and insert them into the database.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - # Define the directory path based on date and obs_dir - obs_path = os.path.join(self.base_dir, date_str, obs_dir) - #if not os.path.exists(obs_path): - # print(f"Warning: Directory {obs_path} does not exist.") - # conn.close() - # return - - # Regex pattern to extract timestamps, instrument, satellite, and obs_type from filenames - pattern = re.compile(r"(\d{14})-OSPO-L3U_GHRSST-(\w+)-(\w+)_(\w+)-ACSPO.*\.nc") - - # Get list of NetCDF files - obs_files = glob.glob(os.path.join(obs_path, "*.nc")) - - for file in obs_files: - filename = os.path.basename(file) - match = pattern.match(filename) - if match: - obs_time = datetime.strptime(match.group(1), "%Y%m%d%H%M%S") - obs_type = match.group(2) # Example: SSTsubskin - instrument = match.group(3) # Example: AVHRRF - satellite = match.group(4) # Example: MB - - try: - cursor.execute(""" - INSERT INTO obs_files (filename, obs_time, obs_dir, instrument, satellite, obs_type) - VALUES (?, ?, ?, ?, ?, ?) - """, (filename, obs_time, obs_dir, instrument, satellite, obs_type)) - except sqlite3.IntegrityError: - pass # Skip duplicates - - conn.commit() - conn.close() - - def get_valid_files(self, da_cycle, window_hours=3, instrument=None, satellite=None, obs_type=None): - """Retrieve observation files within a DA window, filtered by instrument, satellite, and observation type.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - da_time = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") - window = timedelta(hours=window_hours) - - query = """ - SELECT filename FROM obs_files - WHERE obs_time BETWEEN ? AND ? - """ - params = [da_time - window, da_time + window] - - if instrument: - query += " AND instrument = ?" - params.append(instrument) - if satellite: - query += " AND satellite = ?" - params.append(satellite) - if obs_type: - query += " AND obs_type = ?" - params.append(obs_type) - - cursor.execute(query, tuple(params)) - valid_files = [row[0] for row in cursor.fetchall()] - conn.close() - - return valid_files - - def watch_directory(self, date_str, obs_dir): - """Monitor the directory and ingest new files in real-time.""" - class FileHandler(FileSystemEventHandler): - """Handles new file creation events.""" - def on_created(event_handler, event): - if event.is_directory: - return - if event.src_path.endswith(".nc"): - self.ingest_files(date_str, obs_dir) - - obs_path = os.path.join(self.base_dir, date_str, obs_dir) - event_handler = FileHandler() - observer = Observer() - observer.schedule(event_handler, obs_path, recursive=False) - observer.start() - - try: - while True: - time.sleep(5) # Adjust polling interval as needed - except KeyboardInterrupt: - observer.stop() - observer.join() - - -# Example Usage -if __name__ == "__main__": - db = ObsDatabase() - - # Define a specific date and observation directory - date_str = "*" # "20250316" - obs_dir = "sst" - - # Ingest existing files - db.ingest_files(date_str, obs_dir) - - # Query files for a given DA cycle - da_cycle = "20250316000000" # Example DA cycle time - valid_files = db.get_valid_files(da_cycle, instrument="VIIRS", satellite="N20", obs_type="SSTsubskin") - print("Valid observation files:", valid_files) - - print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") - for valid_file in valid_files: - resolved_date_str = datetime.strptime(valid_file.split('-')[0], "%Y%m%d%H%M%S").strftime("%Y%m%d") - print(os.path.join(db.base_dir, resolved_date_str, obs_dir, valid_file)) - - - # Start monitoring the directory for new files (Run in a separate process if needed) - # db.watch_directory(date_str, obs_dir) diff --git a/ush/python/pygfs/obsdb/ghrsst_db.py b/ush/python/pygfs/obsdb/ghrsst_db.py new file mode 100644 index 00000000..3b0cb9cb --- /dev/null +++ b/ush/python/pygfs/obsdb/ghrsst_db.py @@ -0,0 +1,67 @@ +import os +import re +import glob +from datetime import datetime, timedelta +from obsdb import BaseDatabase + + +class GhrSstDatabase(BaseDatabase): + """Class to manage an observation file database for data assimilation.""" + + def __init__(self, db_path="obs_files.db", + dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/", + obs_dir="sst", + pattern=re.compile(r"(\d{14})-OSPO-L3U_GHRSST-(\w+)-(\w+)_(\w+)-ACSPO.*\.nc")): + base_dir = os.path.join(dcom_dir, '*', obs_dir) + super().__init__(db_path, base_dir, pattern) + + def create_database(self): + """Create the SQLite database and observation files table.""" + query = """ + CREATE TABLE IF NOT EXISTS obs_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + filename TEXT UNIQUE, + obs_time TIMESTAMP, + ingest_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + instrument TEXT, + satellite TEXT, + obs_type TEXT + ) + """ + self.execute_query(query) + + def parse_filename(self, filename): + """Extract metadata from filenames matching the expected pattern.""" + match = self.pattern.match(filename) + if match: + obs_time = datetime.strptime(match.group(1)[0:12], "%Y%m%d%H%M") + obs_type = match.group(2) if len(match.groups()) > 1 else None + instrument = match.group(3) if len(match.groups()) > 2 else None + satellite = match.group(4) if len(match.groups()) > 3 else None + return filename, obs_time, instrument, satellite, obs_type + return None + + +# Example Usage +if __name__ == "__main__": + db = GhrSstDatabase(db_path="sst_obs.db", + dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/", + obs_dir="sst", + pattern=re.compile(r"(\d{14})-OSPO-L3U_GHRSST-(\w+)-(\w+)_(\w+)-ACSPO.*\.nc")) + + # Check for new files + #db.ingest_files() + + # Query files for a given DA cycle + da_cycle = "20250316000000" + cutoff_time = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) + valid_files = db.get_valid_files(da_cycle, + instrument="AVHRRF", + satellite="MB", + obs_type="SSTsubskin", + cutoff_time=cutoff_time) + print("Valid observation files:", valid_files) + + print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") + for valid_file in valid_files: + print(valid_file) diff --git a/ush/python/pygfs/obsdb/jrr_aod_db.py b/ush/python/pygfs/obsdb/jrr_aod_db.py new file mode 100644 index 00000000..39d37441 --- /dev/null +++ b/ush/python/pygfs/obsdb/jrr_aod_db.py @@ -0,0 +1,65 @@ +import os +import re +import glob +from datetime import datetime, timedelta +from obsdb import BaseDatabase + +class JrrAodDatabase(BaseDatabase): + """Class to manage an observation file database for JRR-AOD data.""" + + def __init__(self, db_path="jrr_aod_obs.db", + dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/", + obs_dir="jrr_aod", + pattern=re.compile(r"JRR-AOD_v\d+r\d+_n\d+_s(\d{15})_e\d{15}_c\d{15}\.nc")): + base_dir = os.path.join(dcom_dir, '*', obs_dir) + super().__init__(db_path, base_dir, pattern) + + def create_database(self): + """Create the SQLite database and observation files table.""" + query = """ + CREATE TABLE IF NOT EXISTS obs_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + filename TEXT UNIQUE, + obs_time TIMESTAMP, + ingest_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + self.execute_query(query) + + def parse_filename(self, filename): + """Extract metadata from filenames matching the JRR-AOD pattern.""" + match = self.pattern.match(filename) + if match: + obs_time = datetime.strptime(match.group(1)[:12], "%Y%m%d%H%M") # Extract only YYYYMMDDHHMM + return filename, obs_time + return None + + def ingest_files(self): + """Scan the directory for new JRR-AOD observation files and insert them into the database.""" + obs_files = glob.glob(os.path.join(self.base_dir, "*.nc")) + + for file in obs_files: + filename = os.path.basename(file) + parsed_data = self.parse_filename(filename) + if parsed_data: + query = """ + INSERT INTO obs_files (filename, obs_time) + VALUES (?, ?) + """ + self.insert_record(query, parsed_data) + +if __name__ == "__main__": + db = JrrAodDatabase() + + # Check for new files + db.ingest_files() + + # Query files for a given DA cycle + da_cycle = "20250316120000" + cutoff_time = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) + valid_files = db.get_valid_files(da_cycle) #, cutoff_time=cutoff_time) + + print("Valid observation files:", valid_files) + print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") + for valid_file in valid_files: + print(valid_file) diff --git a/ush/python/pygfs/obsdb/obsdb.py b/ush/python/pygfs/obsdb/obsdb.py new file mode 100644 index 00000000..58a28d63 --- /dev/null +++ b/ush/python/pygfs/obsdb/obsdb.py @@ -0,0 +1,103 @@ +import sqlite3 +import os +import re +import glob +from datetime import datetime, timedelta + + +class BaseDatabase: + """Base class for managing different types of file-based databases.""" + + def __init__(self, db_path, base_dir, pattern): + """ + Initialize the database. + + :param db_path: Path to the SQLite database file. + :param base_dir: Directory containing observation files. + :param pattern: Regular expression pattern for extracting metadata. + """ + self.db_path = db_path + self.base_dir = base_dir + self.pattern = pattern + self.create_database() + + def create_database(self): + """Create the SQLite database. Must be implemented by subclasses.""" + raise NotImplementedError("Subclasses must implement create_database method") + + def parse_filename(self, filename): + """Parse a filename and extract relevant metadata. Must be implemented by subclasses.""" + raise NotImplementedError("Subclasses must implement parse_filename method") + + def insert_record(self, query, params): + """Insert a record into the database.""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + try: + cursor.execute(query, params) + conn.commit() + except sqlite3.IntegrityError: + pass # Skip duplicates + finally: + conn.close() + + def execute_query(self, query, params=None): + """Execute a query and return the results.""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + cursor.execute(query, params or []) + results = cursor.fetchall() + conn.close() + return results + + def ingest_files(self): + """Scan the directory for new observation files and insert them into the database.""" + obs_files = glob.glob(os.path.join(self.base_dir, "*.nc")) + print(f"Found {len(obs_files)} new files to ingest") + #print(obs_files) + for file in obs_files: + filename = os.path.basename(file) + parsed_data = self.parse_filename(filename) + if parsed_data: + query = """ + INSERT INTO obs_files (filename, obs_time, instrument, satellite, obs_type) + VALUES (?, ?, ?, ?, ?) + """ + self.insert_record(query, parsed_data) + + def get_valid_files(self, da_cycle, window_hours=3, instrument=None, satellite=None, obs_type=None, cutoff_time=None): + """Retrieve observation files within a DA window, filtered by instrument, satellite, observation type, and cutoff time.""" + da_time = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + window = timedelta(hours=window_hours) + + query = """ + SELECT filename FROM obs_files + WHERE obs_time BETWEEN ? AND ? + """ + params = [da_time - window, da_time + window] + + if instrument: + query += " AND instrument = ?" + params.append(instrument) + if satellite: + query += " AND satellite = ?" + params.append(satellite) + if obs_type: + query += " AND obs_type = ?" + params.append(obs_type) + + results = self.execute_query(query, tuple(params)) + valid_files = [] + for row in results: + valid_files.append(row[0]) + + if cutoff_time: + filtered_files = [] + for file in valid_files: + file_time_str = file.split('-')[0] + file_time = datetime.strptime(file_time_str, "%Y%m%d%H%M%S") + if file_time <= cutoff_time: + filtered_files.append(file) + valid_files = filtered_files + + return valid_files From 31908c719ad6990f77eb6eb159f513063f6e3651 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 15:14:21 -0400 Subject: [PATCH 03/15] added pytests --- .github/workflows/pytest.yaml | 31 +++ .github/workflows/requirements.txt | 33 ++++ ush/python/pygfs/obsdb/__init__.py | 1 + ush/python/pygfs/obsdb/ghrsst_db.py | 56 +++--- ush/python/pygfs/obsdb/jrr_aod_db.py | 41 ++-- ush/python/pygfs/obsdb/obsdb.py | 88 ++++----- ush/python/pygfs/tests/__init__.py | 0 .../pygfs/tests/test_ghrsst_database.py | 142 ++++++++++++++ .../pygfs/tests/test_jrr_aod_database.py | 181 ++++++++++++++++++ 9 files changed, 488 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/pytest.yaml create mode 100644 .github/workflows/requirements.txt create mode 100644 ush/python/pygfs/obsdb/__init__.py create mode 100644 ush/python/pygfs/tests/__init__.py create mode 100644 ush/python/pygfs/tests/test_ghrsst_database.py create mode 100644 ush/python/pygfs/tests/test_jrr_aod_database.py diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml new file mode 100644 index 00000000..4590c21c --- /dev/null +++ b/.github/workflows/pytest.yaml @@ -0,0 +1,31 @@ +name: Run Tests on Pull Request + +on: + pull_request: + branches: + - develop # Change this if you want to run on other branches + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 # Clones the repository + + - name: Set Up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install Dependencies + run: | + python -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install -r requirements.txt + + - name: Run Pytest + run: | + source venv/bin/activate + pytest --maxfail=1 --disable-warnings -v \ No newline at end of file diff --git a/.github/workflows/requirements.txt b/.github/workflows/requirements.txt new file mode 100644 index 00000000..2ce041f1 --- /dev/null +++ b/.github/workflows/requirements.txt @@ -0,0 +1,33 @@ +certifi==2025.1.31 +charset-normalizer==3.4.1 +contourpy==1.3.0 +cycler==0.12.1 +exceptiongroup==1.2.2 +flake8==7.1.2 +fonttools==4.56.0 +idna==3.10 +iniconfig==2.0.0 +Jinja2==3.1.6 +kiwisolver==1.4.7 +MarkupSafe==3.0.2 +matplotlib +mccabe==0.7.0 +numpy +packaging==24.2 +panda==0.3.1 +pandas==2.2.3 +pillow==11.1.0 +pluggy==1.5.0 +pycodestyle==2.12.1 +pyflakes==3.2.0 +pyparsing==3.2.1 +pytest==8.3.5 +python-dateutil==2.9.0.post0 +pytz==2025.1 +PyYAML==6.0.2 +requests==2.32.3 +six==1.17.0 +tomli==2.2.1 +tzdata==2025.1 +urllib3==2.3.0 +pysqlite3 diff --git a/ush/python/pygfs/obsdb/__init__.py b/ush/python/pygfs/obsdb/__init__.py new file mode 100644 index 00000000..96881197 --- /dev/null +++ b/ush/python/pygfs/obsdb/__init__.py @@ -0,0 +1 @@ +from .obsdb import BaseDatabase \ No newline at end of file diff --git a/ush/python/pygfs/obsdb/ghrsst_db.py b/ush/python/pygfs/obsdb/ghrsst_db.py index 3b0cb9cb..966f556b 100644 --- a/ush/python/pygfs/obsdb/ghrsst_db.py +++ b/ush/python/pygfs/obsdb/ghrsst_db.py @@ -1,19 +1,17 @@ import os -import re import glob -from datetime import datetime, timedelta +from datetime import datetime from obsdb import BaseDatabase class GhrSstDatabase(BaseDatabase): """Class to manage an observation file database for data assimilation.""" - def __init__(self, db_path="obs_files.db", + def __init__(self, db_name="obs_files.db", dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/", - obs_dir="sst", - pattern=re.compile(r"(\d{14})-OSPO-L3U_GHRSST-(\w+)-(\w+)_(\w+)-ACSPO.*\.nc")): + obs_dir="sst"): base_dir = os.path.join(dcom_dir, '*', obs_dir) - super().__init__(db_path, base_dir, pattern) + super().__init__(db_name, base_dir) def create_database(self): """Create the SQLite database and observation files table.""" @@ -32,36 +30,50 @@ def create_database(self): def parse_filename(self, filename): """Extract metadata from filenames matching the expected pattern.""" - match = self.pattern.match(filename) - if match: - obs_time = datetime.strptime(match.group(1)[0:12], "%Y%m%d%H%M") - obs_type = match.group(2) if len(match.groups()) > 1 else None - instrument = match.group(3) if len(match.groups()) > 2 else None - satellite = match.group(4) if len(match.groups()) > 3 else None + parts = os.path.basename(filename).replace('_', '-').split('-') + if len(parts) >= 6 and parts[0].isdigit() and len(parts[0]) == 14: + obs_time = datetime.strptime(parts[0][0:12], "%Y%m%d%H%M") + obs_type = parts[4] if len(parts) > 2 else None + instrument = parts[5] if len(parts) > 3 else None + satellite = parts[6] if len(parts) > 4 else None return filename, obs_time, instrument, satellite, obs_type return None + def ingest_files(self): + """Scan the directory for new observation files and insert them into the database.""" + obs_files = glob.glob(os.path.join(self.base_dir, "*.nc")) + print(f"Found {len(obs_files)} new files to ingest") + for file in obs_files: + parsed_data = self.parse_filename(file) + if parsed_data: + query = """ + INSERT INTO obs_files (filename, obs_time, instrument, satellite, obs_type) + VALUES (?, ?, ?, ?, ?) + """ + self.insert_record(query, parsed_data) + # Example Usage if __name__ == "__main__": - db = GhrSstDatabase(db_path="sst_obs.db", + db = GhrSstDatabase(db_name="sst_obs.db", dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/", - obs_dir="sst", - pattern=re.compile(r"(\d{14})-OSPO-L3U_GHRSST-(\w+)-(\w+)_(\w+)-ACSPO.*\.nc")) + obs_dir="sst") # Check for new files - #db.ingest_files() + db.ingest_files() # Query files for a given DA cycle da_cycle = "20250316000000" - cutoff_time = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) + cutoff_delta = 4 valid_files = db.get_valid_files(da_cycle, - instrument="AVHRRF", - satellite="MB", + instrument="VIIRS", + satellite="NPP", obs_type="SSTsubskin", - cutoff_time=cutoff_time) - print("Valid observation files:", valid_files) + cutoff_delta=cutoff_delta) print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") for valid_file in valid_files: - print(valid_file) + if os.path.exists(valid_file): + print(f"Valid file: {valid_file}") + else: + print(f"File does not exist: {valid_file}") diff --git a/ush/python/pygfs/obsdb/jrr_aod_db.py b/ush/python/pygfs/obsdb/jrr_aod_db.py index 39d37441..bcfd66e8 100644 --- a/ush/python/pygfs/obsdb/jrr_aod_db.py +++ b/ush/python/pygfs/obsdb/jrr_aod_db.py @@ -1,18 +1,17 @@ import os -import re import glob -from datetime import datetime, timedelta +from datetime import datetime from obsdb import BaseDatabase +import re class JrrAodDatabase(BaseDatabase): """Class to manage an observation file database for JRR-AOD data.""" - def __init__(self, db_path="jrr_aod_obs.db", + def __init__(self, db_name="jrr_aod_obs.db", dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/", - obs_dir="jrr_aod", - pattern=re.compile(r"JRR-AOD_v\d+r\d+_n\d+_s(\d{15})_e\d{15}_c\d{15}\.nc")): + obs_dir="jrr_aod"): base_dir = os.path.join(dcom_dir, '*', obs_dir) - super().__init__(db_path, base_dir, pattern) + super().__init__(db_name, base_dir) def create_database(self): """Create the SQLite database and observation files table.""" @@ -28,19 +27,25 @@ def create_database(self): def parse_filename(self, filename): """Extract metadata from filenames matching the JRR-AOD pattern.""" - match = self.pattern.match(filename) - if match: - obs_time = datetime.strptime(match.group(1)[:12], "%Y%m%d%H%M") # Extract only YYYYMMDDHHMM - return filename, obs_time + # Make sure the filename matches the expected pattern + # Pattern: JRR-AOD_v3r2_n21_sYYYYMMDDHHMMSS_eYYYYMMDDHHMMSS_cYYYYMMDDHHMMSS.nc + basename = os.path.basename(filename) + parts = basename.split('_') + try: + if len(parts) >= 4 and parts[0] == "JRR-AOD": + obs_time = datetime.strptime(parts[3][1:13], "%Y%m%d%H%M") + return filename, obs_time + except ValueError: + return None + return None def ingest_files(self): """Scan the directory for new JRR-AOD observation files and insert them into the database.""" obs_files = glob.glob(os.path.join(self.base_dir, "*.nc")) - + print(f"Found {len(obs_files)} new files to ingest") for file in obs_files: - filename = os.path.basename(file) - parsed_data = self.parse_filename(filename) + parsed_data = self.parse_filename(file) if parsed_data: query = """ INSERT INTO obs_files (filename, obs_time) @@ -56,10 +61,12 @@ def ingest_files(self): # Query files for a given DA cycle da_cycle = "20250316120000" - cutoff_time = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) - valid_files = db.get_valid_files(da_cycle) #, cutoff_time=cutoff_time) + cutoff_delta = 5 #datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) + valid_files = db.get_valid_files(da_cycle, cutoff_delta=cutoff_delta) - print("Valid observation files:", valid_files) print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") for valid_file in valid_files: - print(valid_file) + if os.path.exists(valid_file): + print(f"Valid file: {valid_file}") + else: + print(f"File does not exist: {valid_file}") diff --git a/ush/python/pygfs/obsdb/obsdb.py b/ush/python/pygfs/obsdb/obsdb.py index 58a28d63..1eb830a6 100644 --- a/ush/python/pygfs/obsdb/obsdb.py +++ b/ush/python/pygfs/obsdb/obsdb.py @@ -1,80 +1,85 @@ import sqlite3 -import os -import re -import glob from datetime import datetime, timedelta +import sys +import os +# TODO(G): Hack to import wxflow module, do this properly. +sys.path.append(os.path.join(os.path.dirname(__file__), '../../../../sorc/wxflow/src/')) +from wxflow.sqlitedb import SQLiteDB -class BaseDatabase: +class BaseDatabase(SQLiteDB): """Base class for managing different types of file-based databases.""" - def __init__(self, db_path, base_dir, pattern): + def __init__(self, db_name: str, base_dir: str) -> None: """ Initialize the database. - :param db_path: Path to the SQLite database file. + :param db_name: Path to the SQLite database file. :param base_dir: Directory containing observation files. - :param pattern: Regular expression pattern for extracting metadata. """ - self.db_path = db_path + super().__init__(db_name) self.base_dir = base_dir - self.pattern = pattern self.create_database() def create_database(self): """Create the SQLite database. Must be implemented by subclasses.""" raise NotImplementedError("Subclasses must implement create_database method") - def parse_filename(self, filename): + def get_connection(self): + """Return the database connection.""" + return self.connection + + def parse_filename(self): """Parse a filename and extract relevant metadata. Must be implemented by subclasses.""" raise NotImplementedError("Subclasses must implement parse_filename method") - def insert_record(self, query, params): + def ingest_files(self): + """Scan the directory for new observation files and insert them into the database.""" + raise NotImplementedError("Subclasses must implement ingest_files method") + + def insert_record(self, query: str, params: tuple) -> None: """Insert a record into the database.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() + self.connect() + cursor = self.connection.cursor() try: cursor.execute(query, params) - conn.commit() + self.connection.commit() except sqlite3.IntegrityError: pass # Skip duplicates finally: - conn.close() + self.disconnect() - def execute_query(self, query, params=None): + def execute_query(self, query: str, params: tuple = None) -> list: """Execute a query and return the results.""" - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() + self.connect() + cursor = self.connection.cursor() cursor.execute(query, params or []) results = cursor.fetchall() - conn.close() + self.disconnect() return results - def ingest_files(self): - """Scan the directory for new observation files and insert them into the database.""" - obs_files = glob.glob(os.path.join(self.base_dir, "*.nc")) - print(f"Found {len(obs_files)} new files to ingest") - #print(obs_files) - for file in obs_files: - filename = os.path.basename(file) - parsed_data = self.parse_filename(filename) - if parsed_data: - query = """ - INSERT INTO obs_files (filename, obs_time, instrument, satellite, obs_type) - VALUES (?, ?, ?, ?, ?) - """ - self.insert_record(query, parsed_data) - - def get_valid_files(self, da_cycle, window_hours=3, instrument=None, satellite=None, obs_type=None, cutoff_time=None): - """Retrieve observation files within a DA window, filtered by instrument, satellite, observation type, and cutoff time.""" + def get_valid_files(self, + da_cycle: str, + window_hours: int = 3, + instrument: str = None, + satellite: str = None, + obs_type: str = None, + cutoff_delta: int = 0) -> list: + """ + Retrieve a list of observation files within a DA window, possibly filtered by instrument, + satellite, observation type, and cutoff delta (known latency to emulate the early cycle if needed). + """ da_time = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") window = timedelta(hours=window_hours) + cutoff_delta = timedelta(hours=cutoff_delta) + window_begin = da_time - window + window_end = da_time + window - cutoff_delta query = """ SELECT filename FROM obs_files WHERE obs_time BETWEEN ? AND ? """ - params = [da_time - window, da_time + window] + params = [window_begin, window_end] if instrument: query += " AND instrument = ?" @@ -91,13 +96,4 @@ def get_valid_files(self, da_cycle, window_hours=3, instrument=None, satellite=N for row in results: valid_files.append(row[0]) - if cutoff_time: - filtered_files = [] - for file in valid_files: - file_time_str = file.split('-')[0] - file_time = datetime.strptime(file_time_str, "%Y%m%d%H%M%S") - if file_time <= cutoff_time: - filtered_files.append(file) - valid_files = filtered_files - return valid_files diff --git a/ush/python/pygfs/tests/__init__.py b/ush/python/pygfs/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ush/python/pygfs/tests/test_ghrsst_database.py b/ush/python/pygfs/tests/test_ghrsst_database.py new file mode 100644 index 00000000..debaf9ce --- /dev/null +++ b/ush/python/pygfs/tests/test_ghrsst_database.py @@ -0,0 +1,142 @@ +import os +import tempfile +import shutil +import sqlite3 +from datetime import datetime + +import pytest + +from obsdb.ghrsst_db import GhrSstDatabase # Adjust as needed + + +@pytest.fixture +def temp_obs_dir(): + """Create a temp directory with mock GHRSST NetCDF files.""" + base_dir = tempfile.mkdtemp() + sub_dir = os.path.join(base_dir, "some_subdir", "sst") + os.makedirs(sub_dir) + + filenames = [ + "20250316100000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO.nc", + "20250316120000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO.nc", + "20250316150000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO.nc", + "20250316100000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MC-ACSPO.nc", + "20250316120000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MC-ACSPO.nc", + "20250316150000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MC-ACSPO.nc", + "20250316100000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO.nc", + "20250316120000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO.nc", + "20250316150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO.nc", + "20250316100000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO.nc", + "20250316120000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO.nc", + "20250316150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO.nc", + "20250316100000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N21-ACSPO.nc", + "20250316120000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N21-ACSPO.nc", + "20250316150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N21-ACSPO.nc", + "invalid_file.nc" + ] + for fname in filenames: + with open(os.path.join(sub_dir, fname), "w") as f: + f.write("fake content") + + yield base_dir + shutil.rmtree(base_dir) + + +@pytest.fixture +def db(temp_obs_dir): + """Initialize test database.""" + db_path = os.path.join(temp_obs_dir, "ghrsst_test.db") + return GhrSstDatabase(db_name=db_path, dcom_dir=temp_obs_dir, obs_dir="sst") + + +def test_create_database(db): + db.create_database() + conn = sqlite3.connect(db.db_name) + cursor = conn.cursor() + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='obs_files'") + assert cursor.fetchone() is not None + conn.close() + + +def test_parse_valid_filename(db): + fname = "20250316123400-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO.nc" + parsed = db.parse_filename(fname) + assert parsed is not None + assert parsed[0] == fname + assert parsed[1] == datetime(2025, 3, 16, 12, 34) + assert parsed[2] == "AVHRRF" + assert parsed[3] == "MB" + assert parsed[4] == "SSTsubskin" + + +def test_parse_invalid_filename(db): + assert db.parse_filename("junk.nc") is None + assert db.parse_filename("20250316_invalid_filename.nc") is None + + +def test_ingest_files(db): + db.ingest_files() + conn = sqlite3.connect(db.db_name) + cursor = conn.cursor() + cursor.execute("SELECT COUNT(*) FROM obs_files") + count = cursor.fetchone()[0] + conn.close() + assert count == 15, "Should ingest 3 valid GHRSST files" + + +def test_get_valid_files(db): + db.ingest_files() + da_cycle = "20250316120000" + cutoff_delta = 3 # hours + + # Test for AVHRRF_MB + valid_files = db.get_valid_files(da_cycle, + instrument="AVHRRF", + satellite="MB", + obs_type="SSTsubskin", + cutoff_delta=cutoff_delta) + + # Files at 10:00 and 12:00 are within +/- 3h of 12:00 + assert any("202503161000" in f for f in valid_files) + assert any("202503161200" in f for f in valid_files) + assert all("202503161500" not in f for f in valid_files) + assert len(valid_files) == 2 + + # Test for VIIRS_NPP + valid_files = db.get_valid_files(da_cycle, + instrument="VIIRS", + satellite="NPP", + obs_type="SSTsubskin", + cutoff_delta=cutoff_delta) + + # Files at 10:00 and 12:00 are within +/- 3h of 12:00 + assert any("202503161000" in f for f in valid_files) + assert any("202503161200" in f for f in valid_files) + assert all("202503161500" not in f for f in valid_files) + assert len(valid_files) == 2 + + # Test for VIIRS_N20 + valid_files = db.get_valid_files(da_cycle, + instrument="VIIRS", + satellite="N20", + obs_type="SSTsubskin", + cutoff_delta=cutoff_delta) + + # Files at 10:00 and 12:00 are within +/- 3h of 12:00 + assert any("202503161000" in f for f in valid_files) + assert any("202503161200" in f for f in valid_files) + assert all("202503161500" not in f for f in valid_files) + assert len(valid_files) == 2 + + # Test for VIIRS_N21 + valid_files = db.get_valid_files(da_cycle, + instrument="VIIRS", + satellite="N21", + obs_type="SSTsubskin", + cutoff_delta=cutoff_delta) + + # Files at 10:00 and 12:00 are within +/- 3h of 12:00 + assert any("202503161000" in f for f in valid_files) + assert any("202503161200" in f for f in valid_files) + assert all("202503161500" not in f for f in valid_files) + assert len(valid_files) == 2 diff --git a/ush/python/pygfs/tests/test_jrr_aod_database.py b/ush/python/pygfs/tests/test_jrr_aod_database.py new file mode 100644 index 00000000..f435fad9 --- /dev/null +++ b/ush/python/pygfs/tests/test_jrr_aod_database.py @@ -0,0 +1,181 @@ +import os +import tempfile +import shutil +import sqlite3 +from datetime import datetime, timedelta + +import pytest + +from obsdb.jrr_aod_db import JrrAodDatabase + + +@pytest.fixture +def temp_obs_dir(): + """Create a temporary directory with mock JRR-AOD NetCDF files.""" + base_dir = tempfile.mkdtemp() + sub_dir = os.path.join(base_dir, "some_subdir", "jrr_aod") + os.makedirs(sub_dir) + + # Create mock NetCDF files (content doesn't matter) + filenames = [ + "JRR-AOD_v3r2_n21_s202503161000000_e202503161030000_c202503161045000.nc", + "JRR-AOD_v3r2_n21_s202503161200000_e202503161230000_c202503161245000.nc", + "JRR-AOD_v3r2_n21_s202503161500000_e202503161530000_c202503161545000.nc", + "invalid_file.nc" + ] + for fname in filenames: + with open(os.path.join(sub_dir, fname), "w") as f: + f.write("dummy") + + yield base_dir + shutil.rmtree(base_dir) + + +@pytest.fixture +def db(temp_obs_dir): + """ + Create an instance of JrrAodDatabase using in-memory SQLite and + the temp_obs_dir, then initialize the database. + """ + db_path = os.path.join(temp_obs_dir, "test_jrr_aod.db") + database = JrrAodDatabase( + db_name=db_path, + dcom_dir=temp_obs_dir, + obs_dir="jrr_aod" + ) + #database.create_database() + #database.ingest_files() + #database.connect() + #print('db:', database) + return database + + +def test_create_database(db): + """ + Test the creation of the database and the 'obs_files' table. + + This test performs the following steps: + 1. Creates the database. + 2. Ingests files into the database. + 3. Connects to the database. + 4. Checks for the existence of the 'obs_files' table. + + Args: + db: The database object to be tested. + + Asserts: + - The 'obs_files' table is created in the database. + """ + db.create_database() + db.ingest_files() + db.connect() + conn = db.get_connection() + cursor = conn.cursor() + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='obs_files'") + result = cursor.fetchall() + conn.close() + + assert result is not None, "obs_files table should be created" + + +def test_parse_valid_filename(db): + """ + Test the parsing of a valid filename in the database. + + This test performs the following steps: + 1. Creates the database. + 2. Ingests files into the database. + 3. Connects to the database. + 4. Checks the tables in the database. + 5. Parses a given filename and verifies the parsed output. + + Args: + db: The database object to be tested. + + Asserts: + - The parsed filename is not None. + - The first element of the parsed result matches the original filename. + - The second element of the parsed result matches the expected datetime. + """ + db.create_database() + db.ingest_files() + db.connect() + fname = "JRR-AOD_v3r2_n21_s202503161234567_e202503161300000_c202503161315000.nc" + conn = db.get_connection() + cursor = conn.cursor() + cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") + tables = cursor.fetchall() + conn.close() + parsed = db.parse_filename(fname) + + assert parsed is not None + assert parsed[0] == fname + assert parsed[1] == datetime(2025, 3, 16, 12, 34) + + +def test_parse_invalid_filename(db): + """ + Test the `parse_filename` method of the database with invalid filenames. + + This test ensures that the `parse_filename` method returns `None` when provided + with filenames that do not conform to the expected format. + + Args: + db: The database object that contains the `parse_filename` method. + + Assertions: + - Asserts that `parse_filename` returns `None` for a completely invalid filename. + - Asserts that `parse_filename` returns `None` for a filename that partially + matches the expected format but contains invalid components. + """ + assert db.parse_filename("garbage.nc") is None + assert db.parse_filename("JRR-AOD_v3r2_n21_invalid.nc") is None + + +def test_ingest_files(db): + """ + Test the ingestion of files into the database. + + This test checks if the `ingest_files` method of the `db` object correctly ingests files into the database. + It connects to the database, queries the `obs_files` table to count the number of ingested files, and asserts + that the count is 3, indicating that 3 valid JRR-AOD files should be ingested. + """ + db.ingest_files() + conn = sqlite3.connect(db.db_name) + cursor = conn.cursor() + cursor.execute("SELECT COUNT(*) FROM obs_files") + count = cursor.fetchone()[0] + conn.close() + assert count == 3, "Should ingest 3 valid JRR-AOD files" + + +def test_get_valid_files(db): + """ + Test the `get_valid_files` method of the database. + + This test ingests files into the database and then retrieves the valid files + for a given data assimilation (DA) cycle time and cutoff delta. It checks that + only the files within the specified cutoff delta are returned as valid. + + Steps: + 1. Ingest files into the database. + 2. Define a DA cycle time (`da_cycle`) and a cutoff delta in hours (`cutoff_delta`). + 3. Retrieve the valid files using the `get_valid_files` method. + 4. Assert that files at 10:00 and 12:00 are within +/- 3 hours of 12:00. + 5. Assert that files at 15:00 are not within the valid range. + 6. Assert that the total number of valid files is 2. + + Args: + db: The database object to be tested. + """ + db.ingest_files() + da_cycle = "20250316120000" + cutoff_delta = 3 # hours + + valid_files = db.get_valid_files(da_cycle, cutoff_delta=cutoff_delta) + + # Only files at 10:00 and 12:00 should be within +/- 3 hours of 12:00 + assert any("202503161000" in f for f in valid_files) + assert any("202503161200" in f for f in valid_files) + assert all("202503161500" not in f for f in valid_files) + assert len(valid_files) == 2 From ace1bf9ec5fa0d8288b44a8eace01c67f4cb1122 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 15:28:46 -0400 Subject: [PATCH 04/15] skip build/ctest --- .github/workflows/build.yaml | 2 +- .github/workflows/pytest.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b17b0d32..cc6ceb5b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,4 +1,4 @@ -on: [pull_request] +#on: [pull_request] jobs: build: diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 4590c21c..a0849644 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -3,7 +3,7 @@ name: Run Tests on Pull Request on: pull_request: branches: - - develop # Change this if you want to run on other branches + - main jobs: test: @@ -20,12 +20,12 @@ jobs: - name: Install Dependencies run: | - python -m venv venv - source venv/bin/activate + python -m venv obsdb + source obsdb/bin/activate pip install --upgrade pip pip install -r requirements.txt - name: Run Pytest run: | - source venv/bin/activate - pytest --maxfail=1 --disable-warnings -v \ No newline at end of file + source obsdb/bin/activate + pytest --maxfail=1 --disable-warnings -v From 05eb8e08187db632491fd6221fd9b4017265b851 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 15:35:14 -0400 Subject: [PATCH 05/15] mved py env reqs --- .github/workflows/requirements.txt => requirements.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/requirements.txt => requirements.txt (100%) diff --git a/.github/workflows/requirements.txt b/requirements.txt similarity index 100% rename from .github/workflows/requirements.txt rename to requirements.txt From dc39028e68ddeb838465f61d335559bbf0e383ba Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 15:38:23 -0400 Subject: [PATCH 06/15] recursive clone --- .github/workflows/pytest.yaml | 2 ++ ush/python/pygfs/obsdb/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index a0849644..a3412765 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -12,6 +12,8 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v4 # Clones the repository + with: + submodules: recursive - name: Set Up Python uses: actions/setup-python@v4 diff --git a/ush/python/pygfs/obsdb/__init__.py b/ush/python/pygfs/obsdb/__init__.py index 96881197..aa733d03 100644 --- a/ush/python/pygfs/obsdb/__init__.py +++ b/ush/python/pygfs/obsdb/__init__.py @@ -1 +1 @@ -from .obsdb import BaseDatabase \ No newline at end of file +from .obsdb import BaseDatabase From 9441b53ba0911187c710eab6b2aeb882dae017a5 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 16:11:02 -0400 Subject: [PATCH 07/15] select subset of pytests --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index a3412765..7a82bff5 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -30,4 +30,4 @@ jobs: - name: Run Pytest run: | source obsdb/bin/activate - pytest --maxfail=1 --disable-warnings -v + pytest ush/python/pygfs/tests/ --disable-warnings -v From 16e817ef8a3ca0b1e42ff752045b1e9d74de6d91 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 17:13:30 -0400 Subject: [PATCH 08/15] reorg, pynorm --- .flake8 | 4 +++ .github/workflows/build.yaml | 2 +- .github/workflows/pytest.yaml | 9 +++++- ush/python/pygfs/obsdb/__init__.py | 1 - .../tests => pyobsforge/obsdb}/__init__.py | 0 .../{pygfs => pyobsforge}/obsdb/ghrsst_db.py | 2 +- .../{pygfs => pyobsforge}/obsdb/jrr_aod_db.py | 31 ++++++++++--------- .../{pygfs => pyobsforge}/obsdb/obsdb.py | 1 + ush/python/pyobsforge/tests/__init__.py | 0 .../tests/test_ghrsst_database.py | 2 +- .../tests/test_jrr_aod_database.py | 9 ++---- 11 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 .flake8 delete mode 100644 ush/python/pygfs/obsdb/__init__.py rename ush/python/{pygfs/tests => pyobsforge/obsdb}/__init__.py (100%) rename ush/python/{pygfs => pyobsforge}/obsdb/ghrsst_db.py (98%) rename ush/python/{pygfs => pyobsforge}/obsdb/jrr_aod_db.py (77%) rename ush/python/{pygfs => pyobsforge}/obsdb/obsdb.py (99%) create mode 100644 ush/python/pyobsforge/tests/__init__.py rename ush/python/{pygfs => pyobsforge}/tests/test_ghrsst_database.py (98%) rename ush/python/{pygfs => pyobsforge}/tests/test_jrr_aod_database.py (96%) diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..e74ea330 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +max-line-length = 135 +exclude = .git,__pycache__ +ignore = E203, W503 diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cc6ceb5b..b17b0d32 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,4 +1,4 @@ -#on: [pull_request] +on: [pull_request] jobs: build: diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 7a82bff5..7d9c7834 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -27,7 +27,14 @@ jobs: pip install --upgrade pip pip install -r requirements.txt + - name: Run Code Style Check + run: | + source obsdb/bin/activate + pip install flake8 + flake8 ush/python/pyforge/obsdb/ + flake8 ush/python/pyforge/tests/ + - name: Run Pytest run: | source obsdb/bin/activate - pytest ush/python/pygfs/tests/ --disable-warnings -v + pytest ush/python/pyforge/tests/ --disable-warnings -v diff --git a/ush/python/pygfs/obsdb/__init__.py b/ush/python/pygfs/obsdb/__init__.py deleted file mode 100644 index aa733d03..00000000 --- a/ush/python/pygfs/obsdb/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .obsdb import BaseDatabase diff --git a/ush/python/pygfs/tests/__init__.py b/ush/python/pyobsforge/obsdb/__init__.py similarity index 100% rename from ush/python/pygfs/tests/__init__.py rename to ush/python/pyobsforge/obsdb/__init__.py diff --git a/ush/python/pygfs/obsdb/ghrsst_db.py b/ush/python/pyobsforge/obsdb/ghrsst_db.py similarity index 98% rename from ush/python/pygfs/obsdb/ghrsst_db.py rename to ush/python/pyobsforge/obsdb/ghrsst_db.py index 966f556b..159a3544 100644 --- a/ush/python/pygfs/obsdb/ghrsst_db.py +++ b/ush/python/pyobsforge/obsdb/ghrsst_db.py @@ -1,7 +1,7 @@ import os import glob from datetime import datetime -from obsdb import BaseDatabase +from pyobsforge.obsdb import BaseDatabase class GhrSstDatabase(BaseDatabase): diff --git a/ush/python/pygfs/obsdb/jrr_aod_db.py b/ush/python/pyobsforge/obsdb/jrr_aod_db.py similarity index 77% rename from ush/python/pygfs/obsdb/jrr_aod_db.py rename to ush/python/pyobsforge/obsdb/jrr_aod_db.py index bcfd66e8..46e1a4f2 100644 --- a/ush/python/pygfs/obsdb/jrr_aod_db.py +++ b/ush/python/pyobsforge/obsdb/jrr_aod_db.py @@ -1,8 +1,8 @@ import os import glob from datetime import datetime -from obsdb import BaseDatabase -import re +from pyobsforge.obsdb import BaseDatabase + class JrrAodDatabase(BaseDatabase): """Class to manage an observation file database for JRR-AOD data.""" @@ -53,20 +53,21 @@ def ingest_files(self): """ self.insert_record(query, parsed_data) + if __name__ == "__main__": - db = JrrAodDatabase() + db = JrrAodDatabase() - # Check for new files - db.ingest_files() + # Check for new files + db.ingest_files() - # Query files for a given DA cycle - da_cycle = "20250316120000" - cutoff_delta = 5 #datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) - valid_files = db.get_valid_files(da_cycle, cutoff_delta=cutoff_delta) + # Query files for a given DA cycle + da_cycle = "20250316120000" + cutoff_delta = 5 + valid_files = db.get_valid_files(da_cycle, cutoff_delta=cutoff_delta) - print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") - for valid_file in valid_files: - if os.path.exists(valid_file): - print(f"Valid file: {valid_file}") - else: - print(f"File does not exist: {valid_file}") + print(f"Found {len(valid_files)} valid files for DA cycle {da_cycle}") + for valid_file in valid_files: + if os.path.exists(valid_file): + print(f"Valid file: {valid_file}") + else: + print(f"File does not exist: {valid_file}") diff --git a/ush/python/pygfs/obsdb/obsdb.py b/ush/python/pyobsforge/obsdb/obsdb.py similarity index 99% rename from ush/python/pygfs/obsdb/obsdb.py rename to ush/python/pyobsforge/obsdb/obsdb.py index 1eb830a6..ec682d9d 100644 --- a/ush/python/pygfs/obsdb/obsdb.py +++ b/ush/python/pyobsforge/obsdb/obsdb.py @@ -4,6 +4,7 @@ import os # TODO(G): Hack to import wxflow module, do this properly. sys.path.append(os.path.join(os.path.dirname(__file__), '../../../../sorc/wxflow/src/')) +# flake8: noqa from wxflow.sqlitedb import SQLiteDB diff --git a/ush/python/pyobsforge/tests/__init__.py b/ush/python/pyobsforge/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ush/python/pygfs/tests/test_ghrsst_database.py b/ush/python/pyobsforge/tests/test_ghrsst_database.py similarity index 98% rename from ush/python/pygfs/tests/test_ghrsst_database.py rename to ush/python/pyobsforge/tests/test_ghrsst_database.py index debaf9ce..02332466 100644 --- a/ush/python/pygfs/tests/test_ghrsst_database.py +++ b/ush/python/pyobsforge/tests/test_ghrsst_database.py @@ -6,7 +6,7 @@ import pytest -from obsdb.ghrsst_db import GhrSstDatabase # Adjust as needed +from pyobsforge.obsdb.ghrsst_db import GhrSstDatabase # Adjust as needed @pytest.fixture diff --git a/ush/python/pygfs/tests/test_jrr_aod_database.py b/ush/python/pyobsforge/tests/test_jrr_aod_database.py similarity index 96% rename from ush/python/pygfs/tests/test_jrr_aod_database.py rename to ush/python/pyobsforge/tests/test_jrr_aod_database.py index f435fad9..91a3329a 100644 --- a/ush/python/pygfs/tests/test_jrr_aod_database.py +++ b/ush/python/pyobsforge/tests/test_jrr_aod_database.py @@ -2,11 +2,11 @@ import tempfile import shutil import sqlite3 -from datetime import datetime, timedelta +from datetime import datetime import pytest -from obsdb.jrr_aod_db import JrrAodDatabase +from pyobsforge.obsdb.jrr_aod_db import JrrAodDatabase @pytest.fixture @@ -43,10 +43,6 @@ def db(temp_obs_dir): dcom_dir=temp_obs_dir, obs_dir="jrr_aod" ) - #database.create_database() - #database.ingest_files() - #database.connect() - #print('db:', database) return database @@ -104,7 +100,6 @@ def test_parse_valid_filename(db): conn = db.get_connection() cursor = conn.cursor() cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") - tables = cursor.fetchall() conn.close() parsed = db.parse_filename(fname) From c71dca1293543da09a0ebd9fb0e7e580587164bf Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 17:17:23 -0400 Subject: [PATCH 09/15] ... --- .github/workflows/pytest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 7d9c7834..6470a1bd 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -31,8 +31,8 @@ jobs: run: | source obsdb/bin/activate pip install flake8 - flake8 ush/python/pyforge/obsdb/ - flake8 ush/python/pyforge/tests/ + flake8 ush/python/pyobsforge/obsdb/ + flake8 ush/python/pyobsforge/tests/ - name: Run Pytest run: | From 522766aba7f544d7477ea3f4296c565f3bab27a1 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 17:19:38 -0400 Subject: [PATCH 10/15] #%$^&! --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 6470a1bd..7fe80a07 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -37,4 +37,4 @@ jobs: - name: Run Pytest run: | source obsdb/bin/activate - pytest ush/python/pyforge/tests/ --disable-warnings -v + pytest ush/python/pyobsforge/tests/ --disable-warnings -v From fed2bd08377848da168397801ded864c45386fe4 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 17:31:51 -0400 Subject: [PATCH 11/15] ... --- .github/workflows/build.yaml | 2 +- ush/python/pyobsforge/obsdb/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b17b0d32..cc6ceb5b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,4 +1,4 @@ -on: [pull_request] +#on: [pull_request] jobs: build: diff --git a/ush/python/pyobsforge/obsdb/__init__.py b/ush/python/pyobsforge/obsdb/__init__.py index e69de29b..aa733d03 100644 --- a/ush/python/pyobsforge/obsdb/__init__.py +++ b/ush/python/pyobsforge/obsdb/__init__.py @@ -0,0 +1 @@ +from .obsdb import BaseDatabase From f6544728907a652a23c6903e35ccf188f389408d Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 17:34:34 -0400 Subject: [PATCH 12/15] pynorm --- ush/python/pyobsforge/obsdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pyobsforge/obsdb/__init__.py b/ush/python/pyobsforge/obsdb/__init__.py index aa733d03..5f2a238e 100644 --- a/ush/python/pyobsforge/obsdb/__init__.py +++ b/ush/python/pyobsforge/obsdb/__init__.py @@ -1 +1 @@ -from .obsdb import BaseDatabase +from .obsdb import BaseDatabase # noqa From 5e8d06c49f81de16389247b3ae7ae520c74f65b9 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 24 Mar 2025 17:36:33 -0400 Subject: [PATCH 13/15] add build and ctest back --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cc6ceb5b..b17b0d32 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,4 +1,4 @@ -#on: [pull_request] +on: [pull_request] jobs: build: From 5358d230bbc150348d6ca34e1365602075e97ae0 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 25 Mar 2025 11:34:23 -0400 Subject: [PATCH 14/15] removed rel path and user path --- ush/python/pyobsforge/obsdb/ghrsst_db.py | 2 +- ush/python/pyobsforge/obsdb/jrr_aod_db.py | 4 ++-- ush/python/pyobsforge/obsdb/obsdb.py | 5 +---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ush/python/pyobsforge/obsdb/ghrsst_db.py b/ush/python/pyobsforge/obsdb/ghrsst_db.py index 159a3544..aa0b3def 100644 --- a/ush/python/pyobsforge/obsdb/ghrsst_db.py +++ b/ush/python/pyobsforge/obsdb/ghrsst_db.py @@ -8,7 +8,7 @@ class GhrSstDatabase(BaseDatabase): """Class to manage an observation file database for data assimilation.""" def __init__(self, db_name="obs_files.db", - dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/", + dcom_dir="/lfs/h1/ops/prod/dcom/", obs_dir="sst"): base_dir = os.path.join(dcom_dir, '*', obs_dir) super().__init__(db_name, base_dir) diff --git a/ush/python/pyobsforge/obsdb/jrr_aod_db.py b/ush/python/pyobsforge/obsdb/jrr_aod_db.py index 46e1a4f2..1ee4ee14 100644 --- a/ush/python/pyobsforge/obsdb/jrr_aod_db.py +++ b/ush/python/pyobsforge/obsdb/jrr_aod_db.py @@ -8,7 +8,7 @@ class JrrAodDatabase(BaseDatabase): """Class to manage an observation file database for JRR-AOD data.""" def __init__(self, db_name="jrr_aod_obs.db", - dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/", + dcom_dir="/lfs/h1/ops/prod/dcom/", obs_dir="jrr_aod"): base_dir = os.path.join(dcom_dir, '*', obs_dir) super().__init__(db_name, base_dir) @@ -55,7 +55,7 @@ def ingest_files(self): if __name__ == "__main__": - db = JrrAodDatabase() + db = JrrAodDatabase(dcom_dir="/home/gvernier/Volumes/hera-s1/runs/realtimeobs/lfs/h1/ops/prod/dcom/") # Check for new files db.ingest_files() diff --git a/ush/python/pyobsforge/obsdb/obsdb.py b/ush/python/pyobsforge/obsdb/obsdb.py index ec682d9d..60682a4f 100644 --- a/ush/python/pyobsforge/obsdb/obsdb.py +++ b/ush/python/pyobsforge/obsdb/obsdb.py @@ -2,9 +2,6 @@ from datetime import datetime, timedelta import sys import os -# TODO(G): Hack to import wxflow module, do this properly. -sys.path.append(os.path.join(os.path.dirname(__file__), '../../../../sorc/wxflow/src/')) -# flake8: noqa from wxflow.sqlitedb import SQLiteDB @@ -15,7 +12,7 @@ def __init__(self, db_name: str, base_dir: str) -> None: """ Initialize the database. - :param db_name: Path to the SQLite database file. + :param db_name: Name of the SQLite database. :param base_dir: Directory containing observation files. """ super().__init__(db_name) From 0045e3813744a744499004045b89f797a33d7d92 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 25 Mar 2025 11:36:06 -0400 Subject: [PATCH 15/15] ... --- ush/python/pyobsforge/obsdb/obsdb.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/python/pyobsforge/obsdb/obsdb.py b/ush/python/pyobsforge/obsdb/obsdb.py index 60682a4f..3e0f290d 100644 --- a/ush/python/pyobsforge/obsdb/obsdb.py +++ b/ush/python/pyobsforge/obsdb/obsdb.py @@ -1,7 +1,5 @@ import sqlite3 from datetime import datetime, timedelta -import sys -import os from wxflow.sqlitedb import SQLiteDB