From 00e1e191021bf2d3a9f4657229fbf794ce6e9a96 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 2 Apr 2025 15:19:36 -0400 Subject: [PATCH 1/8] wip --- parm/config.yaml | 4 ++ parm/nc2ioda/nc2ioda.yaml.j2 | 10 ++++ scripts/exobsforge_global_aod_dump.py | 8 +-- .../test_exobsforge_global_marine_dump.py | 53 +++++++++++-------- ush/python/pyobsforge/obsdb/obsdb.py | 20 ++++++- ush/python/pyobsforge/task/aero_prepobs.py | 31 ++++++++++- ush/python/pyobsforge/task/marine_prepobs.py | 46 +++------------- ush/python/pyobsforge/task/run_nc2ioda.py | 40 ++++++++++++++ 8 files changed, 144 insertions(+), 68 deletions(-) create mode 100644 ush/python/pyobsforge/task/run_nc2ioda.py diff --git a/parm/config.yaml b/parm/config.yaml index 12b47720..6a21d036 100644 --- a/parm/config.yaml +++ b/parm/config.yaml @@ -14,7 +14,11 @@ obsforge: assim_freq: 6 aoddump: + provider: VIIRSAOD platforms: ['npp', 'n20', 'n21'] + thinning_threshold: 0 + channel: 4 + preqc: 2 WALLTIME_AOD_DUMP: '00:30:00' TASK_GEOM_AOD_DUMP: '1:ppn=1:tpp=1' MEMORY_AOD_DUMP: 96GB diff --git a/parm/nc2ioda/nc2ioda.yaml.j2 b/parm/nc2ioda/nc2ioda.yaml.j2 index e59cdb99..03081c97 100644 --- a/parm/nc2ioda/nc2ioda.yaml.j2 +++ b/parm/nc2ioda/nc2ioda.yaml.j2 @@ -14,3 +14,13 @@ output file: {{ output_file }} ocean basin: {{ ocean_basin }} {% endif %} input files: {{ input_files }} + +# Used in JRR_AOD +{% if provider == "VIIRSAOD"%} +variable: aerosolOpticalDepth +thinning: + threshold: {{ thinning_threshold }} +channel: 4 +preqc: 2 +{% endif %} + diff --git a/scripts/exobsforge_global_aod_dump.py b/scripts/exobsforge_global_aod_dump.py index 86290cec..b562e2ab 100755 --- a/scripts/exobsforge_global_aod_dump.py +++ b/scripts/exobsforge_global_aod_dump.py @@ -27,7 +27,7 @@ config = AttrDict(**config_env, **obsforge_dict) config = AttrDict(**config, **config_yaml['aoddump']) - AeroObs = AerosolObsPrep(config) - AeroObs.initialize() - AeroObs.execute() - AeroObs.finalize() + aeroObs = AerosolObsPrep(config) + aeroObs.initialize() + aeroObs.execute() + aeroObs.finalize() diff --git a/scripts/tests/test_exobsforge_global_marine_dump.py b/scripts/tests/test_exobsforge_global_marine_dump.py index 0997bda3..1efc3f7b 100644 --- a/scripts/tests/test_exobsforge_global_marine_dump.py +++ b/scripts/tests/test_exobsforge_global_marine_dump.py @@ -9,8 +9,8 @@ def script_env(tmp_path): # Set environment vars expected by the script env = os.environ.copy() env["cyc"] = "0" - env["current_cycle"] = "20250301500" - env["PDY"] = "20250315" + env["current_cycle"] = "20250301600" + env["PDY"] = "20250316" env["RUN"] = "gdas" return env @@ -60,24 +60,31 @@ def test_run_exobsforge_script(script_env): create_dcom(output_root=os.getenv("DCOMROOT"), dcom_tree_file=Path(__file__).parent / "dcom_tree.txt") - # Run the script using subprocess - exec = Path(__file__).parent.parent / "exobsforge_global_marine_dump.py" - cwd = Path(__file__).parent / "tests_output/RUNDIRS/obsforge" - result = subprocess.run( - ["python3", exec], - cwd=cwd, - env=env, - capture_output=True, - text=True - ) - - # Print the standard output - print("Standard Output:") - print(result.stdout) - - # Optionally, print the standard error - print("Standard Error:") - print(result.stderr) - - # Basic assertions - assert result.returncode == 0 + # List of scripts to run + scripts = [ + "exobsforge_global_marine_dump.py", + "exobsforge_global_aod_dump.py" + ] + + for script_name in scripts: + # Run each script using subprocess + exec = Path(__file__).parent.parent / script_name + cwd = Path(__file__).parent / "tests_output/RUNDIRS/obsforge" + result = subprocess.run( + ["python3", exec], + cwd=cwd, + env=env, + capture_output=True, + text=True + ) + + # Print the standard output + print(f"Standard Output for {script_name}:") + print(result.stdout) + + # Optionally, print the standard error + print(f"Standard Error for {script_name}:") + print(result.stderr) + + # Basic assertions + assert result.returncode == 0 diff --git a/ush/python/pyobsforge/obsdb/obsdb.py b/ush/python/pyobsforge/obsdb/obsdb.py index b22fe2d7..ab66db8c 100644 --- a/ush/python/pyobsforge/obsdb/obsdb.py +++ b/ush/python/pyobsforge/obsdb/obsdb.py @@ -1,6 +1,11 @@ +from logging import getLogger import sqlite3 from datetime import datetime, timedelta from wxflow.sqlitedb import SQLiteDB +from wxflow import FileHandler +from os.path import basename, join + +logger = getLogger(__name__.split('.')[-1]) class BaseDatabase(SQLiteDB): @@ -57,6 +62,7 @@ def execute_query(self, query: str, params: tuple = None) -> list: def get_valid_files(self, window_begin: datetime, window_end: datetime, + dst_dir: str, instrument: str = None, satellite: str = None, obs_type: str = None, @@ -106,4 +112,16 @@ def get_valid_files(self, valid_files.append(filename) - return valid_files + # Copy files to the destination directory + dst_files = [] + if len(valid_files) > 0: + src_dst_obs_list = [] # list of [src_file, dst_file] + for src_file in valid_files: + dst_file = join(dst_dir, f"{basename(src_file)}") + dst_files.append(dst_file) + logger.info(f"copying {src_file} to {dst_file}") + src_dst_obs_list.append([src_file, dst_file]) + FileHandler({'mkdir': [dst_dir]}).sync() + FileHandler({'copy': src_dst_obs_list}).sync() + + return dst_files \ No newline at end of file diff --git a/ush/python/pyobsforge/task/aero_prepobs.py b/ush/python/pyobsforge/task/aero_prepobs.py index 57ba4732..df7bb11e 100644 --- a/ush/python/pyobsforge/task/aero_prepobs.py +++ b/ush/python/pyobsforge/task/aero_prepobs.py @@ -5,6 +5,8 @@ from wxflow import (AttrDict, Task, add_to_datetime, to_timedelta, logit) +from pyobsforge.obsdb.jrr_aod_db import JrrAodDatabase +from pyobsforge.task.run_nc2ioda import run_nc2ioda logger = getLogger(__name__.split('.')[-1]) @@ -31,17 +33,42 @@ def __init__(self, config: Dict[str, Any]) -> None: # task_config is everything that this task should need self.task_config = AttrDict(**self.task_config, **local_dict) + # Initialize the JRR_AOD database + self.jrr_aod_db = JrrAodDatabase(db_name="jrr_aod_obs.db", + dcom_dir=self.task_config.DCOMROOT, + obs_dir="jrr_aod") + @logit(logger) def initialize(self) -> None: """ """ - print("initialize") + # Update the database with new files + self.jrr_aod_db.ingest_files() @logit(logger) def execute(self) -> None: """ """ - print("execute") + for platform in self.task_config.platforms: + print(f"========= platform: {platform}") + input_files = self.jrr_aod_db.get_valid_files(window_begin=self.task_config.window_begin, + window_end=self.task_config.window_end, + dst_dir='jrr_aod', + satellite=platform) + logger.info(f"number of valid files: {len(input_files)}") + + if len(input_files) > 0: + print(f"number of valid files: {len(input_files)}") + obs_space = 'jrr_aod' + output_file = f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.{obs_space}.tm00.nc" + context = {'provider': 'VIIRSAOD', + 'window_begin': self.task_config.window_begin, + 'window_end': self.task_config.window_end, + 'thinning_threshold': 0, + 'input_files': input_files, + 'output_file': output_file} + result = run_nc2ioda(self.task_config, obs_space, context) + logger.info(f"run_nc2ioda result: {result}") @logit(logger) def finalize(self) -> None: diff --git a/ush/python/pyobsforge/task/marine_prepobs.py b/ush/python/pyobsforge/task/marine_prepobs.py index 406ed06d..0e2674b5 100644 --- a/ush/python/pyobsforge/task/marine_prepobs.py +++ b/ush/python/pyobsforge/task/marine_prepobs.py @@ -2,12 +2,11 @@ from logging import getLogger from typing import Dict, Any -import subprocess from wxflow import (AttrDict, Task, add_to_datetime, to_timedelta, - logit, FileHandler, Jinja, save_as_yaml) + logit) from pyobsforge.obsdb.ghrsst_db import GhrSstDatabase -from os.path import basename, join from multiprocessing import Process +from pyobsforge.task.run_nc2ioda import run_nc2ioda logger = getLogger(__name__.split('.')[-1]) @@ -125,26 +124,16 @@ def process_ghrsst(self, binning_min_number_of_obs (int): Minimum number of observations required for binning. """ # Query the database for valid files - valid_files = self.ghrsst_db.get_valid_files(window_begin=self.task_config.window_begin, + input_files = self.ghrsst_db.get_valid_files(window_begin=self.task_config.window_begin, window_end=self.task_config.window_end, + dst_dir=obs_space, instrument=instrument, satellite=platform, obs_type="SSTsubskin") - logger.info(f"number of valid files: {len(valid_files)}") + logger.info(f"number of valid files: {len(input_files)}") # Process the observations if the obs space is not empty - if len(valid_files) > 0: - # Copy the valid files to the RUNDIR - src_dst_obs_list = [] # list of [src_file, dst_file] - input_files = [] # list of dst_files used as input to the ioda converter - for src_file in valid_files: - dst_file = f"{obs_space}/{basename(src_file)}" - input_files.append(dst_file) - logger.info(f"copying {src_file} to {dst_file}") - src_dst_obs_list.append([src_file, dst_file]) - FileHandler({'mkdir': [obs_space]}).sync() - FileHandler({'copy': src_dst_obs_list}).sync() - + if len(input_files) > 0: # Configure the ioda converter output_file = f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.{obs_space}.tm00.nc" context = {'provider': provider.upper(), @@ -156,27 +145,8 @@ def process_ghrsst(self, 'binning_min_number_of_obs': binning_min_number_of_obs, 'input_files': input_files, 'output_file': output_file} - jinja_template = join(self.task_config['HOMEobsforge'], "parm", "nc2ioda", "nc2ioda.yaml.j2") - yaml_config = Jinja(jinja_template, context).render - nc2ioda_yaml = join(self.task_config['DATA'], obs_space, f"{obs_space}_nc2ioda.yaml") - save_as_yaml(yaml_config, nc2ioda_yaml) - - # Run the ioda converter - nc2ioda_exe = join(self.task_config['HOMEobsforge'], 'build', 'bin', 'obsforge_obsprovider2ioda.x') - try: - result = subprocess.run([nc2ioda_exe, nc2ioda_yaml], - cwd=self.task_config['DATA'], - capture_output=True, - text=True) - logger.info(f"Standard Output: \n{result.stdout}") - # TODO (G): Figure out what to do with failure. - # Ignore failures for now and just issue a warning - if result.returncode != 0: - logger.error(f"Standard Error: \n{result.stderr}") - return 0 - except subprocess.CalledProcessError as e: - logger.warning(f"ioda converter failed with error {e}, \ - return code {e.returncode}") + result = run_nc2ioda(self.task_config, obs_space, context) + logger.info(f"run_nc2ioda result: {result}") @logit(logger) def finalize(self) -> None: diff --git a/ush/python/pyobsforge/task/run_nc2ioda.py b/ush/python/pyobsforge/task/run_nc2ioda.py new file mode 100644 index 00000000..a82f82e9 --- /dev/null +++ b/ush/python/pyobsforge/task/run_nc2ioda.py @@ -0,0 +1,40 @@ +from logging import getLogger +import subprocess +from wxflow import Jinja, save_as_yaml +from os.path import join + +logger = getLogger(__name__.split('.')[-1]) + +def run_nc2ioda(task_config: dict, obs_space: str, context: dict) -> int: + """ + Executes the nc2ioda conversion process using a Jinja2 template and a YAML configuration. + + Args: + task_config (dict): Configuration dictionary containing paths and settings for the task. + obs_space (str): Observation space identifier used to generate file paths. + context (dict): Context dictionary with variables to render the Jinja2 template. + + Returns: + int: Returns 0 upon successful execution. Logs errors and warnings for failures. + """ + jinja_template = join(task_config['HOMEobsforge'], "parm", "nc2ioda", "nc2ioda.yaml.j2") + yaml_config = Jinja(jinja_template, context).render + nc2ioda_yaml = join(task_config['DATA'], obs_space, f"{obs_space}_nc2ioda.yaml") + save_as_yaml(yaml_config, nc2ioda_yaml) + + # Run the ioda converter + nc2ioda_exe = join(task_config['HOMEobsforge'], 'build', 'bin', 'obsforge_obsprovider2ioda.x') + try: + result = subprocess.run([nc2ioda_exe, nc2ioda_yaml], + cwd=task_config['DATA'], + capture_output=True, + text=True) + logger.info(f"Standard Output: \n{result.stdout}") + # TODO (G): Figure out what to do with failures. + # Ignore failures for now and just issue a warning + if result.returncode != 0: + logger.error(f"Standard Error: \n{result.stderr}") + return 0 + except subprocess.CalledProcessError as e: + logger.warning(f"ioda converter failed with error {e}, \ + return code {e.returncode}") From f7f0da2dae9e56b4c3606d7d2c687cc7fa94bfa7 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 2 Apr 2025 15:38:58 -0400 Subject: [PATCH 2/8] wip --- parm/nc2ioda/nc2ioda.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/nc2ioda/nc2ioda.yaml.j2 b/parm/nc2ioda/nc2ioda.yaml.j2 index 03081c97..be3cdf27 100644 --- a/parm/nc2ioda/nc2ioda.yaml.j2 +++ b/parm/nc2ioda/nc2ioda.yaml.j2 @@ -6,9 +6,11 @@ binning: stride: {{ binning_stride }} min number of obs: {{ binning_min_number_of_obs }} {% endif %} +{% if bounds_min is defined and bounds_max is defined %} bounds: min: {{ bounds_min }} max: {{ bounds_max }} +{% endif %} output file: {{ output_file }} {% if ocean_basin is defined %} ocean basin: {{ ocean_basin }} From b2eff89fdfd132d67cc7d1a8807854aa9f7b7325 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 2 Apr 2025 16:47:32 -0400 Subject: [PATCH 3/8] ... --- ush/python/pyobsforge/task/run_nc2ioda.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pyobsforge/task/run_nc2ioda.py b/ush/python/pyobsforge/task/run_nc2ioda.py index a82f82e9..67ba82eb 100644 --- a/ush/python/pyobsforge/task/run_nc2ioda.py +++ b/ush/python/pyobsforge/task/run_nc2ioda.py @@ -1,6 +1,6 @@ from logging import getLogger import subprocess -from wxflow import Jinja, save_as_yaml +from wxflow import save_as_yaml, parse_j2yaml from os.path import join logger = getLogger(__name__.split('.')[-1]) @@ -18,7 +18,7 @@ def run_nc2ioda(task_config: dict, obs_space: str, context: dict) -> int: int: Returns 0 upon successful execution. Logs errors and warnings for failures. """ jinja_template = join(task_config['HOMEobsforge'], "parm", "nc2ioda", "nc2ioda.yaml.j2") - yaml_config = Jinja(jinja_template, context).render + yaml_config = parse_j2yaml(jinja_template, context) nc2ioda_yaml = join(task_config['DATA'], obs_space, f"{obs_space}_nc2ioda.yaml") save_as_yaml(yaml_config, nc2ioda_yaml) From de2b0a7c98b8ff745c47a4f2adf14d66df94c671 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 2 Apr 2025 17:05:54 -0400 Subject: [PATCH 4/8] added aod --- ..._marine_dump.py => test_exobsforge_global_dump.py} | 0 ush/python/pyobsforge/obsdb/obsdb.py | 11 +++++++---- ush/python/pyobsforge/task/aero_prepobs.py | 4 ++-- ush/python/pyobsforge/task/marine_prepobs.py | 3 +-- ush/python/pyobsforge/task/run_nc2ioda.py | 1 + 5 files changed, 11 insertions(+), 8 deletions(-) rename scripts/tests/{test_exobsforge_global_marine_dump.py => test_exobsforge_global_dump.py} (100%) diff --git a/scripts/tests/test_exobsforge_global_marine_dump.py b/scripts/tests/test_exobsforge_global_dump.py similarity index 100% rename from scripts/tests/test_exobsforge_global_marine_dump.py rename to scripts/tests/test_exobsforge_global_dump.py diff --git a/ush/python/pyobsforge/obsdb/obsdb.py b/ush/python/pyobsforge/obsdb/obsdb.py index ab66db8c..6b744750 100644 --- a/ush/python/pyobsforge/obsdb/obsdb.py +++ b/ush/python/pyobsforge/obsdb/obsdb.py @@ -68,16 +68,19 @@ def get_valid_files(self, obs_type: str = None, check_receipt: str = "none") -> list: """ - Retrieve a list of observation files within a specified time window, possibly filtered by instrument, - satellite, and observation type. The check_receipt parameter can be 'gdas', 'gfs', or 'none'. + Retrieve and copy to dst_dir a list of observation files within a specified time window, possibly filtered by instrument, + satellite, and observation type. The check_receipt parameter can be 'gdas', 'gfs', or 'none'. If 'gdas' or + 'gfs' is specified, files are further filtered based on their receipt time to ensure they meet the + required delay criteria. :param window_begin: Start of the time window (datetime object). :param window_end: End of the time window (datetime object). + :param dst_dir: Destination directory where valid files will be copied. :param instrument: (Optional) Filter by instrument name. :param satellite: (Optional) Filter by satellite name. :param obs_type: (Optional) Filter by observation type. :param check_receipt: (Optional) Specify receipt time check ('gdas', 'gfs', or 'none'). - :return: List of valid observation file names. + :return: List of valid observation file paths in the destination directory. """ query = """ @@ -124,4 +127,4 @@ def get_valid_files(self, FileHandler({'mkdir': [dst_dir]}).sync() FileHandler({'copy': src_dst_obs_list}).sync() - return dst_files \ No newline at end of file + return dst_files diff --git a/ush/python/pyobsforge/task/aero_prepobs.py b/ush/python/pyobsforge/task/aero_prepobs.py index df7bb11e..9dfe2e3e 100644 --- a/ush/python/pyobsforge/task/aero_prepobs.py +++ b/ush/python/pyobsforge/task/aero_prepobs.py @@ -35,8 +35,8 @@ def __init__(self, config: Dict[str, Any]) -> None: # Initialize the JRR_AOD database self.jrr_aod_db = JrrAodDatabase(db_name="jrr_aod_obs.db", - dcom_dir=self.task_config.DCOMROOT, - obs_dir="jrr_aod") + dcom_dir=self.task_config.DCOMROOT, + obs_dir="jrr_aod") @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pyobsforge/task/marine_prepobs.py b/ush/python/pyobsforge/task/marine_prepobs.py index 0e2674b5..dd398cf7 100644 --- a/ush/python/pyobsforge/task/marine_prepobs.py +++ b/ush/python/pyobsforge/task/marine_prepobs.py @@ -2,8 +2,7 @@ from logging import getLogger from typing import Dict, Any -from wxflow import (AttrDict, Task, add_to_datetime, to_timedelta, - logit) +from wxflow import AttrDict, Task, add_to_datetime, to_timedelta, logit from pyobsforge.obsdb.ghrsst_db import GhrSstDatabase from multiprocessing import Process from pyobsforge.task.run_nc2ioda import run_nc2ioda diff --git a/ush/python/pyobsforge/task/run_nc2ioda.py b/ush/python/pyobsforge/task/run_nc2ioda.py index 67ba82eb..d4be9234 100644 --- a/ush/python/pyobsforge/task/run_nc2ioda.py +++ b/ush/python/pyobsforge/task/run_nc2ioda.py @@ -5,6 +5,7 @@ logger = getLogger(__name__.split('.')[-1]) + def run_nc2ioda(task_config: dict, obs_space: str, context: dict) -> int: """ Executes the nc2ioda conversion process using a Jinja2 template and a YAML configuration. From 9ded4f35f4b9e3e810fb40c4c68c5195e36d916c Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 2 Apr 2025 17:12:31 -0400 Subject: [PATCH 5/8] tidy ... sort of --- scripts/tests/test_exobsforge_global_dump.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/tests/test_exobsforge_global_dump.py b/scripts/tests/test_exobsforge_global_dump.py index 1efc3f7b..42776f50 100644 --- a/scripts/tests/test_exobsforge_global_dump.py +++ b/scripts/tests/test_exobsforge_global_dump.py @@ -78,11 +78,8 @@ def test_run_exobsforge_script(script_env): text=True ) - # Print the standard output print(f"Standard Output for {script_name}:") print(result.stdout) - - # Optionally, print the standard error print(f"Standard Error for {script_name}:") print(result.stderr) From 1f0286278de8d82c7504b0d66a9319e350d22c3e Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 2 Apr 2025 17:24:58 -0400 Subject: [PATCH 6/8] fixed pytests --- ush/python/pyobsforge/tests/test_ghrsst_database.py | 8 +++++++- ush/python/pyobsforge/tests/test_jrr_aod_database.py | 6 ++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ush/python/pyobsforge/tests/test_ghrsst_database.py b/ush/python/pyobsforge/tests/test_ghrsst_database.py index 65180877..e96b940c 100644 --- a/ush/python/pyobsforge/tests/test_ghrsst_database.py +++ b/ush/python/pyobsforge/tests/test_ghrsst_database.py @@ -100,10 +100,11 @@ def test_get_valid_files(db): da_cycle = "20250316120000" window_begin = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") - timedelta(hours=3) window_end = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=1) - + dst_dir = 'sst' # Test for AVHRRF_MB valid_files = db.get_valid_files(window_begin=window_begin, window_end=window_end, + dst_dir=dst_dir, instrument="AVHRRF", satellite="MB", obs_type="SSTsubskin") @@ -117,6 +118,7 @@ def test_get_valid_files(db): # Test for VIIRS_NPP valid_files = db.get_valid_files(window_begin=window_begin, window_end=window_end, + dst_dir=dst_dir, instrument="VIIRS", satellite="NPP", obs_type="SSTsubskin") @@ -131,6 +133,7 @@ def test_get_valid_files(db): window_end = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) valid_files = db.get_valid_files(window_begin=window_begin, window_end=window_end, + dst_dir=dst_dir, instrument="VIIRS", satellite="N20", obs_type="SSTsubskin") @@ -143,6 +146,7 @@ def test_get_valid_files(db): # # Test for VIIRS_N21 valid_files = db.get_valid_files(window_begin=window_begin, window_end=window_end, + dst_dir=dst_dir, instrument="VIIRS", satellite="N21", obs_type="SSTsubskin") @@ -158,10 +162,12 @@ def test_get_valid_files_receipt(db): da_cycle = "20250316120000" window_begin = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") - timedelta(hours=3) window_end = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=1) + dst_dir = 'sst' # Test for AVHRRF_MB valid_files = db.get_valid_files(window_begin=window_begin, window_end=window_end, + dst_dir=dst_dir, instrument="AVHRRF", satellite="MB", obs_type="SSTsubskin", diff --git a/ush/python/pyobsforge/tests/test_jrr_aod_database.py b/ush/python/pyobsforge/tests/test_jrr_aod_database.py index 14fc56a5..c67e829c 100644 --- a/ush/python/pyobsforge/tests/test_jrr_aod_database.py +++ b/ush/python/pyobsforge/tests/test_jrr_aod_database.py @@ -129,9 +129,10 @@ def test_get_valid_files(db): da_cycle = "20250316120000" window_begin = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") - timedelta(hours=3) window_end = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) - + dst_dir = 'jrr_aod' valid_files = db.get_valid_files(window_begin=window_begin, window_end=window_end, + dst_dir=dst_dir, satellite="n21") assert any("202503161000" in f for f in valid_files) @@ -148,9 +149,10 @@ def test_get_valid_files_receipt(db): da_cycle = "20250316120000" window_begin = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") - timedelta(hours=3) window_end = datetime.strptime(da_cycle, "%Y%m%d%H%M%S") + timedelta(hours=3) - + dst_dir = 'jrr_aod' valid_files = db.get_valid_files(window_begin=window_begin, window_end=window_end, + dst_dir=dst_dir, satellite="n21", check_receipt='gfs') From 82301e50203abe6538c613aaeed2ad298e113ea5 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 2 Apr 2025 19:10:16 -0400 Subject: [PATCH 7/8] remove scripts tests from pytest --- .github/workflows/pytest.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 46862f77..71fa2ba8 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -39,4 +39,3 @@ jobs: run: | source obsdb/bin/activate pytest ush/python/pyobsforge/tests/ --disable-warnings -v - pytest scripts/tests/ --disable-warnings -v From c508624a210689dda2e0503cd8806fc2d65e829b Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 3 Apr 2025 12:44:25 -0400 Subject: [PATCH 8/8] glob for l3 only --- ush/python/pyobsforge/obsdb/ghrsst_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pyobsforge/obsdb/ghrsst_db.py b/ush/python/pyobsforge/obsdb/ghrsst_db.py index 2065ce28..b30d938b 100644 --- a/ush/python/pyobsforge/obsdb/ghrsst_db.py +++ b/ush/python/pyobsforge/obsdb/ghrsst_db.py @@ -57,7 +57,7 @@ def parse_filename(self, filename): def ingest_files(self): """Scan the directory for new observation files and insert them into the database.""" - obs_files = glob.glob(os.path.join(self.base_dir, "*.nc")) + obs_files = glob.glob(os.path.join(self.base_dir, "*-OSPO-L3U_GHRSST-*.nc")) print(f"Found {len(obs_files)} new files to ingest") for file in obs_files: parsed_data = self.parse_filename(file)