From f7ea78c5ceb615b36f5e6e9af81aebfaad587ab0 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 23 May 2024 16:33:32 +0000 Subject: [PATCH 01/26] initial commit to add the prepaeroobs job --- env/HERA.env | 7 + jobs/JGLOBAL_PREP_AERO_OBS | 49 +++++ jobs/rocoto/prepaeroobs.sh | 24 +++ parm/config/gfs/config.aeroanl | 5 + parm/config/gfs/config.prepaeroobs | 15 ++ parm/config/gfs/config.resources | 10 +- scripts/exglobal_prep_aero_obs.py | 25 +++ ush/python/pygfs/task/aero_prepobs.py | 258 ++++++++++++++++++++++++++ workflow/applications/gfs_cycled.py | 4 +- workflow/rocoto/gfs_tasks.py | 26 ++- workflow/rocoto/tasks.py | 2 +- 11 files changed, 420 insertions(+), 5 deletions(-) create mode 100755 jobs/JGLOBAL_PREP_AERO_OBS create mode 100755 jobs/rocoto/prepaeroobs.sh create mode 100644 parm/config/gfs/config.prepaeroobs create mode 100755 scripts/exglobal_prep_aero_obs.py create mode 100644 ush/python/pygfs/task/aero_prepobs.py diff --git a/env/HERA.env b/env/HERA.env index edbe420496f..e8854c95587 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -90,6 +90,13 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" +elif [[ "${step}" = "prepaeroobs" ]]; then + + nth_max=$((npe_node_max / npe_node_prepaeroobs)) + + export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} + export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/jobs/JGLOBAL_PREP_AERO_OBS b/jobs/JGLOBAL_PREP_AERO_OBS new file mode 100755 index 00000000000..5c5a6c168c8 --- /dev/null +++ b/jobs/JGLOBAL_PREP_AERO_OBS @@ -0,0 +1,49 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "prepaeroobs" -c "base aeroanl prepaeroobs" + +############################################## +# Set variables used in the script +############################################## +# shellcheck disable=SC2153 +GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +gPDY=${GDATE:0:8} +gcyc=${GDATE:8:2} +GDUMP="gdas" + + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS + +RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ + COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + +mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASPREPAEROOBSPY:-${SCRgfs}/exglobal_prep_aero_obs.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/prepaeroobs.sh b/jobs/rocoto/prepaeroobs.sh new file mode 100755 index 00000000000..4552bf3c724 --- /dev/null +++ b/jobs/rocoto/prepaeroobs.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="prepaeroobs" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_PREP_AERO_OBS" +status=$? +exit "${status}" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 972f393feb5..faed5248e63 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -20,6 +20,11 @@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/fv3jedi_var.x" +export OBSPROCEXE="${EXECgfs}/gdas_obsprovider2ioda.x" +export VIIRS_DATA_DIR="/scratch2/NCEPDEV/stmp3/Yaping.Wang/VIIRS/AWS/" +export SENSORS="npp" + + if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2" diff --git a/parm/config/gfs/config.prepaeroobs b/parm/config/gfs/config.prepaeroobs new file mode 100644 index 00000000000..f99825e056b --- /dev/null +++ b/parm/config/gfs/config.prepaeroobs @@ -0,0 +1,15 @@ +#!/bin/bash -x + +########## config.prepaeroobs ########## +# Aerosol Variance specific + +echo "BEGIN: config.prepaeroobs" + +# Get task specific resources +source "${EXPDIR}/config.resources" prepaeroobs + +export OBSPROCYAML="${PARMgfs}/gdas/aero/obs/list/aero_obsproc.yaml.j2" + + + +echo "END: config.prepaeroobs" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 9e229de11af..a5d2c1c536f 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -13,7 +13,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlrun atmensanlfinal" echo "snowanl" - echo "aeroanlinit aeroanlrun aeroanlfinal" + echo "prepaeroobs aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -274,6 +274,14 @@ case ${step} in export npe_node_snowanl=$(( npe_node_max / nth_snowanl )) ;; + "prepaeroobs") + export wtime_prepaeroobs="00:15:00" + export npe_prepaeroobs=1 + export nth_prepaeroobs=1 + export npe_node_prepaeroobs=1 + export memory_prepaeroobs="96GB" + ;; + "aeroanlinit") # below lines are for creating JEDI YAML case ${CASE} in diff --git a/scripts/exglobal_prep_aero_obs.py b/scripts/exglobal_prep_aero_obs.py new file mode 100755 index 00000000000..1674baf5bfc --- /dev/null +++ b/scripts/exglobal_prep_aero_obs.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# exglobal_prep_aero_obs.py +# This script collect available viirs +# obs files, combine and preprocess +# them. +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.aero_prepobs import AerosolObsPrep + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the aerosol prep obs tasks + AeroObs = AerosolObsPrep(config) + AeroObs.initialize() + AeroObs.runConverter() + AeroObs.finalize() diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py new file mode 100644 index 00000000000..41bb56090f3 --- /dev/null +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 + +import os +import glob +import gzip +import tarfile +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (AttrDict, FileHandler, rm_p, rmdir, + Task, add_to_datetime, to_timedelta, to_datetime, + datetime_to_YMD, + chdir, Executable, WorkflowException, + parse_j2yaml, save_as_yaml, logit) + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolObsPrep(Task): + """ + Class for preparing and managing aerosol observations + """ + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + + _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) + _window_end = add_to_datetime(self.runtime_config.current_cycle, +to_timedelta(f"{self.config['assim_freq']}H") / 2) + + local_dict = AttrDict( + { + 'window_begin': _window_begin, + 'window_end': _window_end, + 'sensors': str(self.config['SENSORS']).split(','), + 'data_dir': self.config['VIIRS_DATA_DIR'], + 'input_files': '', + 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", + 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z." + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + + + + @logit(logger) + def initialize(self) -> None: + """ + List needed raw obs files. + Link over the raw obs files to COM_OBS. + Link over the needed executable. + Generate corrosponding YMAL file. + Run IODA converter. + """ + + self.task_config.COM_OBS_CHEM = os.path.join(self.task_config.COM_OBS,'chem') + if os.path.exists(self.task_config.COM_OBS_CHEM): + rmdir(self.task_config.COM_OBS_CHEM) + FileHandler({'mkdir': [self.task_config.COM_OBS_CHEM]}).sync() + + self.task_config.DATA_OBS = os.path.join(self.task_config.DATA,'obs') + if os.path.exists(self.task_config.DATA_OBS): + rmdir(self.task_config.DATA_OBS) + FileHandler({'mkdir': [self.task_config.DATA_OBS]}).sync() + + self.task_config.prepaero_yaml = [] + for sensor in self.task_config.sensors: + # print(sensor,'file_list',file_list) + raw_files = self.list_raw_files(sensor) + print('raw_files',raw_files) + self.task_config.input_files = self.copy_obs(raw_files) + print('raw files in obs_chem',self.task_config.input_files) + self.link_obsconvexe() + self.task_config.prepaero_config = self.get_obsproc_config(sensor) + + # generate converter YAML file + _prepaero_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_prepobs_viirs_{sensor}.yaml") + self.task_config.prepaero_yaml.append( _prepaero_yaml) + logger.debug(f"Generate PrepAeroObs YAML file: {_prepaero_yaml}") + save_as_yaml(self.task_config.prepaero_config, _prepaero_yaml) + logger.info(f"Wrote PrepAeroObs YAML to: {_prepaero_yaml}") + + + + @logit(logger) + def list_raw_files(self,sensor) -> List[str]: + """ + List all files in the predefined directory that match the predefined sensor and within the time window. + """ + if sensor == 'n20': + sensor = 'j01' + dir1 = f"{self.task_config.data_dir}/{datetime_to_YMD(self.task_config.window_begin)}" + dir2 = f"{self.task_config.data_dir}/{datetime_to_YMD(self.task_config.window_end)}" + + if dir1 == dir2: + files = os.listdir(dir1) + allfiles = [os.path.join(dir1, file) for file in files] + allfiles.sort() + else: + files_1 = os.listdir(dir1) + allfiles_1 = [os.path.join(dir1, file) for file in files_1] + files_2 = os.listdir(dir2) + allfiles_2 = [os.path.join(dir2, file) for file in files_2] + allfiles = sorted(allfiles_1,allfiles_2) + + matching_files = [] + try: + for file in allfiles: + fshort = file.split('/')[-1].split('.') + yyyy = fshort[0][18:22] + mm = fshort[0][22:24] + dd = fshort[0][24:26] + HH = fshort[0][26:28] + MM = fshort[0][28:30] + fstart = to_datetime(yyyy+'-'+mm+'-'+dd+'T'+HH+':'+MM+'Z') + + if sensor in file: + # temporally select obs files based on time stamp inthe filename. + if (fstart > self.task_config.window_begin) and (fstart < self.task_config.window_end): + # print('time window',sensor,fstart, self.task_config.window_begin, self.task_config.window_end) + matching_files.append(os.path.join(self.task_config.data_dir, file)) + logger.info("Found %d matching files.", len(matching_files)) + except FileNotFoundError: + logger.error("The specified file/directory does not exist.") + raise + return matching_files + + + @logit(logger) + def copy_obs(self,inputfiles) -> Dict[str, Any]: + """ + """ + + copylist = [] + destlist = [] + for filename in inputfiles: + basename = os.path.basename(filename) + dest = os.path.join(self.task_config.DATA_OBS, basename) + copylist.append([filename, dest]) + destlist.append(dest) + FileHandler({'copy': copylist}).sync() + + return destlist + + + @logit(logger) + def get_obsproc_config(self,sensor) -> Dict[str, Any]: + """Compile a dictionary of obs proc configuration from OBSPROCYAML template file + Parameters + ---------- + Returns + ---------- + obsproc_config : Dict + a dictionary containing the fully rendered obs proc yaml configuration + """ + + self.task_config.sensor = sensor + # generate JEDI YAML file + logger.info(f"Generate gdas_obsprovider2ioda YAML config: {self.task_config.OBSPROCYAML}") + prepaero_config = parse_j2yaml(self.task_config.OBSPROCYAML, self.task_config) + # logger.debug(f"OBSPROC config:\n{format(obsproc_config)}") + + return prepaero_config + + + @logit(logger) + def link_obsconvexe(self) -> None: + """ + This method links a JEDI executable to the run directory + Parameters + ---------- + Task: GDAS task + Returns + ---------- + None + """ + exe_src = self.task_config.OBSPROCEXE + + logger.info(f"Link executable {exe_src} to DATA/") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return + + @logit(logger) + def runConverter(self) -> None: + chdir(self.task_config.DATA) + for prepaero_yaml in self.task_config.prepaero_yaml: + exec_cmd = Executable(self.task_config.APRUN_PREPAEROOBS) + exec_name = os.path.join(self.task_config.DATA, 'gdas_obsprovider2ioda.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(prepaero_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + + @logit(logger) + def finalize(self) -> None: + # get list of viirs files + obsfiles = glob.glob(os.path.join(self.task_config['DATA'], '*viirs*nc4')) + copylist = [] + for obsfile in obsfiles: + basename = os.path.basename(obsfile) + src = os.path.join(self.task_config['DATA'],basename) + dest = os.path.join(self.task_config.COM_OBS, basename) + copylist.append([src, dest]) + FileHandler({'copy': copylist}).sync() + + # gzip the files first + for obsfile in obsfiles: + with open(obsfile, 'rb') as f_in, gzip.open(f"{obsfile}.gz", 'wb') as f_out: + f_out.writelines(f_in) + + aeroobs = os.path.join(self.task_config.COM_OBS, f"{self.task_config['APREFIX']}aeroobs") + # open tar file for writing + with tarfile.open(aeroobs, "w") as archive: + for obsfile in obsfiles: + aeroobsgzip = f"{obsfile}.gz" + archive.add(aeroobsgzip, arcname=os.path.basename(aeroobsgzip)) + + + # get list of raw viirs L2 files + rawfiles = glob.glob(os.path.join(self.task_config.DATA_OBS, 'JRR-AOD*')) + # gzip the raw L2 files first + for rawfile in rawfiles: + with open(rawfile, 'rb') as f_in, gzip.open(f"{rawfile}.gz", 'wb') as f_out: + f_out.writelines(f_in) + + aerorawobs = os.path.join(self.task_config.COM_OBS_CHEM, f"{self.task_config['APREFIX']}aerorawobs") + # open tar file for writing + with tarfile.open(aerorawobs, "w") as archive: + for rawfile in rawfiles: + aerorawobsgzip = f"{rawfile}.gz" + archive.add(aerorawobsgzip, arcname=os.path.basename(aerorawobsgzip)) + + copylist = [] + for prepaero_yaml in self.task_config.prepaero_yaml: + basename = os.path.basename(prepaero_yaml) + dest = os.path.join(self.task_config.COM_OBS,basename) + copylist.append([prepaero_yaml,dest]) + FileHandler({'copy': copylist}).sync() + + pass + + + + + diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index c2a6a32f02f..0c2dd1fb921 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,7 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + configs += ['prepaeroobs','aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] if self.do_jedisnowda: configs += ['prepsnowobs', 'snowanl'] @@ -151,7 +151,7 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] if self.do_aero: - gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_gfs_common_tasks_before_fcst += ['prepaeroobs','aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] if self.do_jedisnowda: gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index bba7bac3dd3..9a385684025 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -483,13 +483,37 @@ def atmanlfinal(self): return task - def aeroanlinit(self): + def prepaeroobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + resources = self.get_resource('prepaeroobs') + task_name = f'{self.cdump}prepaeroobs' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/prepaeroobs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + + def aeroanlinit(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}prepaeroobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + resources = self.get_resource('aeroanlinit') task_name = f'{self.cdump}aeroanlinit' task_dict = {'task_name': task_name, diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 3abae9b5b7a..8962aecdba7 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -19,7 +19,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlrun', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', + 'prepaeroobs', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', From 9d5aeb7471f0ca8048761afbe0ccb7ddbe8ddcb3 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Fri, 24 May 2024 13:49:36 +0000 Subject: [PATCH 02/26] update yaml file name --- ush/python/pygfs/task/aero_prepobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index 41bb56090f3..637cfd106dc 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -74,7 +74,7 @@ def initialize(self) -> None: self.task_config.prepaero_config = self.get_obsproc_config(sensor) # generate converter YAML file - _prepaero_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_prepobs_viirs_{sensor}.yaml") + _prepaero_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml") self.task_config.prepaero_yaml.append( _prepaero_yaml) logger.debug(f"Generate PrepAeroObs YAML file: {_prepaero_yaml}") save_as_yaml(self.task_config.prepaero_config, _prepaero_yaml) From bdc5338d2c43542824c03a22b77b31ca616cd5f9 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Fri, 24 May 2024 20:06:21 +0000 Subject: [PATCH 03/26] add option to trun on/off prepaeroobs --- parm/config/gfs/config.aeroanl | 4 ---- parm/config/gfs/config.base | 1 + parm/config/gfs/config.prepaeroobs | 4 +++- sorc/gdas.cd | 2 +- sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/gsi_monitor.fd | 2 +- sorc/gsi_utils.fd | 2 +- sorc/ufs_model.fd | 2 +- sorc/upp.fd | 2 +- sorc/wxflow | 2 +- workflow/applications/applications.py | 1 + workflow/applications/gfs_cycled.py | 9 +++++++-- workflow/rocoto/gfs_tasks.py | 5 +++-- workflow/rocoto/tasks.py | 2 +- 15 files changed, 24 insertions(+), 18 deletions(-) diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index f9fde34b6c1..37c8959e8aa 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -20,10 +20,6 @@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" -export OBSPROCEXE="${EXECgfs}/gdas_obsprovider2ioda.x" -export VIIRS_DATA_DIR="/scratch2/NCEPDEV/stmp3/Yaping.Wang/VIIRS/AWS/" -export SENSORS="npp" - if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index f1e25750eff..3ded0a541c1 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -178,6 +178,7 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" +export DO_PREP_AERO_OBS="YES" export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both diff --git a/parm/config/gfs/config.prepaeroobs b/parm/config/gfs/config.prepaeroobs index f99825e056b..be710faf6cc 100644 --- a/parm/config/gfs/config.prepaeroobs +++ b/parm/config/gfs/config.prepaeroobs @@ -9,7 +9,9 @@ echo "BEGIN: config.prepaeroobs" source "${EXPDIR}/config.resources" prepaeroobs export OBSPROCYAML="${PARMgfs}/gdas/aero/obs/list/aero_obsproc.yaml.j2" - +export OBSPROCEXE="${EXECgfs}/gdas_obsprovider2ioda.x" +export VIIRS_DATA_DIR="/scratch2/NCEPDEV/stmp3/Yaping.Wang/VIIRS/AWS/" +export SENSORS="npp,n20" echo "END: config.prepaeroobs" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 249e242e33a..fc62ef5f4dd 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 249e242e33a33feeb1c81bedd51198309f669de0 +Subproject commit fc62ef5f4ddcd2c13df515b593e5ce1c7b5dd10b diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 0cdc2795260..d77c3c99a3b 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 0cdc2795260fc1b59e86a873729433a470794a97 +Subproject commit d77c3c99a3bc6e993c38b8ea987ceb4e68c11ebb diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 8e279f9c734..457510c72e4 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 8e279f9c734097f673b07e80f385b2623d13ba4a +Subproject commit 457510c72e486b7b01db09e5b1a6f407778dc772 diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index f9d6f5f7444..8efe38eadeb 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit f9d6f5f744462a449e70abed8c5860b1c4564ad8 +Subproject commit 8efe38eadebbd5d50284aee44f6d8b6799a7f6e6 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 0cdc3b4f7ff..68bc14d30b3 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 0cdc3b4f7ff8d4f0c54da3dab70ea2743bd68478 +Subproject commit 68bc14d30b3ca8f890f2761c8bdd0a3cea635cf1 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index c54e98637ea..7fdb58cad0d 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit c54e98637ead81b1fc1e336bd0443c8bfb6faf01 +Subproject commit 7fdb58cad0dad2f62ce7813c6719554d1c5a17af diff --git a/sorc/upp.fd b/sorc/upp.fd index 83e83a938b5..4770a2f509b 160000 --- a/sorc/upp.fd +++ b/sorc/upp.fd @@ -1 +1 @@ -Subproject commit 83e83a938b5794a628d30e66a54902dabe58737d +Subproject commit 4770a2f509b7122e76c4f004210031a58ae9502c diff --git a/sorc/wxflow b/sorc/wxflow index 71f6b10f76a..942b90bfaa1 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 71f6b10f76a440993580027ba1183d61277d1299 +Subproject commit 942b90bfaa14f6b6d7374310dbdfd421ddb30548 diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index adfab16496a..2a13a5e607a 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -51,6 +51,7 @@ def __init__(self, conf: Configuration) -> None: self.do_ocean = _base.get('DO_OCN', False) self.do_ice = _base.get('DO_ICE', False) self.do_aero = _base.get('DO_AERO', False) + self.do_prep_aero_obs = _base.get('DO_PREP_AERO_OBS', False) self.do_bufrsnd = _base.get('DO_BUFRSND', False) self.do_gempak = _base.get('DO_GEMPAK', False) self.do_awips = _base.get('DO_AWIPS', False) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index cb702e87a85..dc789a1bec4 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,9 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['prepaeroobs','aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + if self.do_prep_aero_obs: + configs += ['prepaeroobs'] if self.do_jedisnowda: configs += ['prepsnowobs', 'snowanl'] @@ -151,7 +153,10 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] if self.do_aero: - gdas_gfs_common_tasks_before_fcst += ['prepaeroobs','aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + if self.do_prep_aero_obs: + gdas_gfs_common_tasks_before_fcst += ['prepaeroobs'] + if self.do_jedisnowda: gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 1c5eebcc59c..3bc33f28e14 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -484,7 +484,6 @@ def atmanlfinal(self): return task def prepaeroobs(self): - deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) @@ -510,7 +509,9 @@ def prepaeroobs(self): def aeroanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}prepaeroobs'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + if self.app_config.do_prep_aero_obs: + dep_dict = {'type': 'task', 'name': f'{self.cdump}prepaeroobs'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index ce228205acc..a8b4eb9fac4 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -19,7 +19,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlrun', 'atmensanlfinal', - 'prepaeroobs', 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', + 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', From c014ae16796a6ba9db74a7be5d1a2358b44de3f0 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Fri, 24 May 2024 20:52:12 +0000 Subject: [PATCH 04/26] add option to turn on/off prepaeroobs --- ush/python/pygfs/task/aero_prepobs.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index 637cfd106dc..83908c4f9a0 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -23,8 +23,8 @@ class AerosolObsPrep(Task): def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _window_end = add_to_datetime(self.runtime_config.current_cycle, +to_timedelta(f"{self.config['assim_freq']}H") / 2) + _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) + _window_end = add_to_datetime(self.runtime_config.current_cycle, +to_timedelta(f"{self.config['assim_freq']}H") / 2) local_dict = AttrDict( { @@ -41,13 +41,11 @@ def __init__(self, config: Dict[str, Any]) -> None: # task_config is everything that this task should need self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) - - @logit(logger) def initialize(self) -> None: """ List needed raw obs files. - Link over the raw obs files to COM_OBS. + Copy the raw obs files to $DATA/obs. Link over the needed executable. Generate corrosponding YMAL file. Run IODA converter. @@ -129,6 +127,7 @@ def list_raw_files(self,sensor) -> List[str]: @logit(logger) def copy_obs(self,inputfiles) -> Dict[str, Any]: """ + Copy the raw obs files to $DATA/obs. """ copylist = [] @@ -166,7 +165,7 @@ def get_obsproc_config(self,sensor) -> Dict[str, Any]: @logit(logger) def link_obsconvexe(self) -> None: """ - This method links a JEDI executable to the run directory + This method links the gdas executable to the run directory Parameters ---------- Task: GDAS task @@ -186,6 +185,9 @@ def link_obsconvexe(self) -> None: @logit(logger) def runConverter(self) -> None: + """ + Run the IODA converter gdas_obsprovider2ioda.x + """ chdir(self.task_config.DATA) for prepaero_yaml in self.task_config.prepaero_yaml: exec_cmd = Executable(self.task_config.APRUN_PREPAEROOBS) @@ -206,6 +208,11 @@ def runConverter(self) -> None: @logit(logger) def finalize(self) -> None: + """ + Copy the output viirs files to COM_OBS. + Tar and archive the output files. + Tar and archive the raw obs files. + """ # get list of viirs files obsfiles = glob.glob(os.path.join(self.task_config['DATA'], '*viirs*nc4')) copylist = [] From 8d64fdd89d18a0cea1c5ae2731c5229d379a81fd Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Fri, 24 May 2024 10:52:45 -0400 Subject: [PATCH 05/26] Update STMP and PTMP settings in host file for Orion and Hercules (#2614) - Updating STMP and PTMP settings in host file for Orion and Hercules because they are cross mounted. - Also took the opportunity to finally update **SLURM_ACCOUNT** to **HPC_ACCOUT** in CI over rides. - Added a refactor of the `rocotostat.py` tool that is more pythonic and as a execute retry feature because the `rocotostat` utility on Orion has been failing sometimes. --- ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml | 2 +- ci/cases/yamls/gefs_ci_defaults.yaml | 2 +- ci/cases/yamls/gfs_defaults_ci.yaml | 2 +- ci/cases/yamls/gfs_extended_ci.yaml | 2 +- ci/cases/yamls/soca_gfs_defaults_ci.yaml | 2 +- ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml | 2 +- ci/platforms/config.hera | 4 +- ci/platforms/config.hercules | 4 +- ci/platforms/config.orion | 4 +- ci/platforms/config.wcoss2 | 4 +- ci/scripts/run-check_ci.sh | 2 +- ci/scripts/utils/rocotostat.py | 156 ++++++++++++++---- workflow/hosts/hercules.yaml | 4 +- workflow/hosts/orion.yaml | 4 +- 14 files changed, 140 insertions(+), 54 deletions(-) diff --git a/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml b/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml index 417525742e6..624af591fcc 100644 --- a/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml +++ b/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml @@ -3,4 +3,4 @@ defaults: base: DOIAU: "NO" DO_JEDISNOWDA: "YES" - ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} diff --git a/ci/cases/yamls/gefs_ci_defaults.yaml b/ci/cases/yamls/gefs_ci_defaults.yaml index dfb1626cdd8..ceb36d4acb7 100644 --- a/ci/cases/yamls/gefs_ci_defaults.yaml +++ b/ci/cases/yamls/gefs_ci_defaults.yaml @@ -1,4 +1,4 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml base: - ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} + HPC_ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index b66be2a366f..d09f78b8b86 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -1,4 +1,4 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml index 4d4f79e0e86..f3a84c8fdeb 100644 --- a/ci/cases/yamls/gfs_extended_ci.yaml +++ b/ci/cases/yamls/gfs_extended_ci.yaml @@ -2,7 +2,7 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_GOES: "YES" DO_BUFRSND: "YES" DO_GEMPAK: "YES" diff --git a/ci/cases/yamls/soca_gfs_defaults_ci.yaml b/ci/cases/yamls/soca_gfs_defaults_ci.yaml index 126637cd866..3d75cc911a5 100644 --- a/ci/cases/yamls/soca_gfs_defaults_ci.yaml +++ b/ci/cases/yamls/soca_gfs_defaults_ci.yaml @@ -1,5 +1,5 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_JEDIOCNVAR: "YES" diff --git a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml index 1075f55b63b..c4fa54dcc84 100644 --- a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml +++ b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml @@ -4,7 +4,7 @@ base: DOIAU: "NO" DO_JEDIATMVAR: "YES" DO_JEDIATMENS: "YES" - ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} atmanl: LAYOUT_X_ATMANL: 4 LAYOUT_Y_ATMANL: 4 diff --git a/ci/platforms/config.hera b/ci/platforms/config.hera index ae08293edfb..7e85bb33da8 100644 --- a/ci/platforms/config.hera +++ b/ci/platforms/config.hera @@ -2,8 +2,6 @@ export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR -export STMP="/scratch1/NCEPDEV/stmp2/${USER}" -export PTMP="/scratch1/NCEPDEV/stmp2/${USER}" -export SLURM_ACCOUNT=nems +export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/platforms/config.hercules b/ci/platforms/config.hercules index 7e8db00b4c7..12846c18a4b 100644 --- a/ci/platforms/config.hercules +++ b/ci/platforms/config.hercules @@ -2,8 +2,6 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR -export STMP="/work2/noaa/stmp/${USER}/HERCULES" -export PTMP="/work2/noaa/stmp/${USER}/HERCULES" -export SLURM_ACCOUNT=nems +export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/platforms/config.orion b/ci/platforms/config.orion index b6438e8564d..5a7596001b4 100644 --- a/ci/platforms/config.orion +++ b/ci/platforms/config.orion @@ -2,8 +2,6 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR -export STMP="/work2/noaa/stmp/${USER}/ORION" -export PTMP="/work2/noaa/stmp/${USER}/ORION" -export SLURM_ACCOUNT=nems +export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/platforms/config.wcoss2 b/ci/platforms/config.wcoss2 index 44882f3f193..7a840ad2f8e 100644 --- a/ci/platforms/config.wcoss2 +++ b/ci/platforms/config.wcoss2 @@ -2,8 +2,6 @@ export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR -export STMP="/lfs/h2/emc/stmp/${USER}" -export PTMP="/lfs/h2/emc/ptmp/${USER}" -export SLURM_ACCOUNT=GFS-DEV +export HPC_ACCOUNT=GFS-DEV export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index ee089dadc27..5c891fc4bdc 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -64,7 +64,7 @@ while true; do rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}" # Wait before running rocotostat - sleep 10 + sleep 60 # Get job statistics echo "Gather Rocoto statistics" diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py index 884ff82e3df..9b1d8dcc3af 100755 --- a/ci/scripts/utils/rocotostat.py +++ b/ci/scripts/utils/rocotostat.py @@ -2,13 +2,31 @@ import sys import os +import copy +from time import sleep -from wxflow import Executable, which, Logger, CommandNotFoundError +from wxflow import which, Logger, CommandNotFoundError, ProcessError from argparse import ArgumentParser, FileType +from collections import Counter + logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False) +def attempt_multiple_times(expression, max_attempts, sleep_duration=0, exception_class=Exception): + attempt = 0 + last_exception = None + while attempt < max_attempts: + try: + pass + return expression() + except exception_class as last_exception: + attempt += 1 + sleep(sleep_duration) + else: + raise last_exception + + def input_args(): """ Parse command-line arguments. @@ -39,52 +57,130 @@ def input_args(): return args -def rocoto_statcount(): - """ - Run rocotostat and process its output. +def rocotostat_summary(rocotostat): """ + rocoto_summary Run rocotostat and process its output. - args = input_args() + rocoto_summary(rocotostat) adds a default argument '--summary' to the rocotostat + command, runs it, and processes its output to return a dictionary with the total + number of cycles and the number of cycles marked as 'Done'. - try: - rocotostat = which("rocotostat") - except CommandNotFoundError: - logger.exception("rocotostat not found in PATH") - raise CommandNotFoundError("rocotostat not found in PATH") - - rocotostat_all = which("rocotostat") - rocotostat.add_default_arg(['-w', os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name), '-s']) - rocotostat_all.add_default_arg(['-w', os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name), '-a']) + Input: + rocotostat - The rocotostat command. - rocotostat_output = rocotostat(output=str) + Output: + rocoto_status - A dictionary with the total number of cycles and the number of cycles marked as 'Done'. + """ + rocotostat = copy.deepcopy(rocotostat) + rocotostat.add_default_arg('--summary') + rocotostat_output = attempt_multiple_times(lambda: rocotostat(output=str), 3, 90, ProcessError) rocotostat_output = rocotostat_output.splitlines()[1:] rocotostat_output = [line.split()[0:2] for line in rocotostat_output] - rocotostat_output_all = rocotostat_all(output=str) - rocotostat_output_all = rocotostat_output_all.splitlines()[1:] - rocotostat_output_all = [line.split()[0:4] for line in rocotostat_output_all] - rocotostat_output_all = [line for line in rocotostat_output_all if len(line) != 1] - rocoto_status = { 'CYCLES_TOTAL': len(rocotostat_output), 'CYCLES_DONE': sum([sublist.count('Done') for sublist in rocotostat_output]) } + return rocoto_status + + +def rocoto_statcount(rocotostat): + """ + rocoto_statcount Run rocotostat and process its output. + + rocoto_statcount(rocotostat) adds a default argument '--all' to the rocotostat + command, runs it, and processes its output to return a dictionary with the count + of each status case. + + Input: + rocotostat - The rocotostat command. + + Output: + rocoto_status - A dictionary with the count of each status case. + """ + + rocotostat = copy.deepcopy(rocotostat) + rocotostat.add_default_arg('--all') + + rocotostat_output = attempt_multiple_times(lambda: rocotostat(output=str), 4, 120, ProcessError) + rocotostat_output = rocotostat_output.splitlines()[1:] + rocotostat_output = [line.split()[0:4] for line in rocotostat_output] + rocotostat_output = [line for line in rocotostat_output if len(line) != 1] status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED'] + + rocoto_status = {} + status_counts = Counter(case for sublist in rocotostat_output for case in sublist) for case in status_cases: - rocoto_status[case] = sum([sublist.count(case) for sublist in rocotostat_output_all]) + rocoto_status[case] = status_counts[case] return rocoto_status +def is_done(rocoto_status): + """ + is_done Check if all cycles are done. + + is_done(rocoto_status) checks if the total number of cycles equals the number of + done cycles in the rocoto_status dictionary. + + Input: + rocoto_status - A dictionary with the count of each status case. + + Output: + boolean - True if all cycles are done, False otherwise. + """ + + if rocoto_status['CYCLES_TOTAL'] == rocoto_status['CYCLES_DONE']: + return True + else: + return False + + +def is_stalled(rocoto_status): + """ + is_stalled Check if all cycles are stalled. + + is_stalled(rocoto_status) checks if all cycles are stalled by verifying if + there are no jobs that are RUNNING, SUBMITTING, or QUEUED. + + Input: + rocoto_status - A dictionary with the count of each status case. + + Output: + boolean - True if all cycles are stalled, False otherwise. + """ + + if rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0: + return True + else: + return False + + if __name__ == '__main__': + """ + main Execute the script. + + main() parses the input arguments, checks if the rocotostat command is available, + adds default arguments to the rocotostat command, and runs it and reports + out to stdout spcific information of rocoto workflow. + """ args = input_args() - error_return = 0 - rocoto_status = rocoto_statcount() + try: + rocotostat = which("rocotostat") + except CommandNotFoundError: + logger.exception("rocotostat not found in PATH") + raise CommandNotFoundError("rocotostat not found in PATH") - if rocoto_status['CYCLES_TOTAL'] == rocoto_status['CYCLES_DONE']: + rocotostat.add_default_arg(['-w', os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name)]) + + rocoto_status = rocoto_statcount(rocotostat) + rocoto_status.update(rocotostat_summary(rocotostat)) + + error_return = 0 + if is_done(rocoto_status): rocoto_state = 'DONE' elif rocoto_status['DEAD'] > 0: error_return = rocoto_status['FAIL'] + rocoto_status['DEAD'] @@ -92,13 +188,11 @@ def rocoto_statcount(): elif 'UNKNOWN' in rocoto_status: error_return = rocoto_status['UNKNOWN'] rocoto_state = 'UNKNOWN' - elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0: - # - # TODO for now a STALLED state will be just a warning as it can - # produce a false negative if there is a timestamp on a file dependency. - # - # error_return = -3 - rocoto_state = 'STALLED' + elif is_stalled(rocoto_status): + rocoto_status = attempt_multiple_times(rocoto_statcount(rocotostat), 2, 120, ProcessError) + if is_stalled(rocoto_status): + error_return = 3 + rocoto_state = 'STALLED' else: rocoto_state = 'RUNNING' diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index 26236727090..a3c0733b8c4 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -4,8 +4,8 @@ BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' -STMP: '/work/noaa/stmp/${USER}' -PTMP: '/work/noaa/stmp/${USER}' +STMP: '/work/noaa/stmp/${USER}/HERCULES' +PTMP: '/work/noaa/stmp/${USER}/HERCULES' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index dd95def3866..4f8f02b4c66 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -4,8 +4,8 @@ BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' -STMP: '/work/noaa/stmp/${USER}' -PTMP: '/work/noaa/stmp/${USER}' +STMP: '/work/noaa/stmp/${USER}/ORION' +PTMP: '/work/noaa/stmp/${USER}/ORION' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu From 5c31d4a4cfbbb68da256228d47c2e3c2aa617437 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Tue, 28 May 2024 17:41:59 +0000 Subject: [PATCH 06/26] fix commit --- sorc/gdas.cd | 2 +- sorc/gfs_utils.fd | 2 +- sorc/gsi_enkf.fd | 2 +- sorc/gsi_monitor.fd | 2 +- sorc/gsi_utils.fd | 2 +- sorc/ufs_model.fd | 2 +- sorc/upp.fd | 2 +- sorc/wxflow | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index fc62ef5f4dd..249e242e33a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit fc62ef5f4ddcd2c13df515b593e5ce1c7b5dd10b +Subproject commit 249e242e33a33feeb1c81bedd51198309f669de0 diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index d77c3c99a3b..0cdc2795260 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit d77c3c99a3bc6e993c38b8ea987ceb4e68c11ebb +Subproject commit 0cdc2795260fc1b59e86a873729433a470794a97 diff --git a/sorc/gsi_enkf.fd b/sorc/gsi_enkf.fd index 457510c72e4..8e279f9c734 160000 --- a/sorc/gsi_enkf.fd +++ b/sorc/gsi_enkf.fd @@ -1 +1 @@ -Subproject commit 457510c72e486b7b01db09e5b1a6f407778dc772 +Subproject commit 8e279f9c734097f673b07e80f385b2623d13ba4a diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 8efe38eadeb..f9d6f5f7444 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 8efe38eadebbd5d50284aee44f6d8b6799a7f6e6 +Subproject commit f9d6f5f744462a449e70abed8c5860b1c4564ad8 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 68bc14d30b3..0cdc3b4f7ff 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 68bc14d30b3ca8f890f2761c8bdd0a3cea635cf1 +Subproject commit 0cdc3b4f7ff8d4f0c54da3dab70ea2743bd68478 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 7fdb58cad0d..c54e98637ea 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 7fdb58cad0dad2f62ce7813c6719554d1c5a17af +Subproject commit c54e98637ead81b1fc1e336bd0443c8bfb6faf01 diff --git a/sorc/upp.fd b/sorc/upp.fd index 4770a2f509b..83e83a938b5 160000 --- a/sorc/upp.fd +++ b/sorc/upp.fd @@ -1 +1 @@ -Subproject commit 4770a2f509b7122e76c4f004210031a58ae9502c +Subproject commit 83e83a938b5794a628d30e66a54902dabe58737d diff --git a/sorc/wxflow b/sorc/wxflow index 942b90bfaa1..71f6b10f76a 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 942b90bfaa14f6b6d7374310dbdfd421ddb30548 +Subproject commit 71f6b10f76a440993580027ba1183d61277d1299 From 1b7482ad973593aee0278300c84f618e97b435dd Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Tue, 28 May 2024 17:56:27 +0000 Subject: [PATCH 07/26] fix previous commit --- ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml | 2 +- ci/cases/yamls/gefs_ci_defaults.yaml | 2 +- ci/cases/yamls/gfs_defaults_ci.yaml | 2 +- ci/cases/yamls/gfs_extended_ci.yaml | 2 +- ci/cases/yamls/soca_gfs_defaults_ci.yaml | 2 +- ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml | 2 +- ci/platforms/config.hera | 4 +- ci/platforms/config.hercules | 4 +- ci/platforms/config.orion | 4 +- ci/platforms/config.wcoss2 | 4 +- ci/scripts/run-check_ci.sh | 2 +- ci/scripts/utils/rocotostat.py | 156 ++++-------------- 12 files changed, 50 insertions(+), 136 deletions(-) diff --git a/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml b/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml index 624af591fcc..417525742e6 100644 --- a/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml +++ b/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml @@ -3,4 +3,4 @@ defaults: base: DOIAU: "NO" DO_JEDISNOWDA: "YES" - ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} diff --git a/ci/cases/yamls/gefs_ci_defaults.yaml b/ci/cases/yamls/gefs_ci_defaults.yaml index ceb36d4acb7..dfb1626cdd8 100644 --- a/ci/cases/yamls/gefs_ci_defaults.yaml +++ b/ci/cases/yamls/gefs_ci_defaults.yaml @@ -1,4 +1,4 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml base: - HPC_ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml index d09f78b8b86..b66be2a366f 100644 --- a/ci/cases/yamls/gfs_defaults_ci.yaml +++ b/ci/cases/yamls/gfs_defaults_ci.yaml @@ -1,4 +1,4 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml index f3a84c8fdeb..4d4f79e0e86 100644 --- a/ci/cases/yamls/gfs_extended_ci.yaml +++ b/ci/cases/yamls/gfs_extended_ci.yaml @@ -2,7 +2,7 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} DO_GOES: "YES" DO_BUFRSND: "YES" DO_GEMPAK: "YES" diff --git a/ci/cases/yamls/soca_gfs_defaults_ci.yaml b/ci/cases/yamls/soca_gfs_defaults_ci.yaml index 3d75cc911a5..126637cd866 100644 --- a/ci/cases/yamls/soca_gfs_defaults_ci.yaml +++ b/ci/cases/yamls/soca_gfs_defaults_ci.yaml @@ -1,5 +1,5 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} DO_JEDIOCNVAR: "YES" diff --git a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml index c4fa54dcc84..1075f55b63b 100644 --- a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml +++ b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml @@ -4,7 +4,7 @@ base: DOIAU: "NO" DO_JEDIATMVAR: "YES" DO_JEDIATMENS: "YES" - ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} atmanl: LAYOUT_X_ATMANL: 4 LAYOUT_Y_ATMANL: 4 diff --git a/ci/platforms/config.hera b/ci/platforms/config.hera index 7e85bb33da8..ae08293edfb 100644 --- a/ci/platforms/config.hera +++ b/ci/platforms/config.hera @@ -2,6 +2,8 @@ export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR -export HPC_ACCOUNT=nems +export STMP="/scratch1/NCEPDEV/stmp2/${USER}" +export PTMP="/scratch1/NCEPDEV/stmp2/${USER}" +export SLURM_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/platforms/config.hercules b/ci/platforms/config.hercules index 12846c18a4b..7e8db00b4c7 100644 --- a/ci/platforms/config.hercules +++ b/ci/platforms/config.hercules @@ -2,6 +2,8 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR -export HPC_ACCOUNT=nems +export STMP="/work2/noaa/stmp/${USER}/HERCULES" +export PTMP="/work2/noaa/stmp/${USER}/HERCULES" +export SLURM_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/platforms/config.orion b/ci/platforms/config.orion index 5a7596001b4..b6438e8564d 100644 --- a/ci/platforms/config.orion +++ b/ci/platforms/config.orion @@ -2,6 +2,8 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR -export HPC_ACCOUNT=nems +export STMP="/work2/noaa/stmp/${USER}/ORION" +export PTMP="/work2/noaa/stmp/${USER}/ORION" +export SLURM_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/platforms/config.wcoss2 b/ci/platforms/config.wcoss2 index 7a840ad2f8e..44882f3f193 100644 --- a/ci/platforms/config.wcoss2 +++ b/ci/platforms/config.wcoss2 @@ -2,6 +2,8 @@ export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR -export HPC_ACCOUNT=GFS-DEV +export STMP="/lfs/h2/emc/stmp/${USER}" +export PTMP="/lfs/h2/emc/ptmp/${USER}" +export SLURM_ACCOUNT=GFS-DEV export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh index 5c891fc4bdc..ee089dadc27 100755 --- a/ci/scripts/run-check_ci.sh +++ b/ci/scripts/run-check_ci.sh @@ -64,7 +64,7 @@ while true; do rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}" # Wait before running rocotostat - sleep 60 + sleep 10 # Get job statistics echo "Gather Rocoto statistics" diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py index 9b1d8dcc3af..884ff82e3df 100755 --- a/ci/scripts/utils/rocotostat.py +++ b/ci/scripts/utils/rocotostat.py @@ -2,31 +2,13 @@ import sys import os -import copy -from time import sleep -from wxflow import which, Logger, CommandNotFoundError, ProcessError +from wxflow import Executable, which, Logger, CommandNotFoundError from argparse import ArgumentParser, FileType -from collections import Counter - logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False) -def attempt_multiple_times(expression, max_attempts, sleep_duration=0, exception_class=Exception): - attempt = 0 - last_exception = None - while attempt < max_attempts: - try: - pass - return expression() - except exception_class as last_exception: - attempt += 1 - sleep(sleep_duration) - else: - raise last_exception - - def input_args(): """ Parse command-line arguments. @@ -57,130 +39,52 @@ def input_args(): return args -def rocotostat_summary(rocotostat): +def rocoto_statcount(): + """ + Run rocotostat and process its output. """ - rocoto_summary Run rocotostat and process its output. - rocoto_summary(rocotostat) adds a default argument '--summary' to the rocotostat - command, runs it, and processes its output to return a dictionary with the total - number of cycles and the number of cycles marked as 'Done'. + args = input_args() - Input: - rocotostat - The rocotostat command. + try: + rocotostat = which("rocotostat") + except CommandNotFoundError: + logger.exception("rocotostat not found in PATH") + raise CommandNotFoundError("rocotostat not found in PATH") - Output: - rocoto_status - A dictionary with the total number of cycles and the number of cycles marked as 'Done'. - """ - rocotostat = copy.deepcopy(rocotostat) - rocotostat.add_default_arg('--summary') - rocotostat_output = attempt_multiple_times(lambda: rocotostat(output=str), 3, 90, ProcessError) + rocotostat_all = which("rocotostat") + rocotostat.add_default_arg(['-w', os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name), '-s']) + rocotostat_all.add_default_arg(['-w', os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name), '-a']) + + rocotostat_output = rocotostat(output=str) rocotostat_output = rocotostat_output.splitlines()[1:] rocotostat_output = [line.split()[0:2] for line in rocotostat_output] + rocotostat_output_all = rocotostat_all(output=str) + rocotostat_output_all = rocotostat_output_all.splitlines()[1:] + rocotostat_output_all = [line.split()[0:4] for line in rocotostat_output_all] + rocotostat_output_all = [line for line in rocotostat_output_all if len(line) != 1] + rocoto_status = { 'CYCLES_TOTAL': len(rocotostat_output), 'CYCLES_DONE': sum([sublist.count('Done') for sublist in rocotostat_output]) } - return rocoto_status - - -def rocoto_statcount(rocotostat): - """ - rocoto_statcount Run rocotostat and process its output. - - rocoto_statcount(rocotostat) adds a default argument '--all' to the rocotostat - command, runs it, and processes its output to return a dictionary with the count - of each status case. - - Input: - rocotostat - The rocotostat command. - - Output: - rocoto_status - A dictionary with the count of each status case. - """ - - rocotostat = copy.deepcopy(rocotostat) - rocotostat.add_default_arg('--all') - - rocotostat_output = attempt_multiple_times(lambda: rocotostat(output=str), 4, 120, ProcessError) - rocotostat_output = rocotostat_output.splitlines()[1:] - rocotostat_output = [line.split()[0:4] for line in rocotostat_output] - rocotostat_output = [line for line in rocotostat_output if len(line) != 1] status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED'] - - rocoto_status = {} - status_counts = Counter(case for sublist in rocotostat_output for case in sublist) for case in status_cases: - rocoto_status[case] = status_counts[case] + rocoto_status[case] = sum([sublist.count(case) for sublist in rocotostat_output_all]) return rocoto_status -def is_done(rocoto_status): - """ - is_done Check if all cycles are done. - - is_done(rocoto_status) checks if the total number of cycles equals the number of - done cycles in the rocoto_status dictionary. - - Input: - rocoto_status - A dictionary with the count of each status case. - - Output: - boolean - True if all cycles are done, False otherwise. - """ - - if rocoto_status['CYCLES_TOTAL'] == rocoto_status['CYCLES_DONE']: - return True - else: - return False - - -def is_stalled(rocoto_status): - """ - is_stalled Check if all cycles are stalled. - - is_stalled(rocoto_status) checks if all cycles are stalled by verifying if - there are no jobs that are RUNNING, SUBMITTING, or QUEUED. - - Input: - rocoto_status - A dictionary with the count of each status case. - - Output: - boolean - True if all cycles are stalled, False otherwise. - """ - - if rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0: - return True - else: - return False - - if __name__ == '__main__': - """ - main Execute the script. - - main() parses the input arguments, checks if the rocotostat command is available, - adds default arguments to the rocotostat command, and runs it and reports - out to stdout spcific information of rocoto workflow. - """ args = input_args() - try: - rocotostat = which("rocotostat") - except CommandNotFoundError: - logger.exception("rocotostat not found in PATH") - raise CommandNotFoundError("rocotostat not found in PATH") - - rocotostat.add_default_arg(['-w', os.path.abspath(args.w.name), '-d', os.path.abspath(args.d.name)]) - - rocoto_status = rocoto_statcount(rocotostat) - rocoto_status.update(rocotostat_summary(rocotostat)) - error_return = 0 - if is_done(rocoto_status): + rocoto_status = rocoto_statcount() + + if rocoto_status['CYCLES_TOTAL'] == rocoto_status['CYCLES_DONE']: rocoto_state = 'DONE' elif rocoto_status['DEAD'] > 0: error_return = rocoto_status['FAIL'] + rocoto_status['DEAD'] @@ -188,11 +92,13 @@ def is_stalled(rocoto_status): elif 'UNKNOWN' in rocoto_status: error_return = rocoto_status['UNKNOWN'] rocoto_state = 'UNKNOWN' - elif is_stalled(rocoto_status): - rocoto_status = attempt_multiple_times(rocoto_statcount(rocotostat), 2, 120, ProcessError) - if is_stalled(rocoto_status): - error_return = 3 - rocoto_state = 'STALLED' + elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0: + # + # TODO for now a STALLED state will be just a warning as it can + # produce a false negative if there is a timestamp on a file dependency. + # + # error_return = -3 + rocoto_state = 'STALLED' else: rocoto_state = 'RUNNING' From c22400263cb0e6ddc6f6b0c810de97ceaf4443b4 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Tue, 28 May 2024 17:58:06 +0000 Subject: [PATCH 08/26] fix previous commit --- workflow/hosts/hercules.yaml | 4 ++-- workflow/hosts/orion.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index a3c0733b8c4..26236727090 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -4,8 +4,8 @@ BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' -STMP: '/work/noaa/stmp/${USER}/HERCULES' -PTMP: '/work/noaa/stmp/${USER}/HERCULES' +STMP: '/work/noaa/stmp/${USER}' +PTMP: '/work/noaa/stmp/${USER}' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 4f8f02b4c66..dd95def3866 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -4,8 +4,8 @@ BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' PACKAGEROOT: '/work/noaa/global/glopara/nwpara' COMINsyn: '/work/noaa/global/glopara/com/gfs/prod/syndat' HOMEDIR: '/work/noaa/global/${USER}' -STMP: '/work/noaa/stmp/${USER}/ORION' -PTMP: '/work/noaa/stmp/${USER}/ORION' +STMP: '/work/noaa/stmp/${USER}' +PTMP: '/work/noaa/stmp/${USER}' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu From 8beebeef6f35da7dd24dbcc9a57c49f76c13c591 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Tue, 28 May 2024 19:02:13 +0000 Subject: [PATCH 09/26] fix coding norms issues --- scripts/exglobal_prep_aero_obs.py | 5 +- ush/python/pygfs/task/aero_prepobs.py | 68 +++++++++------------------ workflow/applications/gfs_cycled.py | 3 +- 3 files changed, 26 insertions(+), 50 deletions(-) diff --git a/scripts/exglobal_prep_aero_obs.py b/scripts/exglobal_prep_aero_obs.py index 1674baf5bfc..0f5229f78c1 100755 --- a/scripts/exglobal_prep_aero_obs.py +++ b/scripts/exglobal_prep_aero_obs.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # exglobal_prep_aero_obs.py -# This script collect available viirs +# This script collect available viirs # obs files, combine and preprocess -# them. +# them. import os from wxflow import Logger, cast_strdict_as_dtypedict @@ -12,7 +12,6 @@ logger = Logger(level='DEBUG', colored_log=True) - if __name__ == '__main__': # Take configuration from environment and cast it as python dictionary diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index 83908c4f9a0..bf7c5f7e34d 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -50,43 +50,38 @@ def initialize(self) -> None: Generate corrosponding YMAL file. Run IODA converter. """ - - self.task_config.COM_OBS_CHEM = os.path.join(self.task_config.COM_OBS,'chem') + self.task_config.COM_OBS_CHEM = os.path.join(self.task_config.COM_OBS, 'chem') if os.path.exists(self.task_config.COM_OBS_CHEM): - rmdir(self.task_config.COM_OBS_CHEM) + rmdir(self.task_config.COM_OBS_CHEM) FileHandler({'mkdir': [self.task_config.COM_OBS_CHEM]}).sync() - self.task_config.DATA_OBS = os.path.join(self.task_config.DATA,'obs') + self.task_config.DATA_OBS = os.path.join(self.task_config.DATA, 'obs') if os.path.exists(self.task_config.DATA_OBS): - rmdir(self.task_config.DATA_OBS) + rmdir(self.task_config.DATA_OBS) FileHandler({'mkdir': [self.task_config.DATA_OBS]}).sync() self.task_config.prepaero_yaml = [] for sensor in self.task_config.sensors: - # print(sensor,'file_list',file_list) raw_files = self.list_raw_files(sensor) - print('raw_files',raw_files) self.task_config.input_files = self.copy_obs(raw_files) - print('raw files in obs_chem',self.task_config.input_files) self.link_obsconvexe() self.task_config.prepaero_config = self.get_obsproc_config(sensor) - # generate converter YAML file - _prepaero_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml") - self.task_config.prepaero_yaml.append( _prepaero_yaml) + # generate converter YAML file + template = f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml" + _prepaero_yaml = os.path.join(self.runtime_config.DATA, template) + self.task_config.prepaero_yaml.append(_prepaero_yaml) logger.debug(f"Generate PrepAeroObs YAML file: {_prepaero_yaml}") save_as_yaml(self.task_config.prepaero_config, _prepaero_yaml) logger.info(f"Wrote PrepAeroObs YAML to: {_prepaero_yaml}") - - @logit(logger) - def list_raw_files(self,sensor) -> List[str]: + def list_raw_files(self, sensor) -> List[str]: """ List all files in the predefined directory that match the predefined sensor and within the time window. """ - if sensor == 'n20': - sensor = 'j01' + if sensor == 'n20': + sensor = 'j01' dir1 = f"{self.task_config.data_dir}/{datetime_to_YMD(self.task_config.window_begin)}" dir2 = f"{self.task_config.data_dir}/{datetime_to_YMD(self.task_config.window_end)}" @@ -99,8 +94,7 @@ def list_raw_files(self,sensor) -> List[str]: allfiles_1 = [os.path.join(dir1, file) for file in files_1] files_2 = os.listdir(dir2) allfiles_2 = [os.path.join(dir2, file) for file in files_2] - allfiles = sorted(allfiles_1,allfiles_2) - + allfiles = sorted(allfiles_1, allfiles_2) matching_files = [] try: for file in allfiles: @@ -110,26 +104,22 @@ def list_raw_files(self,sensor) -> List[str]: dd = fshort[0][24:26] HH = fshort[0][26:28] MM = fshort[0][28:30] - fstart = to_datetime(yyyy+'-'+mm+'-'+dd+'T'+HH+':'+MM+'Z') - + fstart = to_datetime(yyyy + '-' + mm + '-' + dd + 'T' + HH + ':' + MM + 'Z') if sensor in file: - # temporally select obs files based on time stamp inthe filename. - if (fstart > self.task_config.window_begin) and (fstart < self.task_config.window_end): - # print('time window',sensor,fstart, self.task_config.window_begin, self.task_config.window_end) - matching_files.append(os.path.join(self.task_config.data_dir, file)) + # temporally select obs files based on time stamp inthe filename. + if (fstart > self.task_config.window_begin) and (fstart < self.task_config.window_end): + matching_files.append(os.path.join(self.task_config.data_dir, file)) logger.info("Found %d matching files.", len(matching_files)) except FileNotFoundError: logger.error("The specified file/directory does not exist.") raise return matching_files - @logit(logger) - def copy_obs(self,inputfiles) -> Dict[str, Any]: + def copy_obs(self, inputfiles) -> Dict[str, Any]: """ Copy the raw obs files to $DATA/obs. """ - copylist = [] destlist = [] for filename in inputfiles: @@ -141,10 +131,10 @@ def copy_obs(self,inputfiles) -> Dict[str, Any]: return destlist - @logit(logger) - def get_obsproc_config(self,sensor) -> Dict[str, Any]: - """Compile a dictionary of obs proc configuration from OBSPROCYAML template file + def get_obsproc_config(self, sensor) -> Dict[str, Any]: + """ + Compile a dictionary of obs proc configuration from OBSPROCYAML template file Parameters ---------- Returns @@ -152,16 +142,13 @@ def get_obsproc_config(self,sensor) -> Dict[str, Any]: obsproc_config : Dict a dictionary containing the fully rendered obs proc yaml configuration """ - self.task_config.sensor = sensor # generate JEDI YAML file logger.info(f"Generate gdas_obsprovider2ioda YAML config: {self.task_config.OBSPROCYAML}") prepaero_config = parse_j2yaml(self.task_config.OBSPROCYAML, self.task_config) - # logger.debug(f"OBSPROC config:\n{format(obsproc_config)}") return prepaero_config - @logit(logger) def link_obsconvexe(self) -> None: """ @@ -205,7 +192,6 @@ def runConverter(self) -> None: pass - @logit(logger) def finalize(self) -> None: """ @@ -218,7 +204,7 @@ def finalize(self) -> None: copylist = [] for obsfile in obsfiles: basename = os.path.basename(obsfile) - src = os.path.join(self.task_config['DATA'],basename) + src = os.path.join(self.task_config['DATA'], basename) dest = os.path.join(self.task_config.COM_OBS, basename) copylist.append([src, dest]) FileHandler({'copy': copylist}).sync() @@ -234,8 +220,6 @@ def finalize(self) -> None: for obsfile in obsfiles: aeroobsgzip = f"{obsfile}.gz" archive.add(aeroobsgzip, arcname=os.path.basename(aeroobsgzip)) - - # get list of raw viirs L2 files rawfiles = glob.glob(os.path.join(self.task_config.DATA_OBS, 'JRR-AOD*')) # gzip the raw L2 files first @@ -249,17 +233,11 @@ def finalize(self) -> None: for rawfile in rawfiles: aerorawobsgzip = f"{rawfile}.gz" archive.add(aerorawobsgzip, arcname=os.path.basename(aerorawobsgzip)) - copylist = [] for prepaero_yaml in self.task_config.prepaero_yaml: basename = os.path.basename(prepaero_yaml) - dest = os.path.join(self.task_config.COM_OBS,basename) - copylist.append([prepaero_yaml,dest]) + dest = os.path.join(self.task_config.COM_OBS, basename) + copylist.append([prepaero_yaml, dest]) FileHandler({'copy': copylist}).sync() pass - - - - - diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index dc789a1bec4..27d3c0dae4f 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -109,7 +109,7 @@ def _get_app_configs(self): if self.do_aero: configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] if self.do_prep_aero_obs: - configs += ['prepaeroobs'] + configs += ['prepaeroobs'] if self.do_jedisnowda: configs += ['prepsnowobs', 'snowanl'] @@ -157,7 +157,6 @@ def get_task_names(self): if self.do_prep_aero_obs: gdas_gfs_common_tasks_before_fcst += ['prepaeroobs'] - if self.do_jedisnowda: gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl'] From 18f11fb7dac605ddb5a0b2ac04c115de91d8abb2 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 30 May 2024 13:07:32 +0000 Subject: [PATCH 10/26] update env --- env/HERCULES.env | 7 +++++++ env/JET.env | 7 +++++++ env/ORION.env | 7 +++++++ env/S4.env | 7 +++++++ env/WCOSS2.env | 7 +++++++ 5 files changed, 35 insertions(+) diff --git a/env/HERCULES.env b/env/HERCULES.env index 0824ba913a6..6067bb3de30 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -99,6 +99,13 @@ case ${step} in [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" ;; + + "prepaeroobs") + nth_max=$((npe_node_max / npe_node_prepaeroobs)) + + export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} + export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + "snowanl") nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/JET.env b/env/JET.env index 5bd88dc93a2..900ab9920b0 100755 --- a/env/JET.env +++ b/env/JET.env @@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "prepaeroobs" ]]; then + + nth_max=$((npe_node_max / npe_node_prepaeroobs)) + + export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} + export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/ORION.env b/env/ORION.env index f701e55aa2e..9da4545cc22 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -90,6 +90,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "prepaeroobs" ]]; then + + nth_max=$((npe_node_max / npe_node_prepaeroobs)) + + export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} + export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/S4.env b/env/S4.env index 9ba3a61b01c..8704bcec0e5 100755 --- a/env/S4.env +++ b/env/S4.env @@ -82,6 +82,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "prepaeroobs" ]]; then + + nth_max=$((npe_node_max / npe_node_prepaeroobs)) + + export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} + export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 0876e4127dc..2e2daaf5409 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -76,6 +76,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "prepaeroobs" ]]; then + + nth_max=$((npe_node_max / npe_node_prepaeroobs)) + + export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} + export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) From 7a631d94016ad1ddb650f8d46543e14dc4b8ce2d Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 30 May 2024 20:00:13 +0000 Subject: [PATCH 11/26] fix shellcheck --- env/HERCULES.env | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/env/HERCULES.env b/env/HERCULES.env index 6067bb3de30..5edc99a9317 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -99,13 +99,12 @@ case ${step} in [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" ;; - "prepaeroobs") nth_max=$((npe_node_max / npe_node_prepaeroobs)) export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" - +;; "snowanl") nth_max=$((npe_node_max / npe_node_snowanl)) From 8f78765929ea51bfabf90c80d59c6eeb162fa027 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 30 May 2024 20:21:45 +0000 Subject: [PATCH 12/26] archive obs file --- parm/archive/arcdir.yaml.j2 | 4 ++++ parm/archive/gdas.yaml.j2 | 4 ++++ parm/archive/gfsa.yaml.j2 | 4 ++++ parm/config/gfs/config.prepaeroobs | 2 +- parm/config/gfs/config.resources | 2 +- scripts/exglobal_archive.py | 4 ++-- workflow/applications/gfs_cycled.py | 9 ++++----- 7 files changed, 20 insertions(+), 9 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 6321f6fc416..132b309f3e9 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -26,6 +26,10 @@ deterministic: {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %} - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} + {% if DO_PREP_AERO_OBS %} + - ["{{ COM_OBS }}/{{ head }}aeroobs", "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_OBS }}/{{ head }}aerorawobs", "{{ ARCDIR }}/aerorawobs.{{ RUN }}.{{ cycle_YMDH }}"] + {% endif %} - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"] {% endif %} # Full cycle {% if RUN == "gfs" %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 3c7709cfac6..fce2daf904d 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -76,6 +76,10 @@ gdas: {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} + {% if DO_PREP_AERO_OBS %} + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + {% endif %} {% if DO_JEDISNOWDA %} - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 7ed12819a00..4ee3238fc7d 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -32,6 +32,10 @@ gfsa: {% if AERO_ANL_CDUMP == "gfs" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} + {% if DO_PREP_AERO_OBS %} + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + {% endif %} {% if DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" diff --git a/parm/config/gfs/config.prepaeroobs b/parm/config/gfs/config.prepaeroobs index be710faf6cc..9cf7efe3192 100644 --- a/parm/config/gfs/config.prepaeroobs +++ b/parm/config/gfs/config.prepaeroobs @@ -8,7 +8,7 @@ echo "BEGIN: config.prepaeroobs" # Get task specific resources source "${EXPDIR}/config.resources" prepaeroobs -export OBSPROCYAML="${PARMgfs}/gdas/aero/obs/list/aero_obsproc.yaml.j2" +export OBSPROCYAML="${PARMgfs}/gdas/aero/obs/lists/gdas_aero_obsproc.yaml.j2" export OBSPROCEXE="${EXECgfs}/gdas_obsprovider2ioda.x" export VIIRS_DATA_DIR="/scratch2/NCEPDEV/stmp3/Yaping.Wang/VIIRS/AWS/" export SENSORS="npp,n20" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 0fda51ef7ab..7de71d794eb 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -290,7 +290,7 @@ case ${step} in ;; "prepaeroobs") - export wtime_prepaeroobs="00:15:00" + export wtime_prepaeroobs="00:30:00" export npe_prepaeroobs=1 export nth_prepaeroobs=1 export npe_node_prepaeroobs=1 diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 31b5eb1186b..6224e95a3cc 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -19,8 +19,8 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', - 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'PARMgfs', - 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO','DO_PREP_AERO_OBS', + 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index ad8ba8ec860..b1731df7a59 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -152,11 +152,6 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] - if self.do_aero: - gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] - if self.do_prep_aero_obs: - gdas_gfs_common_tasks_before_fcst += ['prepaeroobs'] - if self.do_jedisnowda: gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl'] @@ -185,6 +180,8 @@ def get_task_names(self): if self.do_aero and 'gdas' in self.aero_anl_cdumps: gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + if self.do_prep_aero_obs: + gdas_tasks += ['prepaeroobs'] gdas_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] @@ -222,6 +219,8 @@ def get_task_names(self): if self.do_aero and 'gfs' in self.aero_anl_cdumps: gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + if self.do_prep_aero_obs: + gfs_tasks += ['prepaeroobs'] gfs_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] From 25616c4b3b8129fd8d25aa957f59aedc58f6071d Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 30 May 2024 20:26:23 +0000 Subject: [PATCH 13/26] fix coding norms check --- scripts/exglobal_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index 6224e95a3cc..cc95e7fa86f 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -19,7 +19,7 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', - 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO','DO_PREP_AERO_OBS', + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'DO_PREP_AERO_OBS', 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', From 8bc479a4accf31648e57643d5a4d8b0533b8fb7e Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 30 May 2024 21:23:50 +0000 Subject: [PATCH 14/26] revert --- parm/config/gfs/config.aeroanl | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 37c8959e8aa..24a5e926448 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -20,7 +20,6 @@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" - if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" export JEDIYAML="${PARMgfs}/gdas/aero/variational/3dvar_fgat_gfs_aero.yaml.j2" From 03324f541a77eb1dd0dbbf9c0023d36bee80f234 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Fri, 31 May 2024 21:38:57 +0000 Subject: [PATCH 15/26] update jobs name to prepobsaero --- env/HERA.env | 8 ++--- env/HERCULES.env | 8 ++--- env/JET.env | 8 ++--- env/ORION.env | 8 ++--- env/S4.env | 8 ++--- env/WCOSS2.env | 6 ++-- ...AL_PREP_AERO_OBS => JGLOBAL_PREP_OBS_AERO} | 28 ++++++--------- .../rocoto/{prepaeroobs.sh => prepobsaero.sh} | 4 +-- parm/config/gfs/config.base | 2 +- ...{config.prepaeroobs => config.prepobsaero} | 8 ++--- parm/config/gfs/config.resources | 14 ++++---- scripts/exglobal_archive.py | 2 +- ..._aero_obs.py => exglobal_prep_obs_aero.py} | 3 +- ush/python/pygfs/task/aero_prepobs.py | 36 ++++++++----------- workflow/applications/applications.py | 2 +- workflow/applications/gfs_cycled.py | 12 +++---- workflow/rocoto/gfs_tasks.py | 12 +++---- 17 files changed, 78 insertions(+), 91 deletions(-) rename jobs/{JGLOBAL_PREP_AERO_OBS => JGLOBAL_PREP_OBS_AERO} (53%) rename jobs/rocoto/{prepaeroobs.sh => prepobsaero.sh} (90%) rename parm/config/gfs/{config.prepaeroobs => config.prepobsaero} (64%) rename scripts/{exglobal_prep_aero_obs.py => exglobal_prep_obs_aero.py} (88%) diff --git a/env/HERA.env b/env/HERA.env index 144b8a3f4c5..810945a7e6d 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -102,12 +102,12 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" -elif [[ "${step}" = "prepaeroobs" ]]; then +elif [[ "${step}" = "prepobsaero" ]]; then - nth_max=$((npe_node_max / npe_node_prepaeroobs)) + nth_max=$((npe_node_max / npe_node_prepobsaero)) - export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} - export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index 5edc99a9317..bd285376a72 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -99,11 +99,11 @@ case ${step} in [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" ;; - "prepaeroobs") - nth_max=$((npe_node_max / npe_node_prepaeroobs)) + "prepobsaero") + nth_max=$((npe_node_max / npe_node_prepobsaero)) - export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} - export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" ;; "snowanl") diff --git a/env/JET.env b/env/JET.env index 900ab9920b0..48772397796 100755 --- a/env/JET.env +++ b/env/JET.env @@ -82,12 +82,12 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" -elif [[ "${step}" = "prepaeroobs" ]]; then +elif [[ "${step}" = "prepobsaero" ]]; then - nth_max=$((npe_node_max / npe_node_prepaeroobs)) + nth_max=$((npe_node_max / npe_node_prepobsaero)) - export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} - export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 9da4545cc22..91de32a361e 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -90,12 +90,12 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" -elif [[ "${step}" = "prepaeroobs" ]]; then +elif [[ "${step}" = "prepobsaero" ]]; then - nth_max=$((npe_node_max / npe_node_prepaeroobs)) + nth_max=$((npe_node_max / npe_node_prepobsaero)) - export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} - export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/S4.env b/env/S4.env index 8704bcec0e5..5a6182d07d4 100755 --- a/env/S4.env +++ b/env/S4.env @@ -82,12 +82,12 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" -elif [[ "${step}" = "prepaeroobs" ]]; then +elif [[ "${step}" = "prepobsaero" ]]; then - nth_max=$((npe_node_max / npe_node_prepaeroobs)) + nth_max=$((npe_node_max / npe_node_prepobsaero)) - export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} - export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 2e2daaf5409..e0cd6443bf3 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -76,12 +76,12 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" -elif [[ "${step}" = "prepaeroobs" ]]; then +elif [[ "${step}" = "prepobsaero" ]]; then nth_max=$((npe_node_max / npe_node_prepaeroobs)) - export NTHREADS_PREPAEROOBS=${nth_prepaeroobs:-1} - export APRUN_PREPAEROOBS="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPAEROOBS}" + export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} + export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/jobs/JGLOBAL_PREP_AERO_OBS b/jobs/JGLOBAL_PREP_OBS_AERO similarity index 53% rename from jobs/JGLOBAL_PREP_AERO_OBS rename to jobs/JGLOBAL_PREP_OBS_AERO index 5c5a6c168c8..08eac8f1413 100755 --- a/jobs/JGLOBAL_PREP_AERO_OBS +++ b/jobs/JGLOBAL_PREP_OBS_AERO @@ -1,40 +1,28 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "prepaeroobs" -c "base aeroanl prepaeroobs" +source "${HOMEgfs}/ush/jjob_header.sh" -e "prepobsaero" -c "base prepobsaero" ############################################## # Set variables used in the script ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" +export COMIN_OBS="${DATA}" +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_OBS:COM_OBS_TMPL ############################################## # Begin JOB SPECIFIC work ############################################## -# Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS - -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" - ############################################################### # Run relevant script -EXSCRIPT=${GDASPREPAEROOBSPY:-${SCRgfs}/exglobal_prep_aero_obs.py} +EXSCRIPT=${GDASPREPAEROOBSPY:-${SCRgfs}/exglobal_prep_obs_aero.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" + ############################################## # End JOB SPECIFIC work ############################################## @@ -46,4 +34,10 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi +########################################## +# Remove the Temporary working directory +########################################## +cd ${DATAROOT} +[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} + exit 0 diff --git a/jobs/rocoto/prepaeroobs.sh b/jobs/rocoto/prepobsaero.sh similarity index 90% rename from jobs/rocoto/prepaeroobs.sh rename to jobs/rocoto/prepobsaero.sh index 4552bf3c724..89da7547e8d 100755 --- a/jobs/rocoto/prepaeroobs.sh +++ b/jobs/rocoto/prepobsaero.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="prepaeroobs" +export job="prepobsaero" export jobid="${job}.$$" ############################################################### @@ -19,6 +19,6 @@ export PYTHONPATH ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_PREP_AERO_OBS" +"${HOMEgfs}/jobs/JGLOBAL_PREP_OBS_AERO" status=$? exit "${status}" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 3ded0a541c1..b675784fbbb 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -178,7 +178,7 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" -export DO_PREP_AERO_OBS="YES" +export DO_PREP_OBS_AERO="YES" export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both diff --git a/parm/config/gfs/config.prepaeroobs b/parm/config/gfs/config.prepobsaero similarity index 64% rename from parm/config/gfs/config.prepaeroobs rename to parm/config/gfs/config.prepobsaero index 9cf7efe3192..f70138991c8 100644 --- a/parm/config/gfs/config.prepaeroobs +++ b/parm/config/gfs/config.prepobsaero @@ -1,12 +1,12 @@ #!/bin/bash -x -########## config.prepaeroobs ########## -# Aerosol Variance specific +########## config.prepobsaero ########## +# Prepare and thin/superob aerosol observations -echo "BEGIN: config.prepaeroobs" +echo "BEGIN: config.prepobsaero" # Get task specific resources -source "${EXPDIR}/config.resources" prepaeroobs +source "${EXPDIR}/config.resources" prepobsaero export OBSPROCYAML="${PARMgfs}/gdas/aero/obs/lists/gdas_aero_obsproc.yaml.j2" export OBSPROCEXE="${EXECgfs}/gdas_obsprovider2ioda.x" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 7de71d794eb..c71ccce9218 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -13,7 +13,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl" - echo "prepaeroobs aeroanlinit aeroanlrun aeroanlfinal" + echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -289,12 +289,12 @@ case ${step} in export npe_node_snowanl=$(( npe_node_max / nth_snowanl )) ;; - "prepaeroobs") - export wtime_prepaeroobs="00:30:00" - export npe_prepaeroobs=1 - export nth_prepaeroobs=1 - export npe_node_prepaeroobs=1 - export memory_prepaeroobs="96GB" + "prepobsaero") + export wtime_prepobsaero="00:30:00" + export npe_prepobsaero=1 + export nth_prepobsaero=1 + export npe_node_prepobsaero=1 + export memory_prepobsaero="96GB" ;; "aeroanlinit") diff --git a/scripts/exglobal_archive.py b/scripts/exglobal_archive.py index cc95e7fa86f..5fb71d76177 100755 --- a/scripts/exglobal_archive.py +++ b/scripts/exglobal_archive.py @@ -19,7 +19,7 @@ def main(): # Pull out all the configuration keys needed to run the rest of archive steps keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', - 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'DO_PREP_AERO_OBS', + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_AERO', 'DO_PREP_OBS_AERO', 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', 'HPSSARCH', 'DO_MOS', 'DO_JEDISNOWDA', 'LOCALARCH', 'REALTIME', 'ROTDIR', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', diff --git a/scripts/exglobal_prep_aero_obs.py b/scripts/exglobal_prep_obs_aero.py similarity index 88% rename from scripts/exglobal_prep_aero_obs.py rename to scripts/exglobal_prep_obs_aero.py index 0f5229f78c1..08548e68749 100755 --- a/scripts/exglobal_prep_aero_obs.py +++ b/scripts/exglobal_prep_obs_aero.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# exglobal_prep_aero_obs.py +# exglobal_prep_obs_aero.py # This script collect available viirs # obs files, combine and preprocess # them. @@ -17,7 +17,6 @@ # Take configuration from environment and cast it as python dictionary config = cast_strdict_as_dtypedict(os.environ) - # Instantiate the aerosol prep obs tasks AeroObs = AerosolObsPrep(config) AeroObs.initialize() AeroObs.runConverter() diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index bf7c5f7e34d..229d6a94e12 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -4,6 +4,7 @@ import glob import gzip import tarfile +import re from logging import getLogger from typing import List, Dict, Any, Union @@ -47,14 +48,9 @@ def initialize(self) -> None: List needed raw obs files. Copy the raw obs files to $DATA/obs. Link over the needed executable. - Generate corrosponding YMAL file. + Generate corresponding YMAL file. Run IODA converter. """ - self.task_config.COM_OBS_CHEM = os.path.join(self.task_config.COM_OBS, 'chem') - if os.path.exists(self.task_config.COM_OBS_CHEM): - rmdir(self.task_config.COM_OBS_CHEM) - FileHandler({'mkdir': [self.task_config.COM_OBS_CHEM]}).sync() - self.task_config.DATA_OBS = os.path.join(self.task_config.DATA, 'obs') if os.path.exists(self.task_config.DATA_OBS): rmdir(self.task_config.DATA_OBS) @@ -82,8 +78,8 @@ def list_raw_files(self, sensor) -> List[str]: """ if sensor == 'n20': sensor = 'j01' - dir1 = f"{self.task_config.data_dir}/{datetime_to_YMD(self.task_config.window_begin)}" - dir2 = f"{self.task_config.data_dir}/{datetime_to_YMD(self.task_config.window_end)}" + dir1 = os.path.join(self.task_config.data_dir, datetime_to_YMD(self.task_config.window_begin)) + dir2 = os.path.join(self.task_config.data_dir, datetime_to_YMD(self.task_config.window_end)) if dir1 == dir2: files = os.listdir(dir1) @@ -98,13 +94,11 @@ def list_raw_files(self, sensor) -> List[str]: matching_files = [] try: for file in allfiles: - fshort = file.split('/')[-1].split('.') - yyyy = fshort[0][18:22] - mm = fshort[0][22:24] - dd = fshort[0][24:26] - HH = fshort[0][26:28] - MM = fshort[0][28:30] - fstart = to_datetime(yyyy + '-' + mm + '-' + dd + 'T' + HH + ':' + MM + 'Z') + fshort = file.split('/')[-1].split('.')[0].split('_')[3] + pattern = r"s(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{3})" + match = re.match(pattern, fshort) + yyyy, mm, dd, HH, MM = match.group(1), match.group(2), match.group(3), match.group(4), match.group(5) + fstart = to_datetime(f'{yyyy}-{mm}-{dd}T{HH}:{MM}Z') if sensor in file: # temporally select obs files based on time stamp inthe filename. if (fstart > self.task_config.window_begin) and (fstart < self.task_config.window_end): @@ -177,7 +171,7 @@ def runConverter(self) -> None: """ chdir(self.task_config.DATA) for prepaero_yaml in self.task_config.prepaero_yaml: - exec_cmd = Executable(self.task_config.APRUN_PREPAEROOBS) + exec_cmd = Executable(self.task_config.APRUN_PREPOBSAERO) exec_name = os.path.join(self.task_config.DATA, 'gdas_obsprovider2ioda.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg(prepaero_yaml) @@ -195,7 +189,7 @@ def runConverter(self) -> None: @logit(logger) def finalize(self) -> None: """ - Copy the output viirs files to COM_OBS. + Copy the output viirs files to COMIN_OBS. Tar and archive the output files. Tar and archive the raw obs files. """ @@ -205,7 +199,7 @@ def finalize(self) -> None: for obsfile in obsfiles: basename = os.path.basename(obsfile) src = os.path.join(self.task_config['DATA'], basename) - dest = os.path.join(self.task_config.COM_OBS, basename) + dest = os.path.join(self.task_config.COMOUT_OBS, basename) copylist.append([src, dest]) FileHandler({'copy': copylist}).sync() @@ -214,7 +208,7 @@ def finalize(self) -> None: with open(obsfile, 'rb') as f_in, gzip.open(f"{obsfile}.gz", 'wb') as f_out: f_out.writelines(f_in) - aeroobs = os.path.join(self.task_config.COM_OBS, f"{self.task_config['APREFIX']}aeroobs") + aeroobs = os.path.join(self.task_config.COMOUT_OBS, f"{self.task_config['APREFIX']}aeroobs") # open tar file for writing with tarfile.open(aeroobs, "w") as archive: for obsfile in obsfiles: @@ -227,7 +221,7 @@ def finalize(self) -> None: with open(rawfile, 'rb') as f_in, gzip.open(f"{rawfile}.gz", 'wb') as f_out: f_out.writelines(f_in) - aerorawobs = os.path.join(self.task_config.COM_OBS_CHEM, f"{self.task_config['APREFIX']}aerorawobs") + aerorawobs = os.path.join(self.task_config.COMOUT_OBS, f"{self.task_config['APREFIX']}aerorawobs") # open tar file for writing with tarfile.open(aerorawobs, "w") as archive: for rawfile in rawfiles: @@ -236,7 +230,7 @@ def finalize(self) -> None: copylist = [] for prepaero_yaml in self.task_config.prepaero_yaml: basename = os.path.basename(prepaero_yaml) - dest = os.path.join(self.task_config.COM_OBS, basename) + dest = os.path.join(self.task_config.COMOUT_OBS, basename) copylist.append([prepaero_yaml, dest]) FileHandler({'copy': copylist}).sync() diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 7e337b00c57..6a4d240fe5f 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -51,7 +51,7 @@ def __init__(self, conf: Configuration) -> None: self.do_ocean = _base.get('DO_OCN', False) self.do_ice = _base.get('DO_ICE', False) self.do_aero = _base.get('DO_AERO', False) - self.do_prep_aero_obs = _base.get('DO_PREP_AERO_OBS', False) + self.do_prep_obs_aero = _base.get('DO_PREP_OBS_AERO', False) self.do_bufrsnd = _base.get('DO_BUFRSND', False) self.do_gempak = _base.get('DO_GEMPAK', False) self.do_awips = _base.get('DO_AWIPS', False) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index b1731df7a59..175ddb07bfa 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -108,8 +108,8 @@ def _get_app_configs(self): if self.do_aero: configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] - if self.do_prep_aero_obs: - configs += ['prepaeroobs'] + if self.do_prep_obs_aero: + configs += ['prepobsaero'] if self.do_jedisnowda: configs += ['prepsnowobs', 'snowanl'] @@ -180,8 +180,8 @@ def get_task_names(self): if self.do_aero and 'gdas' in self.aero_anl_cdumps: gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] - if self.do_prep_aero_obs: - gdas_tasks += ['prepaeroobs'] + if self.do_prep_obs_aero: + gdas_tasks += ['prepobsaero'] gdas_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] @@ -219,8 +219,8 @@ def get_task_names(self): if self.do_aero and 'gfs' in self.aero_anl_cdumps: gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] - if self.do_prep_aero_obs: - gfs_tasks += ['prepaeroobs'] + if self.do_prep_obs_aero: + gfs_tasks += ['prepobsaero'] gfs_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 248aa065a4c..8635703ea6a 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -483,20 +483,20 @@ def atmanlfinal(self): return task - def prepaeroobs(self): + def prepobsaero(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('prepaeroobs') - task_name = f'{self.cdump}prepaeroobs' + resources = self.get_resource('prepobsaero') + task_name = f'{self.cdump}prepobsaero' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/prepaeroobs.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/prepobsaero.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -510,8 +510,8 @@ def aeroanlinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} - if self.app_config.do_prep_aero_obs: - dep_dict = {'type': 'task', 'name': f'{self.cdump}prepaeroobs'} + if self.app_config.do_prep_obs_aero: + dep_dict = {'type': 'task', 'name': f'{self.cdump}prepobsaero'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 3db2938682e05a666983b2f310940d60c3a895a9 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Mon, 3 Jun 2024 15:14:43 +0000 Subject: [PATCH 16/26] fix shellcheck --- jobs/JGLOBAL_PREP_OBS_AERO | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_PREP_OBS_AERO b/jobs/JGLOBAL_PREP_OBS_AERO index 08eac8f1413..30a5770fefd 100755 --- a/jobs/JGLOBAL_PREP_OBS_AERO +++ b/jobs/JGLOBAL_PREP_OBS_AERO @@ -37,7 +37,7 @@ fi ########################################## # Remove the Temporary working directory ########################################## -cd ${DATAROOT} -[[ ${KEEPDATA} = "NO" ]] && rm -rf ${DATA} +cd "${DATAROOT}" || exit +[[ "${KEEPDATA}" = "NO" ]] && rm -rf ${DATA} exit 0 From 8fa0ecd9f47a0757380586c0472e31ddccae5c59 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Wed, 5 Jun 2024 19:44:06 +0000 Subject: [PATCH 17/26] specify location of sensor in filename --- ush/python/pygfs/task/aero_prepobs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index 229d6a94e12..d361342b772 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -94,13 +94,13 @@ def list_raw_files(self, sensor) -> List[str]: matching_files = [] try: for file in allfiles: - fshort = file.split('/')[-1].split('.')[0].split('_')[3] + basename = os.path.basename(file) pattern = r"s(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{3})" - match = re.match(pattern, fshort) + match = re.match(pattern, basename.split('_')[3]) yyyy, mm, dd, HH, MM = match.group(1), match.group(2), match.group(3), match.group(4), match.group(5) fstart = to_datetime(f'{yyyy}-{mm}-{dd}T{HH}:{MM}Z') - if sensor in file: - # temporally select obs files based on time stamp inthe filename. + if sensor == basename.split('_')[2]: + # temporally select obs files based on time stamp in the filename. if (fstart > self.task_config.window_begin) and (fstart < self.task_config.window_end): matching_files.append(os.path.join(self.task_config.data_dir, file)) logger.info("Found %d matching files.", len(matching_files)) From 777437f7b9ace8cfc17248c212d20827cc8658b9 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Wed, 5 Jun 2024 20:39:30 +0000 Subject: [PATCH 18/26] update job name in archive step --- parm/archive/arcdir.yaml.j2 | 2 +- parm/archive/gdas.yaml.j2 | 2 +- parm/archive/gfsa.yaml.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 132b309f3e9..72f05a00af2 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -26,7 +26,7 @@ deterministic: {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %} - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} - {% if DO_PREP_AERO_OBS %} + {% if DO_PREP_OBS_AERO %} - ["{{ COM_OBS }}/{{ head }}aeroobs", "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"] - ["{{ COM_OBS }}/{{ head }}aerorawobs", "{{ ARCDIR }}/aerorawobs.{{ RUN }}.{{ cycle_YMDH }}"] {% endif %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index fce2daf904d..64876e5688e 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -76,7 +76,7 @@ gdas: {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} - {% if DO_PREP_AERO_OBS %} + {% if DO_PREP_OBS_AERO %} - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 4ee3238fc7d..fc00287246c 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -32,7 +32,7 @@ gfsa: {% if AERO_ANL_CDUMP == "gfs" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} - {% if DO_PREP_AERO_OBS %} + {% if DO_PREP_OBS_AERO %} - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" {% endif %} From 43ec97c3ae04c51bf56ab6a754e385329209a45c Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 6 Jun 2024 15:07:26 +0000 Subject: [PATCH 19/26] archive obs file --- parm/archive/arcdir.yaml.j2 | 119 ++++++++++++++++++++++++++++++++++++ parm/archive/gdas.yaml.j2 | 6 ++ parm/archive/gfsa.yaml.j2 | 5 ++ 3 files changed, 130 insertions(+) diff --git a/parm/archive/arcdir.yaml.j2 b/parm/archive/arcdir.yaml.j2 index 8a5b37d0ae9..1ed7422761a 100644 --- a/parm/archive/arcdir.yaml.j2 +++ b/parm/archive/arcdir.yaml.j2 @@ -3,3 +3,122 @@ {% set cycle_YMD = current_cycle | to_YMD %} {% set head = RUN + ".t" + cycle_HH + "z." %} +# Select data to store in the ARCDIR and VFYARC from deterministic runs +Base: &base # GDAS, GFS, ENKFGDAS, or ENKFGFS + common: + mkdir: + - "{{ ARCDIR }}" + +# Common files to be added to both the gfs and gdas keys below +Deterministic: &deterministic + cyclone: + copy: + # Cyclone forecasts, produced for both gdas and gfs cycles + ## Only created if tracking is on and there were systems to track + {% if path_exists(COM_ATMOS_TRACK ~ "/atcfunix." ~ RUN ~ "." ~ cycle_YMDH) %} + - ["{{ COM_ATMOS_TRACK }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunix.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_TRACK }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}", "{{ ARCDIR }}/atcfunixp.{{ RUN }}.{{ cycle_YMDH }}"] + {% endif %} + + # Cyclone tracking data + {% for basin in ["epac", "natl"] %} + {% if path_exists(COM_ATMOS_TRACK + "/" + basin) %} + - ["{{ COM_ATMOS_TRACK }}/{{ basin }}", "{{ ARCDIR }}/{{ basin }}"] + {% endif %} + {% endfor %} + + {% if MODE == "cycled" %} + analysis: + copy: + # Analysis data (if we are running in cycled mode) + - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.anl", "{{ ARCDIR }}/pgbanl.{{ RUN }}.{{ cycle_YMDH }}.grib2"] + + {% if DO_JEDIATMVAR %} + - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}atmstat", "{{ ARCDIR }}/atmstat.{{ RUN }}.{{ cycle_YMDH }}"] + {% else %} + - ["{{ COM_ATMOS_ANALYSIS }}/{{ head }}gsistat", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}"] + {% endif %} + + {% if DO_JEDISNOWDA %} + - ["{{ COM_SNOW_ANALYSIS }}/{{ head }}snowstat.tgz", "{{ ARCDIR }}/snowstat.{{ RUN }}.{{ cycle_YMDH }}.tgz"] + {% endif %} + + {% if AERO_ANL_CDUMP == RUN or AERO_ANL_CDUMP == "both" %} + - ["{{ COM_CHEM_ANALYSIS }}/{{ head }}aerostat", "{{ ARCDIR }}/aerostat.{{ RUN }}.{{ cycle_YMDH }}"] + {% endif %} + + {% if DO_PREP_OBS_AERO %} + - ["{{ COM_OBS }}/{{ head }}aeroobs", "{{ ARCDIR }}/aeroobs.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_OBS }}/{{ head }}aerorawobs", "{{ ARCDIR }}/aerorawobs.{{ RUN }}.{{ cycle_YMDH }}"] + {% endif %} + + {% endif %} + +{% if RUN == "gfs" %} +gfs: # GFS specific + <<: *base + <<: *deterministic + + gfs: + copy: + {% for fhr in range(0, FHMAX_GFS + 1, FHOUT_GFS) %} + - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] + {% endfor %} + + # Cyclone genesis data (only present if there are storms) + {% if path_exists(COM_ATMOS_GENESIS ~ "/storms.gfso.atcf_gen." ~ cycle_YMDH) %} + - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_GENESIS }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}"] + {% endif %} + + {% if path_exists(COM_ATMOS_GENESIS ~ "/trak.gfso.atcfunix." ~ cycle_YMDH) %} + - ["{{ COM_ATMOS_GENESIS }}/trak.gfso.atcfunix.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_GENESIS }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}", "{{ ARCDIR }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}"] + {% endif %} + + {% if DO_FIT2OBS %} + fit2obs: + + mkdir: + {% set VFYARC = ROTDIR + "/vrfyarch" %} + - "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}" + + copy: + {% for fhr in range(0, FHMAX_FITS + 1, 6) %} + {% set sfcfile = "/" + head + "sfcf" + '%03d'|format(fhr) + ".nc" %} + {% set sigfile = "/" + head + "atmf" + '%03d'|format(fhr) + ".nc" %} + - ["{{COM_ATMOS_HISTORY}}/{{ sfcfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sfcfile }}"] + - ["{{COM_ATMOS_HISTORY}}/{{ sigfile }}", "{{ VFYARC }}/{{ RUN }}.{{ cycle_YMD }}/{{ cycle_HH }}/{{ sigfile }}"] + {% endfor %} + + {% endif %} +{% endif %} + +{% if RUN == "gdas" %} +gdas: # GDAS specific + <<: *base + <<: *deterministic + gdas: + copy: + {% for fhr in range(0, FHMAX + 1, FHOUT) %} + - ["{{ COM_ATMOS_GRIB_1p00 }}/{{ head }}pgrb2.1p00.f{{ '%03d' % fhr }}", "{{ ARCDIR }}/pgbf{{ '%02d' % fhr }}.{{ RUN }}.{{ cycle_YMDH }}.grib2"] + {% endfor %} +{% endif %} + +Ensemble: &ensemble # ENKFGDAS or ENKFGFS + analysis: + copy: + # Copy ensemble analyses + {% if DO_JEDIATMENS %} + - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atmensstat", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}atminc.ensmean.nc", "{{ ARCDIR }}/atmensstat.{{ RUN }}.{{ cycle_YMDH }}.ensmean.nc"] + {% else %} + - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}enkfstat", "{{ ARCDIR }}/enkfstat.{{ RUN }}.{{ cycle_YMDH }}"] + - ["{{ COM_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}gsistat.ensmean", "{{ ARCDIR }}/gsistat.{{ RUN }}.{{ cycle_YMDH }}.ensmean"] + {% endif %} + +{% if "enkf" in RUN %} +{{ RUN }}: + <<: *base + <<: *ensemble +{% endif %} diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 17f05506fe9..26540156cd9 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -69,6 +69,10 @@ gdas: {% if AERO_ANL_CDUMP == "gdas" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} + {% if DO_PREP_OBS_AERO %} + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + {% endif %} {% if DO_JEDISNOWDA %} - "{{ COM_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snowstat.tgz" {% endif %} @@ -101,7 +105,9 @@ gdas: - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" - "logs/{{ cycle_YMDH }}/{{ RUN }}vminmon.log" {% endif %} + {% endif %} # End of cycled data + # Forecast and post logs - "logs/{{ cycle_YMDH }}/{{ RUN }}fcst.log" # TODO explicitly name the atmos_prod log files to archive - "logs/{{ cycle_YMDH }}/{{ RUN }}atmos_prod_f*.log" diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index 12ea1b5a7fd..0a8e65d3ef1 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -30,6 +30,7 @@ gfsa: - "{{ COM_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" {% endif %} + # State data {% if DO_JEDIATMVAR %} - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml" - "{{ COM_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat" @@ -39,6 +40,10 @@ gfsa: {% if AERO_ANL_CDUMP == "gfs" or AERO_ANL_CDUMP == "both" %} - "{{ COM_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aerostat" {% endif %} + {% if DO_PREP_OBS_AERO %} + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" + - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + {% endif %} # BUFR inputs - "{{ COM_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" From beaed42c5ff59c21db0e9a0e99aac266cbe84b61 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 6 Jun 2024 15:12:14 +0000 Subject: [PATCH 20/26] update npe --- env/HERA.env | 2 +- env/HERCULES.env | 2 +- env/JET.env | 2 +- env/ORION.env | 2 +- env/S4.env | 2 +- env/WCOSS2.env | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/env/HERA.env b/env/HERA.env index 810945a7e6d..2ac6c6ec1f8 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -107,7 +107,7 @@ elif [[ "${step}" = "prepobsaero" ]]; then nth_max=$((npe_node_max / npe_node_prepobsaero)) export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} - export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index bd285376a72..d43dedad8d8 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -103,7 +103,7 @@ case ${step} in nth_max=$((npe_node_max / npe_node_prepobsaero)) export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} - export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" ;; "snowanl") diff --git a/env/JET.env b/env/JET.env index 48772397796..668ec1c2e4d 100755 --- a/env/JET.env +++ b/env/JET.env @@ -87,7 +87,7 @@ elif [[ "${step}" = "prepobsaero" ]]; then nth_max=$((npe_node_max / npe_node_prepobsaero)) export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} - export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 91de32a361e..afd1cda0521 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -95,7 +95,7 @@ elif [[ "${step}" = "prepobsaero" ]]; then nth_max=$((npe_node_max / npe_node_prepobsaero)) export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} - export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/S4.env b/env/S4.env index 5a6182d07d4..8a368bf1d67 100755 --- a/env/S4.env +++ b/env/S4.env @@ -87,7 +87,7 @@ elif [[ "${step}" = "prepobsaero" ]]; then nth_max=$((npe_node_max / npe_node_prepobsaero)) export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} - export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index e0cd6443bf3..e0d2c8ac6cb 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -81,7 +81,7 @@ elif [[ "${step}" = "prepobsaero" ]]; then nth_max=$((npe_node_max / npe_node_prepaeroobs)) export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} - export APRUN_PREPOBSAERO="${launcher} -n 1 --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then From d0aa0dac12ce5bc989dbb6ada19250ec5a73d2ed Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Thu, 6 Jun 2024 15:21:29 +0000 Subject: [PATCH 21/26] add double quote --- jobs/JGLOBAL_PREP_OBS_AERO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_PREP_OBS_AERO b/jobs/JGLOBAL_PREP_OBS_AERO index 30a5770fefd..7fe701898f2 100755 --- a/jobs/JGLOBAL_PREP_OBS_AERO +++ b/jobs/JGLOBAL_PREP_OBS_AERO @@ -38,6 +38,6 @@ fi # Remove the Temporary working directory ########################################## cd "${DATAROOT}" || exit -[[ "${KEEPDATA}" = "NO" ]] && rm -rf ${DATA} +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" exit 0 From a94115467d6aa51e174d56b822ba700fbfda272d Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 14 Jun 2024 10:35:27 -0400 Subject: [PATCH 22/26] Update ush/python/pygfs/task/aero_prepobs.py Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/python/pygfs/task/aero_prepobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index d361342b772..ffe6d03799b 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -48,7 +48,7 @@ def initialize(self) -> None: List needed raw obs files. Copy the raw obs files to $DATA/obs. Link over the needed executable. - Generate corresponding YMAL file. + Generate corresponding YAML file. Run IODA converter. """ self.task_config.DATA_OBS = os.path.join(self.task_config.DATA, 'obs') From cac1e3dda34225005835cda72a10204e48d59c61 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 14 Jun 2024 10:35:54 -0400 Subject: [PATCH 23/26] Update env/WCOSS2.env Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- env/WCOSS2.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index e0d2c8ac6cb..9fe9179e6b0 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -81,7 +81,7 @@ elif [[ "${step}" = "prepobsaero" ]]; then nth_max=$((npe_node_max / npe_node_prepaeroobs)) export NTHREADS_PREPOBSAERO=${nth_prepobsaero:-1} - export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + export APRUN_PREPOBSAERO="${launcher} -n ${npe_prepobsaero} --ppn ${npe_node_prepobsaero}--cpu-bind depth --depth=${NTHREADS_PREPOBSAERO}" elif [[ "${step}" = "snowanl" ]]; then From 75b6d6be574afcbe115fe64db1696657b48a1600 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 14 Jun 2024 10:36:38 -0400 Subject: [PATCH 24/26] Update ush/python/pygfs/task/aero_prepobs.py Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/python/pygfs/task/aero_prepobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index ffe6d03799b..33773b2b315 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -34,8 +34,8 @@ def __init__(self, config: Dict[str, Any]) -> None: 'sensors': str(self.config['SENSORS']).split(','), 'data_dir': self.config['VIIRS_DATA_DIR'], 'input_files': '', - 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", - 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z." + 'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", + 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z." } ) From 5dd514b1033b8f31c1c6e4672e153da48de29d3d Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 14 Jun 2024 10:37:26 -0400 Subject: [PATCH 25/26] Update ush/python/pygfs/task/aero_prepobs.py Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/python/pygfs/task/aero_prepobs.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index 33773b2b315..f2344241a92 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -170,19 +170,18 @@ def runConverter(self) -> None: Run the IODA converter gdas_obsprovider2ioda.x """ chdir(self.task_config.DATA) - for prepaero_yaml in self.task_config.prepaero_yaml: - exec_cmd = Executable(self.task_config.APRUN_PREPOBSAERO) - exec_name = os.path.join(self.task_config.DATA, 'gdas_obsprovider2ioda.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(prepaero_yaml) + exec_cmd = Executable(self.task_config.APRUN_PREPOBSAERO) + exec_name = os.path.join(self.task_config.DATA, 'gdas_obsprovider2ioda.x') + exec_cmd.add_default_arg(exec_name) + for prepaero_yaml in self.task_config.prepaero_yaml: try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() + logger.debug(f"Executing {exec_cmd} on {prepaero_yaml}") + exec_cmd(f"{prepaero_yaml}") except OSError: - raise OSError(f"Failed to execute {exec_cmd}") + raise OSError(f"Failed to execute {exec_cmd} on {prepaero_yaml}") except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + raise WorkflowException(f"An error occured during execution of {exec_cmd} on {prepaero_yaml}") pass From 8d0ce4298119b42734dd9cb1eb77e39e254df290 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Fri, 14 Jun 2024 10:51:04 -0400 Subject: [PATCH 26/26] Update config.base --- parm/config/gfs/config.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 2e010821586..fd1ffdc415a 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -178,7 +178,7 @@ export DO_WAVE="NO" export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" -export DO_PREP_OBS_AERO="YES" +export DO_PREP_OBS_AERO="NO" export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both