From d3f0b682fa32c695f4526f15918a629c4a24204b Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Feb 2025 14:28:28 -0500 Subject: [PATCH 1/8] update hash of gfs-utils to bring in ocean/ice products in grib2 --- sorc/gfs_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 6274ae7b2eb..c96bbc37ab5 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 6274ae7b2eb6cd3d3ad02b5ad3a16b7d9be1a496 +Subproject commit c96bbc37ab5a1e40293d2886c2b7a8588f3a3d35 From aa025fc24f86c75296d71357235c0701f3b2f883 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 3 Mar 2025 16:55:24 -0500 Subject: [PATCH 2/8] gfs-utils utility ocnicepost.x can also write out grib2 files directly. This is captured here, but needs a little more work in the indexing as well as in copying data back to com --- parm/post/oceanice_products.yaml | 5 +- ush/python/pygfs/task/oceanice_products.py | 79 +++++++++++++++------- 2 files changed, 56 insertions(+), 28 deletions(-) diff --git a/parm/post/oceanice_products.yaml b/parm/post/oceanice_products.yaml index 48e5a5f2045..82fbb09108b 100644 --- a/parm/post/oceanice_products.yaml +++ b/parm/post/oceanice_products.yaml @@ -1,6 +1,8 @@ ocnicepost: executable: "ocnicepost.x" namelist: + write_grib2: True + write_netcdf: False debug: False fix_data: mkdir: @@ -18,9 +20,6 @@ ocnicepost: - ["{{ FIXgfs }}/mom6/post/template.global.{{ grid }}.gb2", "{{ DATA }}/"] {% endfor %} -nc2grib2: - script: "{{ USHgfs }}/oceanice_nc2grib2.sh" - ocean: namelist: ftype: "ocean" diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index 39ec53b100f..67fbdbf5965 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -5,6 +5,7 @@ from typing import List, Dict, Any from pprint import pformat import xarray as xr +from shutil import copyfileobj from wxflow import (AttrDict, parse_j2yaml, @@ -14,7 +15,7 @@ Task, add_to_datetime, to_timedelta, WorkflowException, - Executable) + Executable, which) logger = getLogger(__name__.split('.')[-1]) @@ -171,14 +172,18 @@ def execute(config: Dict, product_grid: str) -> None: # Run the ocnicepost.x executable OceanIceProducts.interp(config.DATA, config.APRUN_OCNICEPOST, exec_name="ocnicepost.x") - # Convert interpolated netCDF file to grib2 - OceanIceProducts.netCDF_to_grib2(config, product_grid) + if config.component in ['ocean']: + # Concatenate the 2D and 3D grib2 files + OceanIceProducts.concatenate(config) + + # Index the interpolated grib2 file + OceanIceProducts.index(config, product_grid) @staticmethod @logit(logger) def interp(workdir: str, aprun_cmd: str, exec_name: str = "ocnicepost.x") -> None: """ - Run the interpolation executable to generate rectilinear netCDF file + Run the interpolation executable to generate interpolated file Parameters ---------- @@ -205,38 +210,62 @@ def interp(workdir: str, aprun_cmd: str, exec_name: str = "ocnicepost.x") -> Non @staticmethod @logit(logger) - def netCDF_to_grib2(config: Dict, grid: str) -> None: - """Convert interpolated netCDF file to grib2 + def index(config: Dict, grid: str) -> None: + """ + Index the grib2 file Parameters ---------- - config : Dict - Configuration dictionary for the task - grid : str - Target product grid to process + workdir : str | os.PathLike + Working directory where to run containing the necessary files and executable + forecast_hour : int + forecast hour to index + + Environment Parameters + ---------------------- + WGRIB2: str (optional) + path to executable "grb2index" + Typically set in the modulefile Returns - ------ + ------- None """ os.chdir(config.DATA) + logger.info("Generate index file") - exec_cmd = Executable(config.oceanice_yaml.nc2grib2.script) - arguments = [config.component, grid, config.current_cycle.strftime("%Y%m%d%H"), config.avg_period] - if config.component == 'ocean': - levs = config.oceanice_yaml.ocean.namelist.ocean_levels - arguments.append(':'.join(map(str, levs))) + wgrib2_cmd = os.environ.get("WGRIB2", None) - logger.info(f"Executing {exec_cmd} with arguments {arguments}") - try: - exec_cmd(*arguments) - except OSError: - logger.exception(f"FATAL ERROR: Failed to execute {exec_cmd}") - raise OSError(f"{exec_cmd}") - except Exception: - logger.exception(f"FATAL ERROR: Error occurred during execution of {exec_cmd}") - raise WorkflowException(f"{exec_cmd}") + grbfile = f"{config.component}.grib2" + grbfidx = f"{grbfile}.grib2.idx" + + if not os.path.exists(grbfile): + logger.info(f"No {grbfile} to process, skipping ...") + return + + logger.info(f"Creating index file for {grbfile}") + exec_cmd = which("wgrib2") if wgrib2_cmd is None else Executable(wgrib2_cmd) + exec_cmd.add_default_arg(os.path.join(config.DATA, grbfile)) + exec_cmd.add_default_arg(os.path.join(config.DATA, grbfidx)) + + OceanIceProducts._call_executable(exec_cmd) + + @staticmethod + @logit(logger) + def concatenate(config: Dict) -> None: + + os.chdir(config.DATA) + outfile = f"{config.component}.grb2" + fout = open(os.path.join(config.DATA, outfile), "wb") + for file in ['2D', '3D']: + infile = f"{config.component}.{file}.grb2" + fin = open(os.path.join(config.DATA, infile), "rb") + copyfileobj(fin, fout) + fin.close() + fout.close() + + return @staticmethod @logit(logger) From d33d138e13521dbc991619ef673337019dc96708 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Mar 2025 09:33:41 -0400 Subject: [PATCH 3/8] update hash of gfs_utils to top of develop --- sorc/gfs_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index c96bbc37ab5..58f79019ddf 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit c96bbc37ab5a1e40293d2886c2b7a8588f3a3d35 +Subproject commit 58f79019ddf2ac1e989fab48fdd69a6a5d784189 From 27ea1bc42eae22092c99fcf10c0f90cf0304366c Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Mar 2025 14:44:23 -0400 Subject: [PATCH 4/8] eliminate duplicates in parm and load modules after input args are met --- parm/post/oceanice_products.yaml | 2 -- test/g2cmp.sh | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/parm/post/oceanice_products.yaml b/parm/post/oceanice_products.yaml index 0e04c28e9b9..82fbb09108b 100644 --- a/parm/post/oceanice_products.yaml +++ b/parm/post/oceanice_products.yaml @@ -4,8 +4,6 @@ ocnicepost: write_grib2: True write_netcdf: False debug: False - write_grib2: False - write_netcdf: True fix_data: mkdir: - "{{ DATA }}" diff --git a/test/g2cmp.sh b/test/g2cmp.sh index c31d10dd623..f4a9e50c238 100755 --- a/test/g2cmp.sh +++ b/test/g2cmp.sh @@ -9,11 +9,11 @@ set -eu HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P) declare -rx HOMEgfs -source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 - file1=${1:?} file2=${2:?} +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 + # Use wgrib2 to compute correlations and print any record that does not have corr=1 for mismatch #shellcheck disable=SC2312 wgrib2 "${file2}" -var -lev -rpn "sto_1" -import_grib "${file1}" -rpn "rcl_1:print_corr:print_rms" | grep -v "rpn_corr=1" From 227dee8226dba7894f806da75e57bd250469bc6b Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Mar 2025 14:46:08 -0400 Subject: [PATCH 5/8] run w/ gfs-utils ocnicepost that creates seperate 2D and 3D ocean files and concatenate into final one --- ush/python/pygfs/task/oceanice_products.py | 79 ++++++++-------------- 1 file changed, 27 insertions(+), 52 deletions(-) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index b32166e8cc5..5c4b775cdf4 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -176,7 +176,7 @@ def execute(config: Dict, product_grid: str) -> None: if config.component in ['ocean']: # Concatenate the 2D and 3D grib2 files - OceanIceProducts.concatenate(config) + OceanIceProducts.concatenate(config, product_grid) # Index the interpolated grib2 file OceanIceProducts.index(config, product_grid) @@ -207,8 +207,11 @@ def interp(workdir: str, aprun_cmd: str, exec_name: str = "ocnicepost.x") -> Non exec_cmd = Executable(aprun_cmd) exec_cmd.add_default_arg(os.path.join(workdir, exec_name)) - - OceanIceProducts._call_executable(exec_cmd) + try: + exec_cmd() + except Exception: + logger.exception(f"FATAL ERROR: Error occurred during execution of {exec_cmd}") + raise WorkflowException(f"{exec_cmd}") @staticmethod @logit(logger) @@ -218,15 +221,15 @@ def index(config: Dict, grid: str) -> None: Parameters ---------- - workdir : str | os.PathLike - Working directory where to run containing the necessary files and executable - forecast_hour : int - forecast hour to index + config : Dict + Configuration dictionary for the task + grid : str + Target product grid to process Environment Parameters ---------------------- WGRIB2: str (optional) - path to executable "grb2index" + path to executable "wgrib2" Typically set in the modulefile Returns @@ -239,33 +242,33 @@ def index(config: Dict, grid: str) -> None: wgrib2_cmd = os.environ.get("WGRIB2", None) - grbfile = f"{config.component}.grib2" - grbfidx = f"{grbfile}.grib2.idx" + grbfile = f"{config.component}.{grid}.grib2" + grbfidx = f"{grbfile}.idx" if not os.path.exists(grbfile): - logger.info(f"No {grbfile} to process, skipping ...") + logger.warning(f"WARNING: No {grbfile} to index!") return logger.info(f"Creating index file for {grbfile}") exec_cmd = which("wgrib2") if wgrib2_cmd is None else Executable(wgrib2_cmd) - exec_cmd.add_default_arg(os.path.join(config.DATA, grbfile)) - exec_cmd.add_default_arg(os.path.join(config.DATA, grbfidx)) - - OceanIceProducts._call_executable(exec_cmd) + exec_cmd.add_default_arg("-s") + try: + exec_cmd(grbfile, output=grbfidx) + except Exception: + logger.exception(f"FATAL ERROR: Error occurred during execution of {exec_cmd}") + raise WorkflowException(f"{exec_cmd}") @staticmethod @logit(logger) - def concatenate(config: Dict) -> None: + def concatenate(config: Dict, grid:str) -> None: os.chdir(config.DATA) - outfile = f"{config.component}.grb2" - fout = open(os.path.join(config.DATA, outfile), "wb") - for file in ['2D', '3D']: - infile = f"{config.component}.{file}.grb2" - fin = open(os.path.join(config.DATA, infile), "rb") - copyfileobj(fin, fout) - fin.close() - fout.close() + with open(os.path.join(config.DATA, f"{config.component}.{grid}.grib2"), "wb") as fout: + for file in ['', '_3D']: + infile = f"{config.component}.{grid}{file}.grb2" + fin = open(os.path.join(config.DATA, infile), "rb") + copyfileobj(fin, fout) + fin.close() return @@ -324,34 +327,6 @@ def subset(config: Dict) -> None: ds.close() ds_subset.close() - @staticmethod - @logit(logger) - def _call_executable(exec_cmd: Executable) -> None: - """Internal method to call executable - - Parameters - ---------- - exec_cmd : Executable - Executable to run - - Raises - ------ - OSError - Failure due to OS issues - WorkflowException - All other exceptions - """ - - logger.info(f"Executing {exec_cmd}") - try: - exec_cmd() - except OSError: - logger.exception(f"FATAL ERROR: Failed to execute {exec_cmd}") - raise OSError(f"{exec_cmd}") - except Exception: - logger.exception(f"FATAL ERROR: Error occurred during execution of {exec_cmd}") - raise WorkflowException(f"{exec_cmd}") - @staticmethod @logit(logger) def finalize(config: Dict) -> None: From 08833481e46ebf106b6239b2b0289a0699963f57 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Mar 2025 15:29:11 -0400 Subject: [PATCH 6/8] update gfs-utils hash --- sorc/gfs_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 58f79019ddf..2bf341a3d76 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 58f79019ddf2ac1e989fab48fdd69a6a5d784189 +Subproject commit 2bf341a3d760d36b86e43187227d34189107e8a5 From be4d29c1a0da315f877ccfd63d99c74cd10fedda Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Mar 2025 15:41:53 -0400 Subject: [PATCH 7/8] update oceanice_products.py to create final grib2 file without the need to concatenate 2D and 3D ocean --- ush/python/pygfs/task/oceanice_products.py | 23 ++-------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index 5c4b775cdf4..d319608ad14 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -2,10 +2,9 @@ import os from logging import getLogger -from typing import List, Dict, Any +from typing import Dict, Any from pprint import pformat import xarray as xr -from shutil import copyfileobj from wxflow import (AttrDict, parse_j2yaml, @@ -174,10 +173,6 @@ def execute(config: Dict, product_grid: str) -> None: # Run the ocnicepost.x executable OceanIceProducts.interp(config.DATA, config.APRUN_OCNICEPOST, exec_name="ocnicepost.x") - if config.component in ['ocean']: - # Concatenate the 2D and 3D grib2 files - OceanIceProducts.concatenate(config, product_grid) - # Index the interpolated grib2 file OceanIceProducts.index(config, product_grid) @@ -258,20 +253,6 @@ def index(config: Dict, grid: str) -> None: logger.exception(f"FATAL ERROR: Error occurred during execution of {exec_cmd}") raise WorkflowException(f"{exec_cmd}") - @staticmethod - @logit(logger) - def concatenate(config: Dict, grid:str) -> None: - - os.chdir(config.DATA) - with open(os.path.join(config.DATA, f"{config.component}.{grid}.grib2"), "wb") as fout: - for file in ['', '_3D']: - infile = f"{config.component}.{grid}{file}.grb2" - fin = open(os.path.join(config.DATA, infile), "rb") - copyfileobj(fin, fout) - fin.close() - - return - @staticmethod @logit(logger) def subset(config: Dict) -> None: @@ -281,7 +262,7 @@ def subset(config: Dict) -> None: Parameters ---------- - config : Dict + config: Dict Configuration dictionary for the task Returns From 8ec272e4cd5e87676128d81f1c581da021762ba4 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Mar 2025 15:58:24 -0400 Subject: [PATCH 8/8] update yaml for gefs oceanice products. remove oceanice_nc2grib2.sh as the program writes out grib2 directly. update CODEOWNERS to reflect the change --- .github/CODEOWNERS | 17 +- parm/post/oceanice_products_gefs.yaml | 13 +- ush/oceanice_nc2grib2.sh | 321 -------------------------- 3 files changed, 14 insertions(+), 337 deletions(-) delete mode 100755 ush/oceanice_nc2grib2.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2922d3da97e..207ba38f757 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -27,11 +27,11 @@ jobs/JGDAS_ATMOS_GEMPAK_META_NCDC @GwenChen-NOAA jobs/JGDAS_ATMOS_VERFOZN @EdwardSafford-NOAA jobs/JGDAS_ATMOS_VERFRAD @EdwardSafford-NOAA jobs/JGDAS_ENKF_* @RussTreadon-NOAA @CoryMartin-NOAA @CatherineThomas-NOAA -jobs/JGDAS_FIT2OBS @jack-woollen +jobs/JGDAS_FIT2OBS @jack-woollen jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN @guillaumevernieres jobs/JGFS_ATMOS_AWIPS_20KM_1P0DEG @GwenChen-NOAA -jobs/JGFS_ATMOS_CYCLONE_GENESIS @JiayiPeng-NOAA -jobs/JGFS_ATMOS_CYCLONE_TRACKER @JiayiPeng-NOAA +jobs/JGFS_ATMOS_CYCLONE_GENESIS @JiayiPeng-NOAA +jobs/JGFS_ATMOS_CYCLONE_TRACKER @JiayiPeng-NOAA jobs/JGFS_ATMOS_FBWIND @GwenChen-NOAA jobs/JGFS_ATMOS_FSU_GENESIS jobs/JGFS_ATMOS_GEMPAK @GwenChen-NOAA @@ -159,7 +159,6 @@ load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA minmon_xtrct_*.pl @EdwardSafford-NOAA module-setup.sh @WalterKolczynski-NOAA @aerorahul -oceanice_nc2grib2.sh @GwenChen-NOAA ocnice_extractvars.sh @EricSinsky-NOAA ozn_xtrct.sh @EdwardSafford-NOAA parse-storm-type.pl @@ -175,10 +174,10 @@ product_functions.sh @WalterKolczynski-NOAA @aerorahul radmon_*.sh @EdwardSafford-NOAA rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA -syndat_getjtbul.sh @JiayiPeng-NOAA -syndat_qctropcy.sh @JiayiPeng-NOAA -tropcy_relocate.sh @JiayiPeng-NOAA -tropcy_relocate_extrkr.sh @JiayiPeng-NOAA +syndat_getjtbul.sh @JiayiPeng-NOAA +syndat_qctropcy.sh @JiayiPeng-NOAA +tropcy_relocate.sh @JiayiPeng-NOAA +tropcy_relocate_extrkr.sh @JiayiPeng-NOAA wave_*.sh @JessicaMeixner-NOAA @sbanihash regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA @@ -189,7 +188,7 @@ ush/python/pygfs/task/__init__.py @aerorahul ush/python/pygfs/task/aero_analysis.py @DavidNew-NOAA @CoryMartin-NOAA ush/python/pygfs/task/aero_bmatrix.py @DavidNew-NOAA @CoryMartin-NOAA ush/python/pygfs/task/aero_emissions.py @bbakernoaa -ush/python/pygfs/task/aero_prepobs.py @CoryMartin-NOAA +ush/python/pygfs/task/aero_prepobs.py @CoryMartin-NOAA ush/python/pygfs/task/analysis.py @DavidNew-NOAA @RussTreadon-NOAA ush/python/pygfs/task/archive.py @DavidHuber-NOAA ush/python/pygfs/task/atm_analysis.py @DavidNew-NOAA @RussTreadon-NOAA diff --git a/parm/post/oceanice_products_gefs.yaml b/parm/post/oceanice_products_gefs.yaml index c228079dcd3..0ef19427abb 100644 --- a/parm/post/oceanice_products_gefs.yaml +++ b/parm/post/oceanice_products_gefs.yaml @@ -1,9 +1,9 @@ ocnicepost: executable: "ocnicepost.x" namelist: + write_grib2: True + write_netcdf: False debug: False - write_grib2: False - write_netcdf: True fix_data: mkdir: - "{{ DATA }}" @@ -20,9 +20,6 @@ ocnicepost: - ["{{ FIXgfs }}/mom6/post/template.global.{{ grid }}.gb2", "{{ DATA }}/"] {% endfor %} -nc2grib2: - script: "{{ USHgfs }}/oceanice_nc2grib2.sh" - ocean: namelist: ftype: "ocean" @@ -49,7 +46,8 @@ ocean: copy: - ["{{ DATA }}/ocean_subset.nc", "{{ COM_OCEAN_NETCDF }}/native/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} - - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COM_OCEAN_NETCDF }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COM_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COM_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] {% endfor %} ice: @@ -73,5 +71,6 @@ ice: copy: - ["{{ DATA }}/ice_subset.nc", "{{ COM_ICE_NETCDF }}/native/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} - - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COM_ICE_NETCDF }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COM_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COM_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] {% endfor %} diff --git a/ush/oceanice_nc2grib2.sh b/ush/oceanice_nc2grib2.sh deleted file mode 100755 index e701edd5db9..00000000000 --- a/ush/oceanice_nc2grib2.sh +++ /dev/null @@ -1,321 +0,0 @@ -#!/bin/bash - -# This script contains functions to convert ocean/ice rectilinear netCDF files to grib2 format -# This script uses the wgrib2 utility to convert the netCDF files to grib2 format and then indexes it - -source "${USHgfs}/preamble.sh" - -################################################################################ -function _ice_nc2grib2 { -# This function converts the ice rectilinear netCDF files to grib2 format - - # Set the inputs - local grid=${1} # 0p25, 0p50, 1p00, 5p00 - local latlon_dims=${2} # 0:721:0:1440, 0:361:0:720, 0:181:0:360, 0:36:0:72 - local current_cycle=${3} # YYYYMMDDHH - local aperiod=${4} # 0-6 - local infile=${5} # ice.0p25.nc - local outfile=${6} # ice.0p25.grib2 - local template=${7} # template.global.0p25.gb2 - - ${WGRIB2} "${template}" \ - -import_netcdf "${infile}" "hi_h" "0:1:${latlon_dims}" \ - -set_var ICETK -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "aice_h" "0:1:${latlon_dims}" \ - -set_var ICEC -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "Tsfc_h" "0:1:${latlon_dims}" \ - -set_var ICETMP -set center 7 -rpn "273.15:+" \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "uvel_h" "0:1:${latlon_dims}" \ - -set_var UICE -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "vvel_h" "0:1:${latlon_dims}" \ - -set_var VICE -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" - -# Additional variables needed for GFSv17/GEFSv13 operational forecast -# files, but GRIB2 parameters not available in NCEP (-set center 7) -# tables in wgrib2 v2.0.8: - -# -import_netcdf "${infile}" "hs_h" "0:1:${latlon_dims}" \ -# -set_var SNVOLSI -set center 7 \ -# -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ -# -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ -# -import_netcdf "${infile}" "frzmlt_h" "0:1:${latlon_dims}" \ -# -set_var FRZMLTPOT -set center 7 \ -# -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ -# -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ -# -import_netcdf "${infile}" "albsni_h" "0:1:${latlon_dims}" \ -# -set_var ALBDOICE -set center 7 -rpn "100.0:/" \ -# -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ -# -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ -# -import_netcdf "${infile}" "mlt_onset_h" "0:1:${latlon_dims}" \ -# -set_var MLTDATE -set center 7 \ -# -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ -# -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ -# -import_netcdf "${infile}" "frz_onset_h" "0:1:${latlon_dims}" \ -# -set_var FRZDATE -set center 7 \ -# -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ -# -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" - - rc=$? - # Check if the conversion was successful - if (( rc != 0 )); then - echo "FATAL ERROR: Failed to convert the ice rectilinear netCDF file to grib2 format" - fi - return "${rc}" - -} - -################################################################################ -function _ocean2D_nc2grib2 { -# This function converts the ocean 2D rectilinear netCDF files to grib2 format - - # Set the inputs - local grid=${1} # 0p25, 0p50, 1p00, 5p00 - local latlon_dims=${2} # 0:721:0:1440, 0:361:0:720, 0:181:0:360, 0:36:0:72 - local current_cycle=${3} # YYYYMMDDHH - local aperiod=${4} # 0-6 - local infile=${5} # ocean.0p25.nc - local outfile=${6} # ocean_2D.0p25.grib2 - local template=${7} # template.global.0p25.gb2 - - ${WGRIB2} "${template}" \ - -import_netcdf "${infile}" "SSH" "0:1:${latlon_dims}" \ - -set_var SSHG -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "SST" "0:1:${latlon_dims}" \ - -set_var WTMP -set center 7 -rpn "273.15:+" \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "SSS" "0:1:${latlon_dims}" \ - -set_var SALIN -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "speed" "0:1:${latlon_dims}" \ - -set_var SPC -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "SSU" "0:1:${latlon_dims}" \ - -set_var UOGRD -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "SSV" "0:1:${latlon_dims}" \ - -set_var VOGRD -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "latent" "0:1:${latlon_dims}" \ - -set_var LHTFL -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "sensible" "0:1:${latlon_dims}" \ - -set_var SHTFL -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "SW" "0:1:${latlon_dims}" \ - -set_var NSWRF -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "LW" "0:1:${latlon_dims}" \ - -set_var NLWRF -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "LwLatSens" "0:1:${latlon_dims}" \ - -set_var THFLX -set center 7 \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ - -import_netcdf "${infile}" "MLD_003" "0:1:${latlon_dims}" \ - -set_var WDEPTH -set center 7 -set_lev "mixed layer depth" \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" - -# Additional variables needed for GFSv17/GEFSv13 operational forecast -# files, but GRIB2 parameters not available in NCEP (-set center 7) -# tables in wgrib2 v2.0.8: -# -# -import_netcdf "${infile}" "Heat_PmE" "0:1:${latlon_dims}" \ -# -set_var DWHFLUX -set center 7 \ -# -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ -# -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ -# -import_netcdf "${infile}" "taux" "0:1:${latlon_dims}" \ -# -set_var XCOMPSS -set center 7 \ -# -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ -# -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" \ -# -import_netcdf "${infile}" "tauy" "0:1:${latlon_dims}" \ -# -set_var YCOMPSS -set center 7 \ -# -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ -# -set_scaling same same -set_grib_type c1 -grib_out "${outfile}" - - rc=$? - # Check if the conversion was successful - if (( rc != 0 )); then - echo "FATAL ERROR: Failed to convert the ocean rectilinear netCDF file to grib2 format" - fi - return "${rc}" - -} - -################################################################################ -function _ocean3D_nc2grib2 { -# This function converts the ocean 3D rectilinear netCDF files to grib2 format - - # Set the inputs - local grid=${1} # 0p25, 0p50, 1p00, 5p00 - local latlon_dims=${2} # 0:721:0:1440, 0:361:0:720, 0:181:0:360, 0:36:0:72 - local levels=${3} # 5:15:25:35:45:55:65:75:85:95:105:115:125 - local current_cycle=${4} # YYYYMMDDHH - local aperiod=${5} # 0-6 - local infile=${6} # ocean.0p25.nc - local outfile=${7} # ocean_3D.0p25.grib2 - local template=${8} # template.global.0p25.gb2 - - IFS=':' read -ra depths <<< "${levels}" - - zl=0 - for depth in "${depths[@]}"; do - - if [[ -f "tmp.gb2" ]]; then - rm -f "tmp.gb2" - fi - - ${WGRIB2} "${template}" \ - -import_netcdf "${infile}" "temp" "0:1:${zl}:1:${latlon_dims}" \ - -set_var WTMP -set center 7 -rpn "273.15:+" \ - -set_lev "${depth} m below sea level" \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out tmp.gb2 \ - -import_netcdf "${infile}" "so" "0:1:${zl}:1:${latlon_dims}" \ - -set_var SALIN -set center 7 \ - -set_lev "${depth} m below sea level" \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out tmp.gb2 \ - -import_netcdf "${infile}" "uo" "0:1:${zl}:1:${latlon_dims}" \ - -set_var UOGRD -set center 7 \ - -set_lev "${depth} m below sea level" \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out tmp.gb2 \ - -import_netcdf "${infile}" "vo" "0:1:${zl}:1:${latlon_dims}" \ - -set_var VOGRD -set center 7 \ - -set_lev "${depth} m below sea level" \ - -set_date "${current_cycle}" -set_ftime "${aperiod} hour ave fcst" \ - -set_scaling same same -set_grib_type c1 -grib_out tmp.gb2 - - rc=$? - # Check if the conversion was successful - if (( rc != 0 )); then - echo "FATAL ERROR: Failed to convert the ocean rectilinear netCDF file to grib2 format at depth ${depth}m, ABORT!" - return "${rc}" - fi - - cat tmp.gb2 >> "${outfile}" - rm -f tmp.gb2 - ((zl = zl + 1)) - - done - - # Notes: - # WATPTEMP (water potential temperature (theta)) may be a better - # GRIB2 parameter than WTMP (water temperature) if MOM6 outputs - # potential temperature. WATPTEMP is not available in NCEP - # (-set center 7) tables in wgrib2 v2.0.8. - - return "${rc}" - -} - -################################################################################ -# Input arguments -component=${1:?"Need a valid component; options: ice|ocean"} -grid=${2:-"0p25"} # Default to 0.25-degree grid -current_cycle=${3:-"2013100100"} # Default to 2013100100 -avg_period=${4:-"0-6"} # Default to 6-hourly average -ocean_levels=${5:-"5:15:25:35:45:55:65:75:85:95:105:115:125"} # Default to 12-levels - -case "${grid}" in - "0p25") - latlon_dims="0:721:0:1440" - ;; - "0p50") - latlon_dims="0:361:0:720" - ;; - "1p00") - latlon_dims="0:181:0:360" - ;; - "5p00") - latlon_dims="0:36:0:72" - ;; - *) - echo "FATAL ERROR: Unsupported grid '${grid}', ABORT!" - exit 1 - ;; -esac - -input_file="${component}.${grid}.nc" -template="template.global.${grid}.gb2" - -# Check if the template file exists -if [[ ! -f "${template}" ]]; then - echo "FATAL ERROR: '${template}' does not exist, ABORT!" - exit 127 -fi - -# Check if the input file exists -if [[ ! -f "${input_file}" ]]; then - echo "FATAL ERROR: '${input_file}' does not exist, ABORT!" - exit 127 -fi - -case "${component}" in - "ice") - rm -f "${component}.${grid}.grib2" || true - _ice_nc2grib2 "${grid}" "${latlon_dims}" "${current_cycle}" "${avg_period}" "${input_file}" "${component}.${grid}.grib2" "${template}" - rc=$? - if (( rc != 0 )); then - echo "FATAL ERROR: Failed to convert the ice rectilinear netCDF file to grib2 format" - exit "${rc}" - fi - ;; - "ocean") - rm -f "${component}_2D.${grid}.grib2" || true - _ocean2D_nc2grib2 "${grid}" "${latlon_dims}" "${current_cycle}" "${avg_period}" "${input_file}" "${component}_2D.${grid}.grib2" "${template}" - rc=$? - if (( rc != 0 )); then - echo "FATAL ERROR: Failed to convert the ocean 2D rectilinear netCDF file to grib2 format" - exit "${rc}" - fi - rm -f "${component}_3D.${grid}.grib2" || true - _ocean3D_nc2grib2 "${grid}" "${latlon_dims}" "${ocean_levels}" "${current_cycle}" "${avg_period}" "${input_file}" "${component}_3D.${grid}.grib2" "${template}" - rc=$? - if (( rc != 0 )); then - echo "FATAL ERROR: Failed to convert the ocean 3D rectilinear netCDF file to grib2 format" - exit "${rc}" - fi - # Combine the 2D and 3D grib2 files into a single file - rm -f "${component}.${grid}.grib2" || true - cat "${component}_2D.${grid}.grib2" "${component}_3D.${grid}.grib2" > "${component}.${grid}.grib2" - - ;; - *) - echo "FATAL ERROR: Unknown component: '${component}'. ABORT!" - exit 3 - ;; -esac - -# Index the output grib2 file -${WGRIB2} -s "${component}.${grid}.grib2" > "${component}.${grid}.grib2.idx" -rc=$? -# Check if the indexing was successful -if (( rc != 0 )); then - echo "FATAL ERROR: Failed to index the file '${component}.${grid}.grib2'" - exit "${rc}" -fi - -exit 0