From 70f6c4039ae215e9b4899ccf47b3f3269472fb3f Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 17 Mar 2022 13:42:52 -0400 Subject: [PATCH 01/13] Save commit of partial work --- ush/jediinc2fv3.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 ush/jediinc2fv3.py diff --git a/ush/jediinc2fv3.py b/ush/jediinc2fv3.py new file mode 100644 index 000000000..18114c4bf --- /dev/null +++ b/ush/jediinc2fv3.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# translate FV3-JEDI increment to FV3 readable format +import argparse +import netCDF4 as nc +import os + +vardict = { + 'ua': 'u_inc', + 'va': 'v_inc', + 'delp': 'delp_inc', + 'DELP': 'delp_inc', + 'delz': 'delz_inc', + 'T': 'T_inc', + 'sphum': 'sphum_inc', + 'liq_wat': 'liq_wat_inc', + 'o3mr': 'o3mr_inc', + 'ice_wat': 'icmr_inc', + 'lat': 'lat', + 'lon': 'lon', +} + + +def jedi_inc_to_fv3(FV3JEDIinc, FV3inc): + assert os.path.exists(FV3JEDIinc), f"File {FV3JEDIinc} not found." + assert os.path.exists(FV3JEDIinc), f"File {FV3JEDIinc} not found." + # open netCDF files + with nc.Dataset(FV3JEDIinc) as ncin, nc.Dataset(FV3inc, "w", format="NETCDF4") as ncout: + # copy over dimensions + for name, dimension in ncin.dimensions.items(): + if name == 'time': + continue + elif name == 'edge': + nameout = 'ilev' + else: + nameout = name + ncout.createDimension(nameout, + (len(dimension) if not dimension.isunlimited() else None)) + # some global attributes + ncout.source = 'jediinc2fv3.py' + ncout.comment = 'Increment produced by FV3-JEDI and modified for use by FV3 read' + # create all the dummy vertical coordinate variables + pfull = range(1,128) + phalf = range(1,129) + levvar = ncout.createVariable('lev', 'f4', ('lev')) + levvar[:] = pfull + pfullvar = ncout.createVariable('pfull', 'f4', ('lev')) + pfullvar[:] = pfull + ilevvar = ncout.createVariable('ilev', 'f4', ('ilev')) + ilevvar[:] = phalf + hyaivar = ncout.createVariable('hyai', 'f4', ('ilev')) + hyaivar[:] = phalf + hybivar = ncout.createVariable('hybi', 'f4', ('ilev')) + hybivar[:] = phalf + # rename and change dimensionality of fields + for name, variable in ncin.variables.items(): + if name in vardict: + print(name, vardict[name], variable.datatype, variable.dimensions) + #x = ncout.createVariable(vardict[name], variable.datatype, dimsout) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('FV3JEDIincrement', type=str, help='Input FV3-JEDI LatLon Increment File') + parser.add_argument('FV3increment', type=str, help='Output FV3 Increment File') + args = parser.parse_args() + jedi_inc_to_fv3(args.FV3JEDIincrement, args.FV3increment) From a768067e658ceeb6e946d3b8c170655f2e93dd22 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 17:59:35 +0000 Subject: [PATCH 02/13] Working converter --- ush/jediinc2fv3.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ush/jediinc2fv3.py b/ush/jediinc2fv3.py index 18114c4bf..3a5facffb 100644 --- a/ush/jediinc2fv3.py +++ b/ush/jediinc2fv3.py @@ -2,6 +2,7 @@ # translate FV3-JEDI increment to FV3 readable format import argparse import netCDF4 as nc +import numpy as np import os vardict = { @@ -39,8 +40,9 @@ def jedi_inc_to_fv3(FV3JEDIinc, FV3inc): ncout.source = 'jediinc2fv3.py' ncout.comment = 'Increment produced by FV3-JEDI and modified for use by FV3 read' # create all the dummy vertical coordinate variables - pfull = range(1,128) - phalf = range(1,129) + nlevs = len(ncin.dimensions['lev']) + pfull = range(1, nlevs+1) + phalf = range(1, nlevs+2) levvar = ncout.createVariable('lev', 'f4', ('lev')) levvar[:] = pfull pfullvar = ncout.createVariable('pfull', 'f4', ('lev')) @@ -53,9 +55,16 @@ def jedi_inc_to_fv3(FV3JEDIinc, FV3inc): hybivar[:] = phalf # rename and change dimensionality of fields for name, variable in ncin.variables.items(): + if len(variable.dimensions) == 4: + dimsout = variable.dimensions[1:] + else: + dimsout = variable.dimensions if name in vardict: - print(name, vardict[name], variable.datatype, variable.dimensions) - #x = ncout.createVariable(vardict[name], variable.datatype, dimsout) + x = ncout.createVariable(vardict[name], 'f4', dimsout) + if len(variable.dimensions) == 4: + ncout[vardict[name]][:] = ncin[name][0,...] + else: + ncout[vardict[name]][:] = ncin[name][:] if __name__ == "__main__": From c75d50dd1627fd8365179936c5f1899dcdac408f Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 18:44:53 +0000 Subject: [PATCH 03/13] Coding norms --- ush/jediinc2fv3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/jediinc2fv3.py b/ush/jediinc2fv3.py index 3a5facffb..c84a37867 100644 --- a/ush/jediinc2fv3.py +++ b/ush/jediinc2fv3.py @@ -62,7 +62,7 @@ def jedi_inc_to_fv3(FV3JEDIinc, FV3inc): if name in vardict: x = ncout.createVariable(vardict[name], 'f4', dimsout) if len(variable.dimensions) == 4: - ncout[vardict[name]][:] = ncin[name][0,...] + ncout[vardict[name]][:] = ncin[name][0, ...] else: ncout[vardict[name]][:] = ncin[name][:] From ed3f13eea39205b530e4482be96def13d21af759 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 19:53:41 +0000 Subject: [PATCH 04/13] Attempt at gathering test data and new unit test --- .github/workflows/unittests.yaml | 2 +- test/CMakeLists.txt | 40 +++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index 6a74e4b4f..1e6932e5a 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -1,5 +1,5 @@ name: Run Unit Tests -on: [push, pull_request] +on: [pull_request] jobs: ctests: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 14fdb5e9c..92b0aa05a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,3 +1,41 @@ -add_test(NAME test_check_yaml_keys +##### get test data from EMC FTP server +# set URL and hash +set(URL "https://ftp.emc.ncep.noaa.gov/static_files/public/GDASApp") +set(SHA "a85becadf39e43fbedb8444cc5a3c4d6331452df") +string(SUBSTRING ${SHA} 0 6 SHORTSHA) +set(TAR "gdasapp-fix-${SHORTSHA}.tgz") +# download test files +file(DOWNLOAD + ${URL}/${TAR} + ${CMAKE_CURRENT_BINARY_DIR}/${TAR} + INACTIVITY_TIMEOUT 30 + TIMEOUT 90 + SHOW_PROGRESS + STATUS status + EXPECTED_HASH SHA256=${SHA} + ) +# Extract downloaded tarball. +file(ARCHIVE_EXTRACT INPUT ${TAR}) + +# list of test files to install +list(APPEND test_data + ${CMAKE_CURRENT_BINARY_DIR}/testdata/atminc_compress.nc4 +) + +# install +install(FILES ${test_data} + DESTINATION "test/testdata") + +##### unit tests +# test for python coding norms +add_test(NAME GDASApp_check_python_norms + COMMAND pycodestyle -v --config ./.pycodestyle . + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) +# test for ush/check_yaml_keys.py +add_test(NAME GDASApp_check_yaml_keys COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/check_yaml_keys.py ${PROJECT_SOURCE_DIR}/test/testinput/check_yaml_keys_ref.yaml ${PROJECT_SOURCE_DIR}/test/testinput/check_yaml_keys_test.yaml WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) +# test for ush/jediinc2fv3.py +add_test(NAME GDASApp_jedi_increment_to_fv3 + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/jediinc2fv3.py ${PROJECT_BINARY_DIR}/test/testdata/atminc_compress.nc4 ${PROJECT_BINARY_DIR}/test/testdata/fv_increment.nc + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) From 313eb870b8d8068b9fd3e548bcf68a7870c0b73e Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 19:58:18 +0000 Subject: [PATCH 05/13] Working ctest --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 92b0aa05a..b8ff8c906 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,7 +1,7 @@ ##### get test data from EMC FTP server # set URL and hash set(URL "https://ftp.emc.ncep.noaa.gov/static_files/public/GDASApp") -set(SHA "a85becadf39e43fbedb8444cc5a3c4d6331452df") +set(SHA "75c4ce84028956c1ce050e07f70abc25c540492c3e1a40f6fa56d35182322e8e") string(SUBSTRING ${SHA} 0 6 SHORTSHA) set(TAR "gdasapp-fix-${SHORTSHA}.tgz") # download test files From 13528e029c989f424224416095d0586169e9544d Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 20:11:01 +0000 Subject: [PATCH 06/13] Add pycodestyle install --- .github/workflows/unittests.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index 1e6932e5a..b89e62b1e 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -8,6 +8,11 @@ jobs: steps: + - name: Install dependencies + run: | + pip install --upgrade pip + pip install pycodestyle + - name: Checkout uses: actions/checkout@v2 with: From 69dda87717ffd51a0432b4c48379414f4eaad336 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 20:13:36 +0000 Subject: [PATCH 07/13] Make ctest verbose in CI --- .github/workflows/unittests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index b89e62b1e..946b0d79a 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -33,4 +33,4 @@ jobs: - name: Run ctest run: | cd $GITHUB_WORKSPACE/GDASApp/build - ctest + ctest -VV From 91e98792f31fe2c8a5456436e6d5217295968920 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 20:16:48 +0000 Subject: [PATCH 08/13] Add netCDF4 to CI --- .github/workflows/unittests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index 946b0d79a..f4e51fb57 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -12,6 +12,7 @@ jobs: run: | pip install --upgrade pip pip install pycodestyle + pip install netCDF4 - name: Checkout uses: actions/checkout@v2 From 54984537967b7cd306dbca2ee8dbc4f4b896310b Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 20:18:34 +0000 Subject: [PATCH 09/13] No need to run pynorms so many times --- .github/workflows/pynorms.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pynorms.yaml b/.github/workflows/pynorms.yaml index 95c790b7b..abd6914db 100644 --- a/.github/workflows/pynorms.yaml +++ b/.github/workflows/pynorms.yaml @@ -1,5 +1,5 @@ name: Check Python Coding Norms -on: [push, pull_request] +on: [push] jobs: check_pynorms: From a8699d298c4b60b27aae7415ce351a93787ae27e Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 17 Mar 2022 20:29:40 +0000 Subject: [PATCH 10/13] update min cmake version --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5598a46e..28e9dd955 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.20) # set the project name project(GDASApp) From 46acaa65c53e223b25215260cbc5341c670abfca Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 21 Mar 2022 15:02:32 -0400 Subject: [PATCH 11/13] remove assert, wrap everything in a try --- ush/jediinc2fv3.py | 86 ++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/ush/jediinc2fv3.py b/ush/jediinc2fv3.py index c84a37867..86eb15b75 100644 --- a/ush/jediinc2fv3.py +++ b/ush/jediinc2fv3.py @@ -22,49 +22,51 @@ def jedi_inc_to_fv3(FV3JEDIinc, FV3inc): - assert os.path.exists(FV3JEDIinc), f"File {FV3JEDIinc} not found." - assert os.path.exists(FV3JEDIinc), f"File {FV3JEDIinc} not found." # open netCDF files - with nc.Dataset(FV3JEDIinc) as ncin, nc.Dataset(FV3inc, "w", format="NETCDF4") as ncout: - # copy over dimensions - for name, dimension in ncin.dimensions.items(): - if name == 'time': - continue - elif name == 'edge': - nameout = 'ilev' - else: - nameout = name - ncout.createDimension(nameout, - (len(dimension) if not dimension.isunlimited() else None)) - # some global attributes - ncout.source = 'jediinc2fv3.py' - ncout.comment = 'Increment produced by FV3-JEDI and modified for use by FV3 read' - # create all the dummy vertical coordinate variables - nlevs = len(ncin.dimensions['lev']) - pfull = range(1, nlevs+1) - phalf = range(1, nlevs+2) - levvar = ncout.createVariable('lev', 'f4', ('lev')) - levvar[:] = pfull - pfullvar = ncout.createVariable('pfull', 'f4', ('lev')) - pfullvar[:] = pfull - ilevvar = ncout.createVariable('ilev', 'f4', ('ilev')) - ilevvar[:] = phalf - hyaivar = ncout.createVariable('hyai', 'f4', ('ilev')) - hyaivar[:] = phalf - hybivar = ncout.createVariable('hybi', 'f4', ('ilev')) - hybivar[:] = phalf - # rename and change dimensionality of fields - for name, variable in ncin.variables.items(): - if len(variable.dimensions) == 4: - dimsout = variable.dimensions[1:] - else: - dimsout = variable.dimensions - if name in vardict: - x = ncout.createVariable(vardict[name], 'f4', dimsout) - if len(variable.dimensions) == 4: - ncout[vardict[name]][:] = ncin[name][0, ...] - else: - ncout[vardict[name]][:] = ncin[name][:] + try: + with nc.Dataset(FV3JEDIinc) as ncin, nc.Dataset(FV3inc, "w", format="NETCDF4") as ncout: + # copy over dimensions + for name, dimension in ncin.dimensions.items(): + if name == 'time': + continue + elif name == 'edge': + nameout = 'ilev' + else: + nameout = name + ncout.createDimension(nameout, + (len(dimension) if not dimension.isunlimited() else None)) + # some global attributes + ncout.source = 'jediinc2fv3.py' + ncout.comment = 'Increment produced by FV3-JEDI and modified for use by FV3 read' + # create all the dummy vertical coordinate variables + nlevs = len(ncin.dimensions['lev']) + pfull = range(1, nlevs+1) + phalf = range(1, nlevs+2) + levvar = ncout.createVariable('lev', 'f4', ('lev')) + levvar[:] = pfull + pfullvar = ncout.createVariable('pfull', 'f4', ('lev')) + pfullvar[:] = pfull + ilevvar = ncout.createVariable('ilev', 'f4', ('ilev')) + ilevvar[:] = phalf + hyaivar = ncout.createVariable('hyai', 'f4', ('ilev')) + hyaivar[:] = phalf + hybivar = ncout.createVariable('hybi', 'f4', ('ilev')) + hybivar[:] = phalf + # rename and change dimensionality of fields + for name, variable in ncin.variables.items(): + if len(variable.dimensions) == 4: + dimsout = variable.dimensions[1:] + else: + dimsout = variable.dimensions + if name in vardict: + x = ncout.createVariable(vardict[name], 'f4', dimsout) + if len(variable.dimensions) == 4: + ncout[vardict[name]][:] = ncin[name][0, ...] + else: + ncout[vardict[name]][:] = ncin[name][:] + except FileNotFoundError as e: + print(e) + raise if __name__ == "__main__": From 4283ca1ee6f9d058a77478295e97d0cac3a8a566 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 21 Mar 2022 15:12:12 -0400 Subject: [PATCH 12/13] Fix funky indentation --- ush/jediinc2fv3.py | 86 +++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/ush/jediinc2fv3.py b/ush/jediinc2fv3.py index 86eb15b75..90ecebc34 100644 --- a/ush/jediinc2fv3.py +++ b/ush/jediinc2fv3.py @@ -24,49 +24,49 @@ def jedi_inc_to_fv3(FV3JEDIinc, FV3inc): # open netCDF files try: - with nc.Dataset(FV3JEDIinc) as ncin, nc.Dataset(FV3inc, "w", format="NETCDF4") as ncout: - # copy over dimensions - for name, dimension in ncin.dimensions.items(): - if name == 'time': - continue - elif name == 'edge': - nameout = 'ilev' - else: - nameout = name - ncout.createDimension(nameout, - (len(dimension) if not dimension.isunlimited() else None)) - # some global attributes - ncout.source = 'jediinc2fv3.py' - ncout.comment = 'Increment produced by FV3-JEDI and modified for use by FV3 read' - # create all the dummy vertical coordinate variables - nlevs = len(ncin.dimensions['lev']) - pfull = range(1, nlevs+1) - phalf = range(1, nlevs+2) - levvar = ncout.createVariable('lev', 'f4', ('lev')) - levvar[:] = pfull - pfullvar = ncout.createVariable('pfull', 'f4', ('lev')) - pfullvar[:] = pfull - ilevvar = ncout.createVariable('ilev', 'f4', ('ilev')) - ilevvar[:] = phalf - hyaivar = ncout.createVariable('hyai', 'f4', ('ilev')) - hyaivar[:] = phalf - hybivar = ncout.createVariable('hybi', 'f4', ('ilev')) - hybivar[:] = phalf - # rename and change dimensionality of fields - for name, variable in ncin.variables.items(): - if len(variable.dimensions) == 4: - dimsout = variable.dimensions[1:] - else: - dimsout = variable.dimensions - if name in vardict: - x = ncout.createVariable(vardict[name], 'f4', dimsout) - if len(variable.dimensions) == 4: - ncout[vardict[name]][:] = ncin[name][0, ...] - else: - ncout[vardict[name]][:] = ncin[name][:] - except FileNotFoundError as e: - print(e) - raise + with nc.Dataset(FV3JEDIinc) as ncin, nc.Dataset(FV3inc, "w", format="NETCDF4") as ncout: + # copy over dimensions + for name, dimension in ncin.dimensions.items(): + if name == 'time': + continue + elif name == 'edge': + nameout = 'ilev' + else: + nameout = name + ncout.createDimension(nameout, + (len(dimension) if not dimension.isunlimited() else None)) + # some global attributes + ncout.source = 'jediinc2fv3.py' + ncout.comment = 'Increment produced by FV3-JEDI and modified for use by FV3 read' + # create all the dummy vertical coordinate variables + nlevs = len(ncin.dimensions['lev']) + pfull = range(1, nlevs+1) + phalf = range(1, nlevs+2) + levvar = ncout.createVariable('lev', 'f4', ('lev')) + levvar[:] = pfull + pfullvar = ncout.createVariable('pfull', 'f4', ('lev')) + pfullvar[:] = pfull + ilevvar = ncout.createVariable('ilev', 'f4', ('ilev')) + ilevvar[:] = phalf + hyaivar = ncout.createVariable('hyai', 'f4', ('ilev')) + hyaivar[:] = phalf + hybivar = ncout.createVariable('hybi', 'f4', ('ilev')) + hybivar[:] = phalf + # rename and change dimensionality of fields + for name, variable in ncin.variables.items(): + if len(variable.dimensions) == 4: + dimsout = variable.dimensions[1:] + else: + dimsout = variable.dimensions + if name in vardict: + x = ncout.createVariable(vardict[name], 'f4', dimsout) + if len(variable.dimensions) == 4: + ncout[vardict[name]][:] = ncin[name][0, ...] + else: + ncout[vardict[name]][:] = ncin[name][:] + except FileNotFoundError as e: + print(e) + raise if __name__ == "__main__": From faa972f3666f0da7422194049aee549c57a781f7 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 21 Mar 2022 15:36:35 -0400 Subject: [PATCH 13/13] Update CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fdc3f2264..36406443b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # ------------------------------------------------------------------------- # # Check for minimim cmake requirement -cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) +cmake_minimum_required( VERSION 3.20 FATAL_ERROR ) find_package(ecbuild 3.5 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)