From 1f4d60007e17f66096fc30de6c126848551cce5e Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 4 Dec 2023 13:08:16 -0500 Subject: [PATCH 01/37] Changes to gitignore and cmakelists --- .gitignore | 48 ++++++++++++++--------------- CMakeLists.txt | 82 ++++++++++++-------------------------------------- 2 files changed, 43 insertions(+), 87 deletions(-) diff --git a/.gitignore b/.gitignore index bd53d9c20..1e28511fa 100644 --- a/.gitignore +++ b/.gitignore @@ -151,28 +151,28 @@ atlas/ crtm/ eckit/ fckit/ -femps/ +/sorc/femps/ fms/ -fv3/ -fv3-jedi/ -fv3-jedi-data/ -fv3-jedi-lm/ -gsibec/ -gsw/ -icepack/ -/ioda/ -ioda-data/ -iodaconv/ -jedicmake/ -land-imsproc/ -land-jediincr/ -mom6/ -oops/ -saber/ -saber-data/ -/soca/ -test-data-release/ -ufo/ -ufo-data/ -vader/ -/gdas-utils +/sorc/fv3/ +/sorc/fv3-jedi/ +/fv3-jedi-data/ +/sorc/fv3-jedi-lm/ +/sorc/gsibec/ +/sorc/gsw/ +/sorc/icepack/ +/sorc/ioda/ +/ioda-data/ +/sorc/iodaconv/ +/sorc/jedicmake/ +/sorc/land-imsproc/ +/sorc/land-jediincr/ +/sorc/mom6/ +/sorc/oops/ +/sorc/saber/ +/saber-data/ +/sorc/soca/ +/test-data-release/ +/sorc/ufo/ +/ufo-data/ +/sorc/vader/ +/sorc/gdas-utils diff --git a/CMakeLists.txt b/CMakeLists.txt index 2091893fd..33bb172b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,7 +33,6 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) # Handle user options. option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON) -option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF) option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) # Initialize bundle @@ -44,8 +43,8 @@ ecbuild_bundle_initialize() if(BUILD_GDASBUNDLE) # jedi-cmake - ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/jcsda/jedi-cmake.git" TAG 1.4.0 RECURSIVE ) - include( jedicmake/cmake/Functions/git_functions.cmake ) + ecbuild_bundle( PROJECT jedicmake SOURCE "./sorc/jedicmake" ) + include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) # ECMWF libraries option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF @@ -67,39 +66,39 @@ if(BUILD_GDASBUNDLE) # Build GSI-B option(BUILD_GSIBEC "Build GSI-B" OFF) if(BUILD_GSIBEC) - ecbuild_bundle( PROJECT gsibec GIT "https://github.com/GEOS-ESM/GSIbec.git" TAG 1.1.2 ) + ecbuild_bundle( PROJECT gsibec SOURCE "./sorc/gsibec" ) endif() # Gibbs seawater - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop ) + ecbuild_bundle( PROJECT gsw SOURCE "./sorc/gsw" ) # Core JEDI repositories - ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda/oops.git" BRANCH develop) - ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop ) - ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop ) + ecbuild_bundle( PROJECT oops SOURCE "./sorc/oops" ) + ecbuild_bundle( PROJECT vader SOURCE "./sorc/vader" ) + ecbuild_bundle( PROJECT saber SOURCE "./sorc/saber" ) option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop ) + ecbuild_bundle( PROJECT ioda SOURCE "./sorc/ioda" ) option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) + ecbuild_bundle( PROJECT ufo SOURCE "./sorc/ufo" ) # FMS and FV3 dynamical core ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) - ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable ) + ecbuild_bundle( PROJECT fv3 SOURCE "./sorc/fv3" ) # fv3-jedi and associated repositories - ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" BRANCH develop ) - ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop ) + ecbuild_bundle( PROJECT femps SOURCE "./sorc/femps" ) + ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./sorc/fv3-jedi-lm" ) option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda/fv3-jedi.git" BRANCH develop ) + ecbuild_bundle( PROJECT fv3-jedi SOURCE "./sorc/fv3-jedi" ) # SOCA associated repositories # TODO: Move the Icepack fork to EMC github set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") if ( BUILD_ICEPACK ) - ecbuild_bundle( PROJECT icepack GIT "https://github.com/JCSDA-internal/Icepack.git" BRANCH feature/ecbuild-new ) + ecbuild_bundle( PROJECT icepack SOURCE "./sorc/icepack" ) endif() - ecbuild_bundle( PROJECT mom6 GIT "https://github.com/jcsda-internal/MOM6.git" BRANCH main-ecbuild RECURSIVE ) - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH develop ) + ecbuild_bundle( PROJECT mom6 SOURCE "./sorc/mom6" ) + ecbuild_bundle( PROJECT soca SOURCE "./sorc/soca" ) # Build JEDI/DA or other peripherals ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) @@ -107,55 +106,12 @@ if(BUILD_GDASBUNDLE) # Build IODA converters option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" BRANCH develop ) + ecbuild_bundle( PROJECT iodaconv SOURCE "./sorc/iodaconv" ) endif() # Land associated repositories - ecbuild_bundle( PROJECT land-imsproc GIT "https://github.com/NOAA-PSL/land-IMS_proc.git" TAG 6373819 ) - ecbuild_bundle( PROJECT land-jediincr GIT "https://github.com/NOAA-PSL/land-apply_jedi_incr.git" TAG 2923344) - -# ioda, ufo, fv3-jedi, and saber test data -#--------------------------------- -if(CLONE_JCSDADATA) - - # If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch. - # If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case, - # IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned. - # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored - # in a local directory, ioda-data repo will not be cloned - - find_branch_name(REPO_DIR_NAME ioda) - # When LOCAL_PATH_JEDI_TESTFILES is set to the directory of IODA test files stored - # in a local directory, ioda-data repo will not be cloned - if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) - ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop ) - - # If IODA's current branch is available in ioda-data repo, that branch will be checked out - branch_checkout (REPO_DIR_NAME ioda-data - BRANCH ${GIT_BRANCH_FUNC} ) - endif() - - # same procedure for ufo-data - find_branch_name(REPO_DIR_NAME ufo) - if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) - ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/JCSDA-internal/ufo-data.git" BRANCH develop ) - - # If UFO's current branch is available in ioda-data repo, that branch will be checked out - branch_checkout (REPO_DIR_NAME ufo-data - BRANCH ${GIT_BRANCH_FUNC} ) - endif() - - # same procedure for fv3-jedi-data - find_branch_name(REPO_DIR_NAME fv3-jedi) - if( NOT DEFINED ENV{LOCAL_PATH_JEDI_TESTFILES} AND NOT DEFINED GIT_TAG_FUNC ) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop ) - - # If fv3-jedi's current branch is available in ioda-data repo, that branch will be checked out - branch_checkout (REPO_DIR_NAME fv3-jedi-data - BRANCH ${GIT_BRANCH_FUNC} ) - endif() - - endif(CLONE_JCSDADATA) + ecbuild_bundle( PROJECT land-imsproc SOURCE "./sorc/land-imsproc" ) + ecbuild_bundle( PROJECT land-jediincr SOURCE "./sorc/land-jediincr" ) endif(BUILD_GDASBUNDLE) From f980d48506ce38c1da72872c6c94b138d6204cbf Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 4 Dec 2023 13:27:05 -0500 Subject: [PATCH 02/37] Add submodules --- .gitignore | 18 ---------------- .gitmodules | 54 ++++++++++++++++++++++++++++++++++++++++++++++ sorc/femps | 1 + sorc/fv3 | 1 + sorc/fv3-jedi | 1 + sorc/fv3-jedi-lm | 1 + sorc/gsibec | 1 + sorc/gsw | 1 + sorc/icepack | 1 + sorc/ioda | 1 + sorc/iodaconv | 1 + sorc/jedicmake | 1 + sorc/land-imsproc | 1 + sorc/land-jediincr | 1 + sorc/mom6 | 1 + sorc/oops | 1 + sorc/saber | 1 + sorc/soca | 1 + sorc/ufo | 1 + sorc/vader | 1 + 20 files changed, 72 insertions(+), 18 deletions(-) create mode 100644 .gitmodules create mode 160000 sorc/femps create mode 160000 sorc/fv3 create mode 160000 sorc/fv3-jedi create mode 160000 sorc/fv3-jedi-lm create mode 160000 sorc/gsibec create mode 160000 sorc/gsw create mode 160000 sorc/icepack create mode 160000 sorc/ioda create mode 160000 sorc/iodaconv create mode 160000 sorc/jedicmake create mode 160000 sorc/land-imsproc create mode 160000 sorc/land-jediincr create mode 160000 sorc/mom6 create mode 160000 sorc/oops create mode 160000 sorc/saber create mode 160000 sorc/soca create mode 160000 sorc/ufo create mode 160000 sorc/vader diff --git a/.gitignore b/.gitignore index 1e28511fa..58d164822 100644 --- a/.gitignore +++ b/.gitignore @@ -151,28 +151,10 @@ atlas/ crtm/ eckit/ fckit/ -/sorc/femps/ fms/ -/sorc/fv3/ -/sorc/fv3-jedi/ /fv3-jedi-data/ -/sorc/fv3-jedi-lm/ -/sorc/gsibec/ -/sorc/gsw/ -/sorc/icepack/ -/sorc/ioda/ /ioda-data/ -/sorc/iodaconv/ -/sorc/jedicmake/ -/sorc/land-imsproc/ -/sorc/land-jediincr/ -/sorc/mom6/ -/sorc/oops/ -/sorc/saber/ /saber-data/ -/sorc/soca/ /test-data-release/ -/sorc/ufo/ /ufo-data/ -/sorc/vader/ /sorc/gdas-utils diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..10e144d99 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,54 @@ +[submodule "sorc/jedicmake"] + path = sorc/jedicmake + url = https://github.com/jcsda/jedi-cmake.git +[submodule "sorc/gsibec"] + path = sorc/gsibec + url = https://github.com/GEOS-ESM/GSIbec.git +[submodule "sorc/gsw"] + path = sorc/gsw + url = https://github.com/jcsda-internal/GSW-Fortran.git +[submodule "sorc/oops"] + path = sorc/oops + url = https://github.com/jcsda/oops.git +[submodule "sorc/vader"] + path = sorc/vader + url = https://github.com/jcsda/vader.git +[submodule "sorc/saber"] + path = sorc/saber + url = https://github.com/jcsda/saber.git +[submodule "sorc/ioda"] + path = sorc/ioda + url = https://github.com/jcsda/ioda.git +[submodule "sorc/ufo"] + path = sorc/ufo + url = https://github.com/jcsda/ufo.git +[submodule "sorc/fv3"] + path = sorc/fv3 + url = https://github.com/jcsda/GFDL_atmos_cubed_sphere.git +[submodule "sorc/femps"] + path = sorc/femps + url = https://github.com/jcsda/femps.git +[submodule "sorc/fv3-jedi-lm"] + path = sorc/fv3-jedi-lm + url = https://github.com/jcsda/fv3-jedi-linearmodel.git +[submodule "sorc/fv3-jedi"] + path = sorc/fv3-jedi + url = https://github.com/jcsda/fv3-jedi.git +[submodule "sorc/icepack"] + path = sorc/icepack + url = https://github.com/JCSDA-internal/Icepack.git +[submodule "sorc/mom6"] + path = sorc/mom6 + url = https://github.com/jcsda-internal/MOM6.git +[submodule "sorc/soca"] + path = sorc/soca + url = https://github.com/jcsda-internal/soca.git +[submodule "sorc/iodaconv"] + path = sorc/iodaconv + url = https://github.com/JCSDA-internal/ioda-converters.git +[submodule "sorc/land-imsproc"] + path = sorc/land-imsproc + url = https://github.com/NOAA-PSL/land-IMS_proc.git +[submodule "sorc/land-jediincr"] + path = sorc/land-jediincr + url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git diff --git a/sorc/femps b/sorc/femps new file mode 160000 index 000000000..cb396811e --- /dev/null +++ b/sorc/femps @@ -0,0 +1 @@ +Subproject commit cb396811eb26380478c4d3f177d95096ed2ddd8f diff --git a/sorc/fv3 b/sorc/fv3 new file mode 160000 index 000000000..61450b4e3 --- /dev/null +++ b/sorc/fv3 @@ -0,0 +1 @@ +Subproject commit 61450b4e3e80bb96b26c5f3808ce60b5e5cb4207 diff --git a/sorc/fv3-jedi b/sorc/fv3-jedi new file mode 160000 index 000000000..702ac6b57 --- /dev/null +++ b/sorc/fv3-jedi @@ -0,0 +1 @@ +Subproject commit 702ac6b573769c58e63731dcb48ef26215053178 diff --git a/sorc/fv3-jedi-lm b/sorc/fv3-jedi-lm new file mode 160000 index 000000000..d68f1a859 --- /dev/null +++ b/sorc/fv3-jedi-lm @@ -0,0 +1 @@ +Subproject commit d68f1a8595ac62fb945d505e1962a69da11205c2 diff --git a/sorc/gsibec b/sorc/gsibec new file mode 160000 index 000000000..e760ec73c --- /dev/null +++ b/sorc/gsibec @@ -0,0 +1 @@ +Subproject commit e760ec73c117d1456fd6f7ead6bf4c7f4c38e139 diff --git a/sorc/gsw b/sorc/gsw new file mode 160000 index 000000000..1a02ebaf6 --- /dev/null +++ b/sorc/gsw @@ -0,0 +1 @@ +Subproject commit 1a02ebaf6f7a4e9f2c2d2dd973fb050e697bcc74 diff --git a/sorc/icepack b/sorc/icepack new file mode 160000 index 000000000..73136ee8d --- /dev/null +++ b/sorc/icepack @@ -0,0 +1 @@ +Subproject commit 73136ee8dcdbe378821e540488a5980a03d8abe6 diff --git a/sorc/ioda b/sorc/ioda new file mode 160000 index 000000000..b27373f08 --- /dev/null +++ b/sorc/ioda @@ -0,0 +1 @@ +Subproject commit b27373f084f63d7565338f13588118a73787890d diff --git a/sorc/iodaconv b/sorc/iodaconv new file mode 160000 index 000000000..c4f173139 --- /dev/null +++ b/sorc/iodaconv @@ -0,0 +1 @@ +Subproject commit c4f173139d2c159ac3474c519a87b445295fd2c4 diff --git a/sorc/jedicmake b/sorc/jedicmake new file mode 160000 index 000000000..36fc99bdf --- /dev/null +++ b/sorc/jedicmake @@ -0,0 +1 @@ +Subproject commit 36fc99bdff5d3d8835480b37a3dcc75e5f8da256 diff --git a/sorc/land-imsproc b/sorc/land-imsproc new file mode 160000 index 000000000..6373819ca --- /dev/null +++ b/sorc/land-imsproc @@ -0,0 +1 @@ +Subproject commit 6373819ca034d66523cb7852cd7b1b66f3f8ae07 diff --git a/sorc/land-jediincr b/sorc/land-jediincr new file mode 160000 index 000000000..2923344b3 --- /dev/null +++ b/sorc/land-jediincr @@ -0,0 +1 @@ +Subproject commit 2923344b33511d80c685c30261ad37896eb50768 diff --git a/sorc/mom6 b/sorc/mom6 new file mode 160000 index 000000000..51ec489ad --- /dev/null +++ b/sorc/mom6 @@ -0,0 +1 @@ +Subproject commit 51ec489ad7d8a86762bef4c46eabd9af5fc41fa4 diff --git a/sorc/oops b/sorc/oops new file mode 160000 index 000000000..1aa83c95a --- /dev/null +++ b/sorc/oops @@ -0,0 +1 @@ +Subproject commit 1aa83c95a3a46cad21fa8322c886c98ccf9966b8 diff --git a/sorc/saber b/sorc/saber new file mode 160000 index 000000000..660bdae35 --- /dev/null +++ b/sorc/saber @@ -0,0 +1 @@ +Subproject commit 660bdae35b1c14afbf9cb728ed4b72215be1982f diff --git a/sorc/soca b/sorc/soca new file mode 160000 index 000000000..0abcfaba3 --- /dev/null +++ b/sorc/soca @@ -0,0 +1 @@ +Subproject commit 0abcfaba3cadb9ebf26f0e4e806fba404a952e8e diff --git a/sorc/ufo b/sorc/ufo new file mode 160000 index 000000000..552be6c93 --- /dev/null +++ b/sorc/ufo @@ -0,0 +1 @@ +Subproject commit 552be6c93cc672bf109719b36895a5ef585e0508 diff --git a/sorc/vader b/sorc/vader new file mode 160000 index 000000000..099cb5db5 --- /dev/null +++ b/sorc/vader @@ -0,0 +1 @@ +Subproject commit 099cb5db5a2a16049ef959a4feef463fe23dee7e From cde0f2872b1f821f521102b4c21bd72cebf64f6a Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 4 Dec 2023 19:17:02 +0000 Subject: [PATCH 03/37] Changes to gitignore and ufo submodule --- .gitignore | 30 ++++++++++++++++++++++++------ sorc/ufo | 2 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 58d164822..bd53d9c20 100644 --- a/.gitignore +++ b/.gitignore @@ -151,10 +151,28 @@ atlas/ crtm/ eckit/ fckit/ +femps/ fms/ -/fv3-jedi-data/ -/ioda-data/ -/saber-data/ -/test-data-release/ -/ufo-data/ -/sorc/gdas-utils +fv3/ +fv3-jedi/ +fv3-jedi-data/ +fv3-jedi-lm/ +gsibec/ +gsw/ +icepack/ +/ioda/ +ioda-data/ +iodaconv/ +jedicmake/ +land-imsproc/ +land-jediincr/ +mom6/ +oops/ +saber/ +saber-data/ +/soca/ +test-data-release/ +ufo/ +ufo-data/ +vader/ +/gdas-utils diff --git a/sorc/ufo b/sorc/ufo index 552be6c93..9cab31e56 160000 --- a/sorc/ufo +++ b/sorc/ufo @@ -1 +1 @@ -Subproject commit 552be6c93cc672bf109719b36895a5ef585e0508 +Subproject commit 9cab31e5624f9d16d9734c319c8640262e3e1d6e From cf89d5dd7b581acf1cbda380dbdbb07b40e7511c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 4 Dec 2023 19:19:01 +0000 Subject: [PATCH 04/37] update gitignore --- .gitignore | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index bd53d9c20..f32cad3e8 100644 --- a/.gitignore +++ b/.gitignore @@ -146,33 +146,33 @@ build*/ install*/ # Ignore the following bundle repositories -ecbuild/ -atlas/ -crtm/ -eckit/ -fckit/ -femps/ -fms/ -fv3/ -fv3-jedi/ -fv3-jedi-data/ -fv3-jedi-lm/ -gsibec/ -gsw/ -icepack/ -/ioda/ -ioda-data/ -iodaconv/ -jedicmake/ -land-imsproc/ -land-jediincr/ -mom6/ -oops/ -saber/ -saber-data/ -/soca/ -test-data-release/ -ufo/ -ufo-data/ -vader/ +/ecbuild +/atlas +/crtm +/eckit +/fckit +/femps +/fms +/fv3 +/fv3-jedi +/fv3-jedi-data +/fv3-jedi-lm +/gsibec +/gsw +/icepack +/ioda +/ioda-data +/iodaconv +/jedicmake +/land-imsproc +/land-jediincr +/mom6 +/oops +/saber +/saber-data +/soca +/test-data-release +/ufo +/ufo-data +/vader /gdas-utils From 4df2e8e38e552c8697f75d547f8cd8adb0389554 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Mon, 4 Dec 2023 14:49:05 -0500 Subject: [PATCH 05/37] Check out branches --- .gitmodules | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitmodules b/.gitmodules index 10e144d99..7e83a5745 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,21 +7,27 @@ [submodule "sorc/gsw"] path = sorc/gsw url = https://github.com/jcsda-internal/GSW-Fortran.git + branch = develop [submodule "sorc/oops"] path = sorc/oops url = https://github.com/jcsda/oops.git + branch = develop [submodule "sorc/vader"] path = sorc/vader url = https://github.com/jcsda/vader.git + branch = develop [submodule "sorc/saber"] path = sorc/saber url = https://github.com/jcsda/saber.git + branch = develop [submodule "sorc/ioda"] path = sorc/ioda url = https://github.com/jcsda/ioda.git + branch = develop [submodule "sorc/ufo"] path = sorc/ufo url = https://github.com/jcsda/ufo.git + branch = develop [submodule "sorc/fv3"] path = sorc/fv3 url = https://github.com/jcsda/GFDL_atmos_cubed_sphere.git @@ -34,6 +40,7 @@ [submodule "sorc/fv3-jedi"] path = sorc/fv3-jedi url = https://github.com/jcsda/fv3-jedi.git + branch = develop [submodule "sorc/icepack"] path = sorc/icepack url = https://github.com/JCSDA-internal/Icepack.git @@ -43,9 +50,11 @@ [submodule "sorc/soca"] path = sorc/soca url = https://github.com/jcsda-internal/soca.git + branch = develop [submodule "sorc/iodaconv"] path = sorc/iodaconv url = https://github.com/JCSDA-internal/ioda-converters.git + branch = develop [submodule "sorc/land-imsproc"] path = sorc/land-imsproc url = https://github.com/NOAA-PSL/land-IMS_proc.git From 99a6e889c2d831ffd6a665f0e519c120df615a5b Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 12 Dec 2023 21:04:04 +0000 Subject: [PATCH 06/37] try to move bundle into subdirectory --- CMakeLists.txt | 75 +-------------------------------------------- sorc/CMakeLists.txt | 74 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 74 deletions(-) create mode 100644 sorc/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 33bb172b7..c2760148e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,80 +40,7 @@ option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) ecbuild_bundle_initialize() # Build bundle source code. -if(BUILD_GDASBUNDLE) - -# jedi-cmake - ecbuild_bundle( PROJECT jedicmake SOURCE "./sorc/jedicmake" ) - include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) - -# ECMWF libraries - option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF - option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF - option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF - -# turn off optional OOPS toy models - option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) - option( ENABLE_QG_MODEL "Build QG toy model" OFF ) - - ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) - ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) - ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) - -# External (required) observation operators - option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF - ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) - -# Build GSI-B - option(BUILD_GSIBEC "Build GSI-B" OFF) - if(BUILD_GSIBEC) - ecbuild_bundle( PROJECT gsibec SOURCE "./sorc/gsibec" ) - endif() - -# Gibbs seawater - ecbuild_bundle( PROJECT gsw SOURCE "./sorc/gsw" ) - -# Core JEDI repositories - ecbuild_bundle( PROJECT oops SOURCE "./sorc/oops" ) - ecbuild_bundle( PROJECT vader SOURCE "./sorc/vader" ) - ecbuild_bundle( PROJECT saber SOURCE "./sorc/saber" ) - option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ioda SOURCE "./sorc/ioda" ) - option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ufo SOURCE "./sorc/ufo" ) - -# FMS and FV3 dynamical core - ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) - ecbuild_bundle( PROJECT fv3 SOURCE "./sorc/fv3" ) - -# fv3-jedi and associated repositories - ecbuild_bundle( PROJECT femps SOURCE "./sorc/femps" ) - ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./sorc/fv3-jedi-lm" ) - option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT fv3-jedi SOURCE "./sorc/fv3-jedi" ) - -# SOCA associated repositories -# TODO: Move the Icepack fork to EMC github - set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") - if ( BUILD_ICEPACK ) - ecbuild_bundle( PROJECT icepack SOURCE "./sorc/icepack" ) - endif() - ecbuild_bundle( PROJECT mom6 SOURCE "./sorc/mom6" ) - ecbuild_bundle( PROJECT soca SOURCE "./sorc/soca" ) - - # Build JEDI/DA or other peripherals - ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) - -# Build IODA converters - option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) - if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv SOURCE "./sorc/iodaconv" ) - endif() - -# Land associated repositories - ecbuild_bundle( PROJECT land-imsproc SOURCE "./sorc/land-imsproc" ) - ecbuild_bundle( PROJECT land-jediincr SOURCE "./sorc/land-jediincr" ) - -endif(BUILD_GDASBUNDLE) +add_subdirectory(sorc) # Install utility scripts. add_subdirectory(ush) diff --git a/sorc/CMakeLists.txt b/sorc/CMakeLists.txt new file mode 100644 index 000000000..4f7ac669d --- /dev/null +++ b/sorc/CMakeLists.txt @@ -0,0 +1,74 @@ +if(BUILD_GDASBUNDLE) + +# jedi-cmake + ecbuild_bundle( PROJECT jedicmake SOURCE "./jedicmake" ) + include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) + +# ECMWF libraries + option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF + option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF + option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF + +# turn off optional OOPS toy models + option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) + option( ENABLE_QG_MODEL "Build QG toy model" OFF ) + + ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) + ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) + ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) + +# External (required) observation operators + option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF + ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) + +# Build GSI-B + option(BUILD_GSIBEC "Build GSI-B" OFF) + if(BUILD_GSIBEC) + ecbuild_bundle( PROJECT gsibec SOURCE "./gsibec" ) + endif() + +# Gibbs seawater + ecbuild_bundle( PROJECT gsw SOURCE "./gsw" ) + +# Core JEDI repositories + ecbuild_bundle( PROJECT oops SOURCE "./oops" ) + ecbuild_bundle( PROJECT vader SOURCE "./vader" ) + ecbuild_bundle( PROJECT saber SOURCE "./saber" ) + option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ioda SOURCE "./ioda" ) + option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ufo SOURCE "./ufo" ) + +# FMS and FV3 dynamical core + ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) + ecbuild_bundle( PROJECT fv3 SOURCE "./fv3" ) + +# fv3-jedi and associated repositories + ecbuild_bundle( PROJECT femps SOURCE "./femps" ) + ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./fv3-jedi-lm" ) + option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT fv3-jedi SOURCE "./fv3-jedi" ) + +# SOCA associated repositories +# TODO: Move the Icepack fork to EMC github + set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") + if ( BUILD_ICEPACK ) + ecbuild_bundle( PROJECT icepack SOURCE "./icepack" ) + endif() + ecbuild_bundle( PROJECT mom6 SOURCE "./mom6" ) + ecbuild_bundle( PROJECT soca SOURCE "./soca" ) + + # Build JEDI/DA or other peripherals + ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) + +# Build IODA converters + option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) + if(BUILD_IODA_CONVERTERS) + ecbuild_bundle( PROJECT iodaconv SOURCE "./iodaconv" ) + endif() + +# Land associated repositories + ecbuild_bundle( PROJECT land-imsproc SOURCE "./land-imsproc" ) + ecbuild_bundle( PROJECT land-jediincr SOURCE "./land-jediincr" ) + +endif(BUILD_GDASBUNDLE) From 77c47dbfd334a772803777eb0072a15fcc59b825 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 12 Dec 2023 21:39:26 +0000 Subject: [PATCH 07/37] Revert "try to move bundle into subdirectory" This reverts commit 99a6e889c2d831ffd6a665f0e519c120df615a5b. --- CMakeLists.txt | 75 ++++++++++++++++++++++++++++++++++++++++++++- sorc/CMakeLists.txt | 74 -------------------------------------------- 2 files changed, 74 insertions(+), 75 deletions(-) delete mode 100644 sorc/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index c2760148e..33bb172b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,7 +40,80 @@ option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) ecbuild_bundle_initialize() # Build bundle source code. -add_subdirectory(sorc) +if(BUILD_GDASBUNDLE) + +# jedi-cmake + ecbuild_bundle( PROJECT jedicmake SOURCE "./sorc/jedicmake" ) + include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) + +# ECMWF libraries + option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF + option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF + option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF + +# turn off optional OOPS toy models + option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) + option( ENABLE_QG_MODEL "Build QG toy model" OFF ) + + ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) + ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) + ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) + +# External (required) observation operators + option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF + ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) + +# Build GSI-B + option(BUILD_GSIBEC "Build GSI-B" OFF) + if(BUILD_GSIBEC) + ecbuild_bundle( PROJECT gsibec SOURCE "./sorc/gsibec" ) + endif() + +# Gibbs seawater + ecbuild_bundle( PROJECT gsw SOURCE "./sorc/gsw" ) + +# Core JEDI repositories + ecbuild_bundle( PROJECT oops SOURCE "./sorc/oops" ) + ecbuild_bundle( PROJECT vader SOURCE "./sorc/vader" ) + ecbuild_bundle( PROJECT saber SOURCE "./sorc/saber" ) + option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ioda SOURCE "./sorc/ioda" ) + option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ufo SOURCE "./sorc/ufo" ) + +# FMS and FV3 dynamical core + ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) + ecbuild_bundle( PROJECT fv3 SOURCE "./sorc/fv3" ) + +# fv3-jedi and associated repositories + ecbuild_bundle( PROJECT femps SOURCE "./sorc/femps" ) + ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./sorc/fv3-jedi-lm" ) + option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT fv3-jedi SOURCE "./sorc/fv3-jedi" ) + +# SOCA associated repositories +# TODO: Move the Icepack fork to EMC github + set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") + if ( BUILD_ICEPACK ) + ecbuild_bundle( PROJECT icepack SOURCE "./sorc/icepack" ) + endif() + ecbuild_bundle( PROJECT mom6 SOURCE "./sorc/mom6" ) + ecbuild_bundle( PROJECT soca SOURCE "./sorc/soca" ) + + # Build JEDI/DA or other peripherals + ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) + +# Build IODA converters + option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) + if(BUILD_IODA_CONVERTERS) + ecbuild_bundle( PROJECT iodaconv SOURCE "./sorc/iodaconv" ) + endif() + +# Land associated repositories + ecbuild_bundle( PROJECT land-imsproc SOURCE "./sorc/land-imsproc" ) + ecbuild_bundle( PROJECT land-jediincr SOURCE "./sorc/land-jediincr" ) + +endif(BUILD_GDASBUNDLE) # Install utility scripts. add_subdirectory(ush) diff --git a/sorc/CMakeLists.txt b/sorc/CMakeLists.txt deleted file mode 100644 index 4f7ac669d..000000000 --- a/sorc/CMakeLists.txt +++ /dev/null @@ -1,74 +0,0 @@ -if(BUILD_GDASBUNDLE) - -# jedi-cmake - ecbuild_bundle( PROJECT jedicmake SOURCE "./jedicmake" ) - include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) - -# ECMWF libraries - option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF - option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF - option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF - -# turn off optional OOPS toy models - option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) - option( ENABLE_QG_MODEL "Build QG toy model" OFF ) - - ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) - ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) - ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) - -# External (required) observation operators - option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF - ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) - -# Build GSI-B - option(BUILD_GSIBEC "Build GSI-B" OFF) - if(BUILD_GSIBEC) - ecbuild_bundle( PROJECT gsibec SOURCE "./gsibec" ) - endif() - -# Gibbs seawater - ecbuild_bundle( PROJECT gsw SOURCE "./gsw" ) - -# Core JEDI repositories - ecbuild_bundle( PROJECT oops SOURCE "./oops" ) - ecbuild_bundle( PROJECT vader SOURCE "./vader" ) - ecbuild_bundle( PROJECT saber SOURCE "./saber" ) - option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ioda SOURCE "./ioda" ) - option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ufo SOURCE "./ufo" ) - -# FMS and FV3 dynamical core - ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) - ecbuild_bundle( PROJECT fv3 SOURCE "./fv3" ) - -# fv3-jedi and associated repositories - ecbuild_bundle( PROJECT femps SOURCE "./femps" ) - ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./fv3-jedi-lm" ) - option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT fv3-jedi SOURCE "./fv3-jedi" ) - -# SOCA associated repositories -# TODO: Move the Icepack fork to EMC github - set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") - if ( BUILD_ICEPACK ) - ecbuild_bundle( PROJECT icepack SOURCE "./icepack" ) - endif() - ecbuild_bundle( PROJECT mom6 SOURCE "./mom6" ) - ecbuild_bundle( PROJECT soca SOURCE "./soca" ) - - # Build JEDI/DA or other peripherals - ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) - -# Build IODA converters - option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) - if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv SOURCE "./iodaconv" ) - endif() - -# Land associated repositories - ecbuild_bundle( PROJECT land-imsproc SOURCE "./land-imsproc" ) - ecbuild_bundle( PROJECT land-jediincr SOURCE "./land-jediincr" ) - -endif(BUILD_GDASBUNDLE) From ee412a868f628beef95fe81e4cfc61fb3a640596 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 12 Dec 2023 21:52:08 +0000 Subject: [PATCH 08/37] Update modules --- .gitmodules | 8 ++++++++ sorc/fv3-jedi | 2 +- sorc/ioda | 2 +- sorc/land-imsproc | 2 +- sorc/oops | 2 +- sorc/saber | 2 +- sorc/soca | 2 +- sorc/ufo | 2 +- 8 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7e83a5745..bc2f0b99e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,6 +4,7 @@ [submodule "sorc/gsibec"] path = sorc/gsibec url = https://github.com/GEOS-ESM/GSIbec.git + branch = develop [submodule "sorc/gsw"] path = sorc/gsw url = https://github.com/jcsda-internal/GSW-Fortran.git @@ -31,12 +32,15 @@ [submodule "sorc/fv3"] path = sorc/fv3 url = https://github.com/jcsda/GFDL_atmos_cubed_sphere.git + branch = release-stable [submodule "sorc/femps"] path = sorc/femps url = https://github.com/jcsda/femps.git + branch = develop [submodule "sorc/fv3-jedi-lm"] path = sorc/fv3-jedi-lm url = https://github.com/jcsda/fv3-jedi-linearmodel.git + branch = develop [submodule "sorc/fv3-jedi"] path = sorc/fv3-jedi url = https://github.com/jcsda/fv3-jedi.git @@ -44,9 +48,11 @@ [submodule "sorc/icepack"] path = sorc/icepack url = https://github.com/JCSDA-internal/Icepack.git + branch = feature/ecbuild-new [submodule "sorc/mom6"] path = sorc/mom6 url = https://github.com/jcsda-internal/MOM6.git + branch = main-ecbuild [submodule "sorc/soca"] path = sorc/soca url = https://github.com/jcsda-internal/soca.git @@ -58,6 +64,8 @@ [submodule "sorc/land-imsproc"] path = sorc/land-imsproc url = https://github.com/NOAA-PSL/land-IMS_proc.git + branch = develop [submodule "sorc/land-jediincr"] path = sorc/land-jediincr url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git + branch = develop diff --git a/sorc/fv3-jedi b/sorc/fv3-jedi index 702ac6b57..1ccb1c7bc 160000 --- a/sorc/fv3-jedi +++ b/sorc/fv3-jedi @@ -1 +1 @@ -Subproject commit 702ac6b573769c58e63731dcb48ef26215053178 +Subproject commit 1ccb1c7bcc934c0497259d258aacb0c0df215fbf diff --git a/sorc/ioda b/sorc/ioda index b27373f08..3194c749f 160000 --- a/sorc/ioda +++ b/sorc/ioda @@ -1 +1 @@ -Subproject commit b27373f084f63d7565338f13588118a73787890d +Subproject commit 3194c749f999bfd26319792042350327a444961c diff --git a/sorc/land-imsproc b/sorc/land-imsproc index 6373819ca..2ac7c0d90 160000 --- a/sorc/land-imsproc +++ b/sorc/land-imsproc @@ -1 +1 @@ -Subproject commit 6373819ca034d66523cb7852cd7b1b66f3f8ae07 +Subproject commit 2ac7c0d90d3ebd449ad11bce155a71d381b4534a diff --git a/sorc/oops b/sorc/oops index 1aa83c95a..6d4a414f4 160000 --- a/sorc/oops +++ b/sorc/oops @@ -1 +1 @@ -Subproject commit 1aa83c95a3a46cad21fa8322c886c98ccf9966b8 +Subproject commit 6d4a414f4017d1ad0df3be90d8a40d095a0556b6 diff --git a/sorc/saber b/sorc/saber index 660bdae35..426fbb003 160000 --- a/sorc/saber +++ b/sorc/saber @@ -1 +1 @@ -Subproject commit 660bdae35b1c14afbf9cb728ed4b72215be1982f +Subproject commit 426fbb003c8fb806dd7ea6c30c32fdfa438a63b4 diff --git a/sorc/soca b/sorc/soca index 0abcfaba3..15eab9640 160000 --- a/sorc/soca +++ b/sorc/soca @@ -1 +1 @@ -Subproject commit 0abcfaba3cadb9ebf26f0e4e806fba404a952e8e +Subproject commit 15eab96405cd59c12b659d4db76ce239937ba52f diff --git a/sorc/ufo b/sorc/ufo index 9cab31e56..c042c2fc4 160000 --- a/sorc/ufo +++ b/sorc/ufo @@ -1 +1 @@ -Subproject commit 9cab31e5624f9d16d9734c319c8640262e3e1d6e +Subproject commit c042c2fc473edb719b942971dc4ed694057f5226 From f3edb412c022422cc6ab50006a4a617ab92ee17c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 13 Dec 2023 15:42:33 +0000 Subject: [PATCH 09/37] Start to have scripts to update submodules --- ci/stable_driver.sh | 22 +----------------- ci/stable_mark.sh | 38 -------------------------------- ush/submodules/update_develop.sh | 23 +++++++++++++++++++ 3 files changed, 24 insertions(+), 59 deletions(-) delete mode 100755 ci/stable_mark.sh create mode 100755 ush/submodules/update_develop.sh diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index ffcf88c3f..b7f8ea981 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -67,18 +67,10 @@ cd gdas.cd git checkout develop git pull -# ============================================================================== -# run ecbuild to get the repos cloned -mkdir -p build -cd build -ecbuild ../ -cd .. -rm -rf build - # ============================================================================== # update the hashes to the most recent gdasdir=$stableroot/$datestr/global-workflow/sorc/gdas.cd -$my_dir/stable_mark.sh $gdasdir +$my_dir/../ush/submodules/update_develop.sh $gdasdir/sorc # ============================================================================== # run the automated testing @@ -87,12 +79,6 @@ ci_status=$? total=0 if [ $ci_status -eq 0 ]; then cd $gdasdir - # copy the CMakeLists file for safe keeping - cp $gdasdir/CMakeLists.txt $gdasdir/CMakeLists.txt.new - total=$(($total+$?)) - if [ $total -ne 0 ]; then - echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output - fi # checkout feature/stable-nightly git stash total=$(($total+$?)) @@ -110,12 +96,6 @@ if [ $ci_status -eq 0 ]; then if [ $total -ne 0 ]; then echo "Unable to merge develop" >> $stableroot/$datestr/output fi - # force move the copy to the original path of CMakeLists.txt - /bin/mv -f $gdasdir/CMakeLists.txt.new $gdasdir/CMakeLists.txt - total=$(($total+$?)) - if [ $total -ne 0 ]; then - echo "Unable to mv CMakeLists" >> $stableroot/$datestr/output - fi # commit this change and push git add CMakeLists.txt total=$(($total+$?)) diff --git a/ci/stable_mark.sh b/ci/stable_mark.sh deleted file mode 100755 index b3a67a2d0..000000000 --- a/ci/stable_mark.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Stolen from Travis via soca-science -# original script: -# https://github.com/JCSDA-internal/soca-science/blob/develop/.github/travisci/stable_mark.sh -# -set -e - -# figure out what git hash is associated with each repo in the bundle. -# Note that there are several places where this repo could exist. -bundle_dir=$1 -bundle_repos="oops vader saber ioda ufo iodaconv ioda-data ufo-data saber-data \ - fms fv3 femps fv3-jedi-lm fv3-jedi fv3-jedi-data gsibec \ - gsw mom6 soca" -for r in $bundle_repos; do - - echo "" - echo "Finding hash tag for $r..." - hash="none" - - # check the repo source ( for uncached repos, i.e. the main test repo) - if [[ "$hash" == "none" ]]; then - echo -n "searching src.. " - src_dir=$bundle_dir/$r - [[ -d $src_dir ]] \ - && cd $src_dir \ - && hash=$(git rev-parse HEAD || echo "none") - [[ "$hash" == "none" ]] && echo "NOT found" || echo "FOUND" - fi - - # if a git hash was found, update the bundle with a tagged version - echo "git_hash: $hash" - if [[ $hash != "none" ]]; then - hash=${hash:0:7} - echo "changing $r to $hash in $bundle_dir/CMakeLists.txt" - sed -i "s/\(.*PROJECT \+$r .*\)\(BRANCH\|TAG\) *\([a-zA-Z0-9\/\_\.\-]*\)\(.*\)/\1TAG $hash\4/g" $bundle_dir/CMakeLists.txt - fi -done diff --git a/ush/submodules/update_develop.sh b/ush/submodules/update_develop.sh new file mode 100755 index 000000000..1d23ffad2 --- /dev/null +++ b/ush/submodules/update_develop.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# update_develop.sh +# update specified repositories to most recent develop hash + +repos=" +oops +vader +saber +ioda +ufo +fv3-jedi +soca +iodaconv +" + +my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" + +sorcdir=${1:-${my_dir}/../../sorc} + +for r in $repos; do + echo "Updating ${sorcdir}/${r}" + git submodule update --remote --merge +done From 2d308101912891a06667d12db5c41edabc28480e Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 13 Dec 2023 21:00:52 +0000 Subject: [PATCH 10/37] Actually cd --- ush/submodules/update_develop.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/submodules/update_develop.sh b/ush/submodules/update_develop.sh index 1d23ffad2..d1ab5eaea 100755 --- a/ush/submodules/update_develop.sh +++ b/ush/submodules/update_develop.sh @@ -19,5 +19,6 @@ sorcdir=${1:-${my_dir}/../../sorc} for r in $repos; do echo "Updating ${sorcdir}/${r}" + cd ${sorcdir}/${r} git submodule update --remote --merge done From b6154489537f8c580b0980648039f05eabf91a6c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 13 Dec 2023 21:21:00 +0000 Subject: [PATCH 11/37] Add JCSDA test data to submodules --- .gitmodules | 9 +++++++++ CMakeLists.txt | 11 +++++++++++ test/jcsda/fv3-jedi-data | 1 + test/jcsda/ioda-data | 1 + test/jcsda/ufo-data | 1 + 5 files changed, 23 insertions(+) create mode 160000 test/jcsda/fv3-jedi-data create mode 160000 test/jcsda/ioda-data create mode 160000 test/jcsda/ufo-data diff --git a/.gitmodules b/.gitmodules index bc2f0b99e..d8072a0c2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -69,3 +69,12 @@ path = sorc/land-jediincr url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git branch = develop +[submodule "test/jcsda/ioda-data"] + path = test/jcsda/ioda-data + url = https://github.com/JCSDA-internal/ioda-data.git +[submodule "test/jcsda/ufo-data"] + path = test/jcsda/ufo-data + url = https://github.com/JCSDA-internal/ufo-data.git +[submodule "test/jcsda/fv3-jedi-data"] + path = test/jcsda/fv3-jedi-data + url = https://github.com/JCSDA-internal/fv3-jedi-data.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 33bb172b7..789e33602 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,6 +33,7 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) # Handle user options. option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON) +option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF) option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) # Initialize bundle @@ -113,6 +114,16 @@ if(BUILD_GDASBUNDLE) ecbuild_bundle( PROJECT land-imsproc SOURCE "./sorc/land-imsproc" ) ecbuild_bundle( PROJECT land-jediincr SOURCE "./sorc/land-jediincr" ) +# ioda, ufo, fv3-jedi, and saber test data +#--------------------------------- + if(CLONE_JCSDADATA) + + ecbuild_bundle( PROJECT ioda-data SOURCE "./test/jcsda/ioda-data" ) + ecbuild_bundle( PROJECT ufo-data SOURCE "./test/jcsda/ufo-data" ) + ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "./test/jcsda/fv3-jedi-data" ) + + endif(CLONE_JCSDADATA) + endif(BUILD_GDASBUNDLE) # Install utility scripts. diff --git a/test/jcsda/fv3-jedi-data b/test/jcsda/fv3-jedi-data new file mode 160000 index 000000000..0767e0838 --- /dev/null +++ b/test/jcsda/fv3-jedi-data @@ -0,0 +1 @@ +Subproject commit 0767e0838857739234004d34dde38a5866f1d77c diff --git a/test/jcsda/ioda-data b/test/jcsda/ioda-data new file mode 160000 index 000000000..58516bdf0 --- /dev/null +++ b/test/jcsda/ioda-data @@ -0,0 +1 @@ +Subproject commit 58516bdf051d8d80f2f892c9be1fe6f5d4769d24 diff --git a/test/jcsda/ufo-data b/test/jcsda/ufo-data new file mode 160000 index 000000000..11cff0536 --- /dev/null +++ b/test/jcsda/ufo-data @@ -0,0 +1 @@ +Subproject commit 11cff053665d11e96ad1892d8983e6b8dfd76ff7 From 509126e2ac15e7cc64b42c6e012e0ff3f44c7e9c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 13 Dec 2023 21:24:37 +0000 Subject: [PATCH 12/37] Update data repos too --- .gitmodules | 3 +++ ci/stable_driver.sh | 2 +- ush/submodules/update_develop.sh | 18 +++++++++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index d8072a0c2..61a5214b8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -72,9 +72,12 @@ [submodule "test/jcsda/ioda-data"] path = test/jcsda/ioda-data url = https://github.com/JCSDA-internal/ioda-data.git + branch = develop [submodule "test/jcsda/ufo-data"] path = test/jcsda/ufo-data url = https://github.com/JCSDA-internal/ufo-data.git + branch = develop [submodule "test/jcsda/fv3-jedi-data"] path = test/jcsda/fv3-jedi-data url = https://github.com/JCSDA-internal/fv3-jedi-data.git + branch = develop diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index b7f8ea981..b6f97fad9 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -70,7 +70,7 @@ git pull # ============================================================================== # update the hashes to the most recent gdasdir=$stableroot/$datestr/global-workflow/sorc/gdas.cd -$my_dir/../ush/submodules/update_develop.sh $gdasdir/sorc +$my_dir/../ush/submodules/update_develop.sh $gdasdir # ============================================================================== # run the automated testing diff --git a/ush/submodules/update_develop.sh b/ush/submodules/update_develop.sh index d1ab5eaea..4584c5671 100755 --- a/ush/submodules/update_develop.sh +++ b/ush/submodules/update_develop.sh @@ -15,10 +15,22 @@ iodaconv my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" -sorcdir=${1:-${my_dir}/../../sorc} +gdasdir=${1:-${my_dir}/../../} for r in $repos; do - echo "Updating ${sorcdir}/${r}" - cd ${sorcdir}/${r} + echo "Updating ${gdasdir}/sorc/${r}" + cd ${gdasdir}/sorc/${r} + git submodule update --remote --merge +done + +datarepos=" +ufo-data +ioda-data +fv3-jedi-data +" + +for r in $datarepos; do + echo "Updating ${gdasdir}/test/jcsda/${r}" + cd ${gdasdir}/test/jcsda/${r} git submodule update --remote --merge done From 337e3b4d8b6516f0261e683c8dad3be3d74166c4 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 14 Dec 2023 19:06:39 +0000 Subject: [PATCH 13/37] Add CRTM as a submodule --- .gitmodules | 3 +++ CMakeLists.txt | 6 ++++-- sorc/crtm | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) create mode 160000 sorc/crtm diff --git a/.gitmodules b/.gitmodules index 61a5214b8..a1d895072 100644 --- a/.gitmodules +++ b/.gitmodules @@ -81,3 +81,6 @@ path = test/jcsda/fv3-jedi-data url = https://github.com/JCSDA-internal/fv3-jedi-data.git branch = develop +[submodule "sorc/crtm"] + path = sorc/crtm + url = https://github.com/jcsda/crtm.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 789e33602..a5d329537 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,8 +61,10 @@ if(BUILD_GDASBUNDLE) ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) # External (required) observation operators - option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF - ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) + # TODO remove the CRTM from here and use it as a library + #option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF + #ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) + ecbuild_bundle( PROJECT crtm SOURCE "./sorc/crtm" ) # Build GSI-B option(BUILD_GSIBEC "Build GSI-B" OFF) diff --git a/sorc/crtm b/sorc/crtm new file mode 160000 index 000000000..bb7adbfc4 --- /dev/null +++ b/sorc/crtm @@ -0,0 +1 @@ +Subproject commit bb7adbfc4f3deadedac98743bcb4e18a2bca11f9 From a1fc81201f7a2d98cffc5eae6b097f3a168d22fc Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 14 Dec 2023 19:32:46 +0000 Subject: [PATCH 14/37] Remove data submodules --- .gitmodules | 12 ------------ ci/stable_driver.sh | 15 ++++++++++----- test/jcsda/fv3-jedi-data | 1 - test/jcsda/ioda-data | 1 - test/jcsda/ufo-data | 1 - 5 files changed, 10 insertions(+), 20 deletions(-) delete mode 160000 test/jcsda/fv3-jedi-data delete mode 160000 test/jcsda/ioda-data delete mode 160000 test/jcsda/ufo-data diff --git a/.gitmodules b/.gitmodules index a1d895072..f1fbf0f8a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -69,18 +69,6 @@ path = sorc/land-jediincr url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git branch = develop -[submodule "test/jcsda/ioda-data"] - path = test/jcsda/ioda-data - url = https://github.com/JCSDA-internal/ioda-data.git - branch = develop -[submodule "test/jcsda/ufo-data"] - path = test/jcsda/ufo-data - url = https://github.com/JCSDA-internal/ufo-data.git - branch = develop -[submodule "test/jcsda/fv3-jedi-data"] - path = test/jcsda/fv3-jedi-data - url = https://github.com/JCSDA-internal/fv3-jedi-data.git - branch = develop [submodule "sorc/crtm"] path = sorc/crtm url = https://github.com/jcsda/crtm.git diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index b6f97fad9..31232837a 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -83,12 +83,12 @@ if [ $ci_status -eq 0 ]; then git stash total=$(($total+$?)) if [ $total -ne 0 ]; then - echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output + echo "Unable to git stash" >> $stableroot/$datestr/output fi git checkout feature/stable-nightly total=$(($total+$?)) if [ $total -ne 0 ]; then - echo "Unable to cp CMakeLists" >> $stableroot/$datestr/output + echo "Unable to checkout feature/stable-nightly" >> $stableroot/$datestr/output fi # merge in develop git merge develop @@ -96,11 +96,16 @@ if [ $ci_status -eq 0 ]; then if [ $total -ne 0 ]; then echo "Unable to merge develop" >> $stableroot/$datestr/output fi - # commit this change and push - git add CMakeLists.txt + # add in submodules + git stash pop total=$(($total+$?)) if [ $total -ne 0 ]; then - echo "Unable to add CMakeLists to commit" >> $stableroot/$datestr/output + echo "Unable to git stash pop" >> $stableroot/$datestr/output + fi + $my_dir/../ush/submodules/add_submodules.sh $gdasdir + total=$(($total+$?)) + if [ $total -ne 0 ]; then + echo "Unable to add updated submodules to commit" >> $stableroot/$datestr/output fi git diff-index --quiet HEAD || git commit -m "Update to new stable build on $datestr" total=$(($total+$?)) diff --git a/test/jcsda/fv3-jedi-data b/test/jcsda/fv3-jedi-data deleted file mode 160000 index 0767e0838..000000000 --- a/test/jcsda/fv3-jedi-data +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0767e0838857739234004d34dde38a5866f1d77c diff --git a/test/jcsda/ioda-data b/test/jcsda/ioda-data deleted file mode 160000 index 58516bdf0..000000000 --- a/test/jcsda/ioda-data +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 58516bdf051d8d80f2f892c9be1fe6f5d4769d24 diff --git a/test/jcsda/ufo-data b/test/jcsda/ufo-data deleted file mode 160000 index 11cff0536..000000000 --- a/test/jcsda/ufo-data +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 11cff053665d11e96ad1892d8983e6b8dfd76ff7 From a99a29af256332aab9496836cabc0ab64c5ab483 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 14 Dec 2023 19:37:38 +0000 Subject: [PATCH 15/37] Try cmake magic --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5d329537..390ab0ec9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,9 +120,10 @@ if(BUILD_GDASBUNDLE) #--------------------------------- if(CLONE_JCSDADATA) - ecbuild_bundle( PROJECT ioda-data SOURCE "./test/jcsda/ioda-data" ) - ecbuild_bundle( PROJECT ufo-data SOURCE "./test/jcsda/ufo-data" ) - ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "./test/jcsda/fv3-jedi-data" ) + set(JCSDA_DATA_ROOT "$ENV{GDASAPP_TESTDATA}/jcsda" + ecbuild_bundle( PROJECT ioda-data SOURCE "${JCSDA_DATA_ROOT}/ioda-data" ) + ecbuild_bundle( PROJECT ufo-data SOURCE "${JCSDA_DATA_ROOT}/ufo-data" ) + ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "${JCSDA_DATA_ROOT}/fv3-jedi-data" ) endif(CLONE_JCSDADATA) From 3e6045a2f77fcc98d6fdca9f1a38f426f37946ab Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 14 Dec 2023 19:45:47 +0000 Subject: [PATCH 16/37] Bugfix --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 390ab0ec9..f7c376b72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,7 +120,7 @@ if(BUILD_GDASBUNDLE) #--------------------------------- if(CLONE_JCSDADATA) - set(JCSDA_DATA_ROOT "$ENV{GDASAPP_TESTDATA}/jcsda" + set(JCSDA_DATA_ROOT "$ENV{GDASAPP_TESTDATA}/jcsda") ecbuild_bundle( PROJECT ioda-data SOURCE "${JCSDA_DATA_ROOT}/ioda-data" ) ecbuild_bundle( PROJECT ufo-data SOURCE "${JCSDA_DATA_ROOT}/ufo-data" ) ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "${JCSDA_DATA_ROOT}/fv3-jedi-data" ) From 1877dc2b37f00555a1e9c32f857a437334bd600c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 14 Dec 2023 20:06:51 +0000 Subject: [PATCH 17/37] More stuff --- ush/submodules/add_submodules.sh | 24 ++++++++++++++++++++++++ ush/submodules/update_develop.sh | 12 ------------ 2 files changed, 24 insertions(+), 12 deletions(-) create mode 100755 ush/submodules/add_submodules.sh diff --git a/ush/submodules/add_submodules.sh b/ush/submodules/add_submodules.sh new file mode 100755 index 000000000..16d239afe --- /dev/null +++ b/ush/submodules/add_submodules.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# add_submodules.sh +# add submodules to the git commit + +my_dir="$( cd "$( dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd )" + +gdasdir=${1:-${my_dir}/../../} + +repos=" +oops +vader +saber +ioda +ufo +fv3-jedi +soca +iodaconv +" + +for r in $repos; do + echo "Adding ${gdasdir}/sorc/${r}" + cd ${gdasdir}/sorc + git add ${r} +done diff --git a/ush/submodules/update_develop.sh b/ush/submodules/update_develop.sh index 4584c5671..403d97d5a 100755 --- a/ush/submodules/update_develop.sh +++ b/ush/submodules/update_develop.sh @@ -22,15 +22,3 @@ for r in $repos; do cd ${gdasdir}/sorc/${r} git submodule update --remote --merge done - -datarepos=" -ufo-data -ioda-data -fv3-jedi-data -" - -for r in $datarepos; do - echo "Updating ${gdasdir}/test/jcsda/${r}" - cd ${gdasdir}/test/jcsda/${r} - git submodule update --remote --merge -done From 6ba88ffd5f6ecbed966735351c287bbc368ed847 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 15 Dec 2023 21:36:34 +0000 Subject: [PATCH 18/37] this is not working but I want to save before the weekend --- CMakeLists.txt | 184 ++++++++++++++++++++++---------------------- build.sh | 2 +- sorc/CMakeLists.txt | 137 +++++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 93 deletions(-) create mode 100644 sorc/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f7c376b72..c64af831a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,98 +36,98 @@ option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON) option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF) option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) -# Initialize bundle -# ----------------- -ecbuild_bundle_initialize() - -# Build bundle source code. -if(BUILD_GDASBUNDLE) - -# jedi-cmake - ecbuild_bundle( PROJECT jedicmake SOURCE "./sorc/jedicmake" ) - include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) - -# ECMWF libraries - option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF - option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF - option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF - -# turn off optional OOPS toy models - option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) - option( ENABLE_QG_MODEL "Build QG toy model" OFF ) - - ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) - ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) - ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) - -# External (required) observation operators - # TODO remove the CRTM from here and use it as a library - #option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF - #ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) - ecbuild_bundle( PROJECT crtm SOURCE "./sorc/crtm" ) - -# Build GSI-B - option(BUILD_GSIBEC "Build GSI-B" OFF) - if(BUILD_GSIBEC) - ecbuild_bundle( PROJECT gsibec SOURCE "./sorc/gsibec" ) - endif() - -# Gibbs seawater - ecbuild_bundle( PROJECT gsw SOURCE "./sorc/gsw" ) - -# Core JEDI repositories - ecbuild_bundle( PROJECT oops SOURCE "./sorc/oops" ) - ecbuild_bundle( PROJECT vader SOURCE "./sorc/vader" ) - ecbuild_bundle( PROJECT saber SOURCE "./sorc/saber" ) - option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ioda SOURCE "./sorc/ioda" ) - option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ufo SOURCE "./sorc/ufo" ) - -# FMS and FV3 dynamical core - ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) - ecbuild_bundle( PROJECT fv3 SOURCE "./sorc/fv3" ) - -# fv3-jedi and associated repositories - ecbuild_bundle( PROJECT femps SOURCE "./sorc/femps" ) - ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./sorc/fv3-jedi-lm" ) - option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT fv3-jedi SOURCE "./sorc/fv3-jedi" ) - -# SOCA associated repositories -# TODO: Move the Icepack fork to EMC github - set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") - if ( BUILD_ICEPACK ) - ecbuild_bundle( PROJECT icepack SOURCE "./sorc/icepack" ) - endif() - ecbuild_bundle( PROJECT mom6 SOURCE "./sorc/mom6" ) - ecbuild_bundle( PROJECT soca SOURCE "./sorc/soca" ) - - # Build JEDI/DA or other peripherals - ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) - -# Build IODA converters - option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) - if(BUILD_IODA_CONVERTERS) - ecbuild_bundle( PROJECT iodaconv SOURCE "./sorc/iodaconv" ) - endif() - -# Land associated repositories - ecbuild_bundle( PROJECT land-imsproc SOURCE "./sorc/land-imsproc" ) - ecbuild_bundle( PROJECT land-jediincr SOURCE "./sorc/land-jediincr" ) - -# ioda, ufo, fv3-jedi, and saber test data -#--------------------------------- - if(CLONE_JCSDADATA) - - set(JCSDA_DATA_ROOT "$ENV{GDASAPP_TESTDATA}/jcsda") - ecbuild_bundle( PROJECT ioda-data SOURCE "${JCSDA_DATA_ROOT}/ioda-data" ) - ecbuild_bundle( PROJECT ufo-data SOURCE "${JCSDA_DATA_ROOT}/ufo-data" ) - ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "${JCSDA_DATA_ROOT}/fv3-jedi-data" ) - - endif(CLONE_JCSDADATA) - -endif(BUILD_GDASBUNDLE) +## Initialize bundle +## ----------------- +#ecbuild_bundle_initialize() +# +## Build bundle source code. +#if(BUILD_GDASBUNDLE) +# +## jedi-cmake +# ecbuild_bundle( PROJECT jedicmake SOURCE "./sorc/jedicmake" ) +# include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) +# +## ECMWF libraries +# option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF +# option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF +# option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF +# +## turn off optional OOPS toy models +# option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) +# option( ENABLE_QG_MODEL "Build QG toy model" OFF ) +# +# ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) +# ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) +# ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) +# +## External (required) observation operators +# # TODO remove the CRTM from here and use it as a library +# #option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF +# #ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) +# ecbuild_bundle( PROJECT crtm SOURCE "./sorc/crtm" ) +# +## Build GSI-B +# option(BUILD_GSIBEC "Build GSI-B" OFF) +# if(BUILD_GSIBEC) +# ecbuild_bundle( PROJECT gsibec SOURCE "./sorc/gsibec" ) +# endif() +# +## Gibbs seawater +# ecbuild_bundle( PROJECT gsw SOURCE "./sorc/gsw" ) +# +## Core JEDI repositories +# ecbuild_bundle( PROJECT oops SOURCE "./sorc/oops" ) +# ecbuild_bundle( PROJECT vader SOURCE "./sorc/vader" ) +# ecbuild_bundle( PROJECT saber SOURCE "./sorc/saber" ) +# option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) +# ecbuild_bundle( PROJECT ioda SOURCE "./sorc/ioda" ) +# option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) +# ecbuild_bundle( PROJECT ufo SOURCE "./sorc/ufo" ) +# +## FMS and FV3 dynamical core +# ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) +# ecbuild_bundle( PROJECT fv3 SOURCE "./sorc/fv3" ) +# +## fv3-jedi and associated repositories +# ecbuild_bundle( PROJECT femps SOURCE "./sorc/femps" ) +# ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./sorc/fv3-jedi-lm" ) +# option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) +# ecbuild_bundle( PROJECT fv3-jedi SOURCE "./sorc/fv3-jedi" ) +# +## SOCA associated repositories +## TODO: Move the Icepack fork to EMC github +# set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") +# if ( BUILD_ICEPACK ) +# ecbuild_bundle( PROJECT icepack SOURCE "./sorc/icepack" ) +# endif() +# ecbuild_bundle( PROJECT mom6 SOURCE "./sorc/mom6" ) +# ecbuild_bundle( PROJECT soca SOURCE "./sorc/soca" ) +# +# # Build JEDI/DA or other peripherals +# ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) +# +## Build IODA converters +# option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) +# if(BUILD_IODA_CONVERTERS) +# ecbuild_bundle( PROJECT iodaconv SOURCE "./sorc/iodaconv" ) +# endif() +# +## Land associated repositories +# ecbuild_bundle( PROJECT land-imsproc SOURCE "./sorc/land-imsproc" ) +# ecbuild_bundle( PROJECT land-jediincr SOURCE "./sorc/land-jediincr" ) +# +## ioda, ufo, fv3-jedi, and saber test data +##--------------------------------- +# if(CLONE_JCSDADATA) +# +# set(JCSDA_DATA_ROOT "$ENV{GDASAPP_TESTDATA}/jcsda") +# ecbuild_bundle( PROJECT ioda-data SOURCE "${JCSDA_DATA_ROOT}/ioda-data" ) +# ecbuild_bundle( PROJECT ufo-data SOURCE "${JCSDA_DATA_ROOT}/ufo-data" ) +# ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "${JCSDA_DATA_ROOT}/fv3-jedi-data" ) +# +# endif(CLONE_JCSDADATA) +# +#endif(BUILD_GDASBUNDLE) # Install utility scripts. add_subdirectory(ush) diff --git a/build.sh b/build.sh index 448bd0a50..473870d90 100755 --- a/build.sh +++ b/build.sh @@ -112,7 +112,7 @@ echo "Configuring ..." set -x cmake \ ${CMAKE_OPTS:-} \ - $dir_root + $dir_root/sorc set +x # Build diff --git a/sorc/CMakeLists.txt b/sorc/CMakeLists.txt new file mode 100644 index 000000000..f424da87f --- /dev/null +++ b/sorc/CMakeLists.txt @@ -0,0 +1,137 @@ +# ------------------------------------------------------------------------- # +# JEDI GDAS Bundle # +# ------------------------------------------------------------------------- # + +# Check for minimim cmake requirement +cmake_minimum_required( VERSION 3.20 FATAL_ERROR ) + +find_package(ecbuild 3.5 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) + +project(GDAS-bundle VERSION 1.0.0 LANGUAGES C CXX Fortran ) + +include(GNUInstallDirs) +enable_testing() + +# Build type. +if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$") + message(STATUS "Setting build type to 'Release' as none was specified.") + set(CMAKE_BUILD_TYPE + "Release" + CACHE STRING "Choose the type of build." FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") +endif() + +# Find dependencies. +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +# Include ecbuild_bundle macro +include( ecbuild_bundle ) + +# Enable MPI +set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) + +# Handle user options. +option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON) +option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF) +option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) + +# Initialize bundle +# ----------------- +ecbuild_bundle_initialize() + +# Build bundle source code. +if(BUILD_GDASBUNDLE) + +# jedi-cmake + ecbuild_bundle( PROJECT jedicmake SOURCE "./jedicmake" ) + include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) + +# ECMWF libraries + option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF + option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF + option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF + +# turn off optional OOPS toy models + option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) + option( ENABLE_QG_MODEL "Build QG toy model" OFF ) + + ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) + ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) + ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) + +# External (required) observation operators + # TODO remove the CRTM from here and use it as a library + #option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF + #ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) + ecbuild_bundle( PROJECT crtm SOURCE "./crtm" ) + +# Build GSI-B + option(BUILD_GSIBEC "Build GSI-B" OFF) + if(BUILD_GSIBEC) + ecbuild_bundle( PROJECT gsibec SOURCE "./gsibec" ) + endif() + +# Gibbs seawater + ecbuild_bundle( PROJECT gsw SOURCE "./gsw" ) + +# Core JEDI repositories + ecbuild_bundle( PROJECT oops SOURCE "./oops" ) + ecbuild_bundle( PROJECT vader SOURCE "./vader" ) + ecbuild_bundle( PROJECT saber SOURCE "./saber" ) + option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ioda SOURCE "./ioda" ) + option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT ufo SOURCE "./ufo" ) + +# FMS and FV3 dynamical core + ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) + ecbuild_bundle( PROJECT fv3 SOURCE "./fv3" ) + +# fv3-jedi and associated repositories + ecbuild_bundle( PROJECT femps SOURCE "./femps" ) + ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./fv3-jedi-lm" ) + option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) + ecbuild_bundle( PROJECT fv3-jedi SOURCE "./fv3-jedi" ) + +# SOCA associated repositories +# TODO: Move the Icepack fork to EMC github + set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") + if ( BUILD_ICEPACK ) + ecbuild_bundle( PROJECT icepack SOURCE "./icepack" ) + endif() + ecbuild_bundle( PROJECT mom6 SOURCE "./mom6" ) + ecbuild_bundle( PROJECT soca SOURCE "./soca" ) + + # Build JEDI/DA or other peripherals + ecbuild_bundle( PROJECT gdas-utils SOURCE "../utils" ) + +# Build IODA converters + option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) + if(BUILD_IODA_CONVERTERS) + ecbuild_bundle( PROJECT iodaconv SOURCE "./iodaconv" ) + endif() + +# Land associated repositories + ecbuild_bundle( PROJECT land-imsproc SOURCE "./land-imsproc" ) + ecbuild_bundle( PROJECT land-jediincr SOURCE "./land-jediincr" ) + +# GDASApp + ecbuild_bundle( PROJECT gdas SOURCE "../") + +# ioda, ufo, fv3-jedi, and saber test data +#--------------------------------- + if(CLONE_JCSDADATA) + + set(JCSDA_DATA_ROOT "$ENV{GDASAPP_TESTDATA}/jcsda") + ecbuild_bundle( PROJECT ioda-data SOURCE "${JCSDA_DATA_ROOT}/ioda-data" ) + ecbuild_bundle( PROJECT ufo-data SOURCE "${JCSDA_DATA_ROOT}/ufo-data" ) + ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "${JCSDA_DATA_ROOT}/fv3-jedi-data" ) + + endif(CLONE_JCSDADATA) + +endif(BUILD_GDASBUNDLE) + +# Finalize bundle +# --------------- +ecbuild_bundle_finalize() From f0feb8f9c6672a360490f2764c11f6b86bf6cb7c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 18 Dec 2023 14:42:18 +0000 Subject: [PATCH 19/37] add recursive clones everywhere for CI --- ci/driver.sh | 2 +- ci/gw_driver.sh | 2 +- ci/stable_driver.sh | 2 +- ci/validation/validation_driver.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/driver.sh b/ci/driver.sh index eb3a2854c..d1932cd47 100755 --- a/ci/driver.sh +++ b/ci/driver.sh @@ -64,7 +64,7 @@ for pr in $open_pr_list; do cd $GDAS_CI_ROOT/PR/$pr # clone copy of repo - git clone $repo_url + git clone --recursive $repo_url cd GDASApp # checkout pull request diff --git a/ci/gw_driver.sh b/ci/gw_driver.sh index a919b97de..a2dd781a4 100755 --- a/ci/gw_driver.sh +++ b/ci/gw_driver.sh @@ -65,7 +65,7 @@ for pr in $open_pr_list; do cd $GDAS_CI_ROOT/workflow/PR/$pr # clone global workflow develop branch - git clone $workflow_url + git clone --recursive $workflow_url # run checkout script for all other components cd $GDAS_CI_ROOT/workflow/PR/$pr/global-workflow/sorc diff --git a/ci/stable_driver.sh b/ci/stable_driver.sh index 31232837a..a5bac375d 100755 --- a/ci/stable_driver.sh +++ b/ci/stable_driver.sh @@ -56,7 +56,7 @@ mkdir -p $stableroot/$datestr cd $stableroot/$datestr # clone global workflow develop branch -git clone $workflow_url +git clone --recursive $workflow_url # run checkout script for all other components cd $stableroot/$datestr/global-workflow/sorc diff --git a/ci/validation/validation_driver.sh b/ci/validation/validation_driver.sh index 5e9868093..57f4c694a 100755 --- a/ci/validation/validation_driver.sh +++ b/ci/validation/validation_driver.sh @@ -55,7 +55,7 @@ mkdir -p $GDAS_CI_ROOT/validation/$today cd $GDAS_CI_ROOT/validation/$today # clone copy of repo -git clone $repo_url +git clone --recursive $repo_url cd GDASApp # load modules From 64c05de130ddd4743a6e6ecbf119beab2cac37d7 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 18 Dec 2023 16:34:30 +0000 Subject: [PATCH 20/37] Working maybe --- CMakeLists.txt | 2 +- sorc/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c64af831a..da5a24041 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,4 +137,4 @@ add_subdirectory(test) # Finalize bundle # --------------- -ecbuild_bundle_finalize() +#ecbuild_bundle_finalize() diff --git a/sorc/CMakeLists.txt b/sorc/CMakeLists.txt index f424da87f..8d30114d6 100644 --- a/sorc/CMakeLists.txt +++ b/sorc/CMakeLists.txt @@ -45,7 +45,7 @@ if(BUILD_GDASBUNDLE) # jedi-cmake ecbuild_bundle( PROJECT jedicmake SOURCE "./jedicmake" ) - include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) + include( jedicmake/cmake/Functions/git_functions.cmake ) # ECMWF libraries option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF From 3ba469d78b99eca2f4619789839102ce07dc2a44 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 18 Dec 2023 17:46:55 +0000 Subject: [PATCH 21/37] Working script to update hashes --- ush/submodules/update_develop.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/submodules/update_develop.sh b/ush/submodules/update_develop.sh index 403d97d5a..a6e9210bb 100755 --- a/ush/submodules/update_develop.sh +++ b/ush/submodules/update_develop.sh @@ -19,6 +19,6 @@ gdasdir=${1:-${my_dir}/../../} for r in $repos; do echo "Updating ${gdasdir}/sorc/${r}" - cd ${gdasdir}/sorc/${r} - git submodule update --remote --merge + cd ${gdasdir}/sorc + git submodule update --remote --merge ${r} done From 54fea318755bc17dce47399149b7f62c4bc0cc46 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 18 Dec 2023 21:26:39 +0000 Subject: [PATCH 22/37] Everything working but stable nightly --- .gitignore | 7 +++++++ .gitmodules | 4 ++++ sorc/CMakeLists.txt | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f32cad3e8..9b863f43b 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,10 @@ install*/ /ufo-data /vader /gdas-utils +/sorc/fv3-jedi-data +/sorc/ufo-data +/sorc/ioda-data +/sorc/test-data-release +/sorc/gdas +/sorc/gdas-utils + diff --git a/.gitmodules b/.gitmodules index f1fbf0f8a..c7bf485ae 100644 --- a/.gitmodules +++ b/.gitmodules @@ -72,3 +72,7 @@ [submodule "sorc/crtm"] path = sorc/crtm url = https://github.com/jcsda/crtm.git +[submodule "sorc/fms"] + path = sorc/fms + url = https://github.com/jcsda/FMS.git + branch = release-stable diff --git a/sorc/CMakeLists.txt b/sorc/CMakeLists.txt index 8d30114d6..435ca013b 100644 --- a/sorc/CMakeLists.txt +++ b/sorc/CMakeLists.txt @@ -85,7 +85,7 @@ if(BUILD_GDASBUNDLE) ecbuild_bundle( PROJECT ufo SOURCE "./ufo" ) # FMS and FV3 dynamical core - ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) + ecbuild_bundle( PROJECT fms SOURCE "./fms" ) ecbuild_bundle( PROJECT fv3 SOURCE "./fv3" ) # fv3-jedi and associated repositories From c38bdaa8bd6da2932713916ef84541938f0b3f91 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 18 Dec 2023 21:28:43 +0000 Subject: [PATCH 23/37] detabulate --- .gitmodules | 98 ++++++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/.gitmodules b/.gitmodules index c7bf485ae..18546c6d7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,77 +1,77 @@ [submodule "sorc/jedicmake"] - path = sorc/jedicmake - url = https://github.com/jcsda/jedi-cmake.git + path = sorc/jedicmake + url = https://github.com/jcsda/jedi-cmake.git [submodule "sorc/gsibec"] - path = sorc/gsibec - url = https://github.com/GEOS-ESM/GSIbec.git + path = sorc/gsibec + url = https://github.com/GEOS-ESM/GSIbec.git branch = develop [submodule "sorc/gsw"] - path = sorc/gsw - url = https://github.com/jcsda-internal/GSW-Fortran.git - branch = develop + path = sorc/gsw + url = https://github.com/jcsda-internal/GSW-Fortran.git + branch = develop [submodule "sorc/oops"] - path = sorc/oops - url = https://github.com/jcsda/oops.git - branch = develop + path = sorc/oops + url = https://github.com/jcsda/oops.git + branch = develop [submodule "sorc/vader"] - path = sorc/vader - url = https://github.com/jcsda/vader.git - branch = develop + path = sorc/vader + url = https://github.com/jcsda/vader.git + branch = develop [submodule "sorc/saber"] - path = sorc/saber - url = https://github.com/jcsda/saber.git - branch = develop + path = sorc/saber + url = https://github.com/jcsda/saber.git + branch = develop [submodule "sorc/ioda"] - path = sorc/ioda - url = https://github.com/jcsda/ioda.git - branch = develop + path = sorc/ioda + url = https://github.com/jcsda/ioda.git + branch = develop [submodule "sorc/ufo"] - path = sorc/ufo - url = https://github.com/jcsda/ufo.git - branch = develop + path = sorc/ufo + url = https://github.com/jcsda/ufo.git + branch = develop [submodule "sorc/fv3"] - path = sorc/fv3 - url = https://github.com/jcsda/GFDL_atmos_cubed_sphere.git + path = sorc/fv3 + url = https://github.com/jcsda/GFDL_atmos_cubed_sphere.git branch = release-stable [submodule "sorc/femps"] - path = sorc/femps - url = https://github.com/jcsda/femps.git + path = sorc/femps + url = https://github.com/jcsda/femps.git branch = develop [submodule "sorc/fv3-jedi-lm"] - path = sorc/fv3-jedi-lm - url = https://github.com/jcsda/fv3-jedi-linearmodel.git + path = sorc/fv3-jedi-lm + url = https://github.com/jcsda/fv3-jedi-linearmodel.git branch = develop [submodule "sorc/fv3-jedi"] - path = sorc/fv3-jedi - url = https://github.com/jcsda/fv3-jedi.git - branch = develop + path = sorc/fv3-jedi + url = https://github.com/jcsda/fv3-jedi.git + branch = develop [submodule "sorc/icepack"] - path = sorc/icepack - url = https://github.com/JCSDA-internal/Icepack.git + path = sorc/icepack + url = https://github.com/JCSDA-internal/Icepack.git branch = feature/ecbuild-new [submodule "sorc/mom6"] - path = sorc/mom6 - url = https://github.com/jcsda-internal/MOM6.git + path = sorc/mom6 + url = https://github.com/jcsda-internal/MOM6.git branch = main-ecbuild [submodule "sorc/soca"] - path = sorc/soca - url = https://github.com/jcsda-internal/soca.git - branch = develop + path = sorc/soca + url = https://github.com/jcsda-internal/soca.git + branch = develop [submodule "sorc/iodaconv"] - path = sorc/iodaconv - url = https://github.com/JCSDA-internal/ioda-converters.git - branch = develop + path = sorc/iodaconv + url = https://github.com/JCSDA-internal/ioda-converters.git + branch = develop [submodule "sorc/land-imsproc"] - path = sorc/land-imsproc - url = https://github.com/NOAA-PSL/land-IMS_proc.git - branch = develop + path = sorc/land-imsproc + url = https://github.com/NOAA-PSL/land-IMS_proc.git + branch = develop [submodule "sorc/land-jediincr"] - path = sorc/land-jediincr - url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git - branch = develop + path = sorc/land-jediincr + url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git + branch = develop [submodule "sorc/crtm"] - path = sorc/crtm - url = https://github.com/jcsda/crtm.git + path = sorc/crtm + url = https://github.com/jcsda/crtm.git [submodule "sorc/fms"] path = sorc/fms url = https://github.com/jcsda/FMS.git From 175c99e4122f509a51079d0a9f7afd57277b1343 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 19 Dec 2023 14:28:07 +0000 Subject: [PATCH 24/37] get the cmakelists to show up --- CMakeLists.txt | 99 +------------------------------------------------- 1 file changed, 1 insertion(+), 98 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index da5a24041..f4f2fd98f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------- # -# Application for all the components needed for the GDAS system # +# Application for the GDAS system # # ------------------------------------------------------------------------- # # Check for minimim cmake requirement @@ -36,105 +36,8 @@ option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON) option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF) option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF) -## Initialize bundle -## ----------------- -#ecbuild_bundle_initialize() -# -## Build bundle source code. -#if(BUILD_GDASBUNDLE) -# -## jedi-cmake -# ecbuild_bundle( PROJECT jedicmake SOURCE "./sorc/jedicmake" ) -# include( sorc/jedicmake/cmake/Functions/git_functions.cmake ) -# -## ECMWF libraries -# option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF -# option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Skip fckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF -# option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF -# -## turn off optional OOPS toy models -# option( ENABLE_LORENZ95_MODEL "Build LORENZ95 toy model" OFF ) -# option( ENABLE_QG_MODEL "Build QG toy model" OFF ) -# -# ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) -# ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) -# ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.35.0 ) -# -## External (required) observation operators -# # TODO remove the CRTM from here and use it as a library -# #option("BUNDLE_SKIP_CRTM" "Don't build CRTM" "OFF") # Don't build crtm unless user passes -DBUNDLE_SKIP_CRTM=OFF -# #ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/crtm.git" TAG v2.4.1-jedi.1 ) -# ecbuild_bundle( PROJECT crtm SOURCE "./sorc/crtm" ) -# -## Build GSI-B -# option(BUILD_GSIBEC "Build GSI-B" OFF) -# if(BUILD_GSIBEC) -# ecbuild_bundle( PROJECT gsibec SOURCE "./sorc/gsibec" ) -# endif() -# -## Gibbs seawater -# ecbuild_bundle( PROJECT gsw SOURCE "./sorc/gsw" ) -# -## Core JEDI repositories -# ecbuild_bundle( PROJECT oops SOURCE "./sorc/oops" ) -# ecbuild_bundle( PROJECT vader SOURCE "./sorc/vader" ) -# ecbuild_bundle( PROJECT saber SOURCE "./sorc/saber" ) -# option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) -# ecbuild_bundle( PROJECT ioda SOURCE "./sorc/ioda" ) -# option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) -# ecbuild_bundle( PROJECT ufo SOURCE "./sorc/ufo" ) -# -## FMS and FV3 dynamical core -# ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) -# ecbuild_bundle( PROJECT fv3 SOURCE "./sorc/fv3" ) -# -## fv3-jedi and associated repositories -# ecbuild_bundle( PROJECT femps SOURCE "./sorc/femps" ) -# ecbuild_bundle( PROJECT fv3-jedi-lm SOURCE "./sorc/fv3-jedi-lm" ) -# option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) -# ecbuild_bundle( PROJECT fv3-jedi SOURCE "./sorc/fv3-jedi" ) -# -## SOCA associated repositories -## TODO: Move the Icepack fork to EMC github -# set(BUILD_ICEPACK "ON" CACHE STRING "Build the icepack library") -# if ( BUILD_ICEPACK ) -# ecbuild_bundle( PROJECT icepack SOURCE "./sorc/icepack" ) -# endif() -# ecbuild_bundle( PROJECT mom6 SOURCE "./sorc/mom6" ) -# ecbuild_bundle( PROJECT soca SOURCE "./sorc/soca" ) -# -# # Build JEDI/DA or other peripherals -# ecbuild_bundle( PROJECT gdas-utils SOURCE "./utils" ) -# -## Build IODA converters -# option(BUILD_IODA_CONVERTERS "Build IODA Converters" ON) -# if(BUILD_IODA_CONVERTERS) -# ecbuild_bundle( PROJECT iodaconv SOURCE "./sorc/iodaconv" ) -# endif() -# -## Land associated repositories -# ecbuild_bundle( PROJECT land-imsproc SOURCE "./sorc/land-imsproc" ) -# ecbuild_bundle( PROJECT land-jediincr SOURCE "./sorc/land-jediincr" ) -# -## ioda, ufo, fv3-jedi, and saber test data -##--------------------------------- -# if(CLONE_JCSDADATA) -# -# set(JCSDA_DATA_ROOT "$ENV{GDASAPP_TESTDATA}/jcsda") -# ecbuild_bundle( PROJECT ioda-data SOURCE "${JCSDA_DATA_ROOT}/ioda-data" ) -# ecbuild_bundle( PROJECT ufo-data SOURCE "${JCSDA_DATA_ROOT}/ufo-data" ) -# ecbuild_bundle( PROJECT fv3-jedi-data SOURCE "${JCSDA_DATA_ROOT}/fv3-jedi-data" ) -# -# endif(CLONE_JCSDADATA) -# -#endif(BUILD_GDASBUNDLE) - # Install utility scripts. add_subdirectory(ush) # Include testing. add_subdirectory(test) - -# Finalize bundle -# --------------- -#ecbuild_bundle_finalize() From 64192bf7b6a31fd77fa2ce7c251a4eeb8699e46b Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 19 Dec 2023 16:39:42 +0000 Subject: [PATCH 25/37] FMS submodule issue --- .gitmodules | 4 ++-- sorc/fms | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 160000 sorc/fms diff --git a/.gitmodules b/.gitmodules index 18546c6d7..ce3d33e2c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -73,6 +73,6 @@ path = sorc/crtm url = https://github.com/jcsda/crtm.git [submodule "sorc/fms"] - path = sorc/fms - url = https://github.com/jcsda/FMS.git + path = sorc/fms + url = https://github.com/jcsda/fms.git branch = release-stable diff --git a/sorc/fms b/sorc/fms new file mode 160000 index 000000000..1f739141e --- /dev/null +++ b/sorc/fms @@ -0,0 +1 @@ +Subproject commit 1f739141ef8b000a0bd75ae8bebfadea340299ba From 88175e2151d1f1ccf0dc7977d69808e2f3bc22cb Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 20 Dec 2023 14:16:20 +0000 Subject: [PATCH 26/37] cmake fix hopefully --- test/atm/CMakeLists.txt | 2 +- test/land/test_bufr2ioda.sh | 2 +- test/soca/test_bufr2ioda.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/atm/CMakeLists.txt b/test/atm/CMakeLists.txt index 117e2ea8e..dc6f5cdf8 100644 --- a/test/atm/CMakeLists.txt +++ b/test/atm/CMakeLists.txt @@ -7,7 +7,7 @@ if (BUILD_GDASBUNDLE) file(CREATE_LINK ${PROJECT_SOURCE_DIR}/iodaconv/test/testinput/gdas.t06z.adpsfc.tm00.bufr_d ${PROJECT_BINARY_DIR}/test/testdata/gdas.t06z.adpsfc.tm00.bufr_d SYMBOLIC) # test convert BUFR to IODA add_test(NAME test_gdasapp_convert_bufr_adpsfc - COMMAND ${PROJECT_BINARY_DIR}/bin/bufr2ioda.x ${PROJECT_BINARY_DIR}/test/testinput/bufr_adpsfc.yaml + COMMAND ${CMAKE_BINARY_DIR}/bin/bufr2ioda.x ${PROJECT_BINARY_DIR}/test/testinput/bufr_adpsfc.yaml WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) # test generate UFO CRTM sat bias files from GSI sat bias files diff --git a/test/land/test_bufr2ioda.sh b/test/land/test_bufr2ioda.sh index f21e30f2f..bdb5ecd75 100755 --- a/test/land/test_bufr2ioda.sh +++ b/test/land/test_bufr2ioda.sh @@ -4,7 +4,7 @@ set -x PROJECT_BINARY_DIR=${1} OBSSOURCE=${2} -CMD=${PROJECT_BINARY_DIR}/bin/bufr2ioda.x +CMD=${CMAKE_BINARY_DIR}/bin/bufr2ioda.x OBSYAML=${PROJECT_BINARY_DIR}/../ush/land/${OBSSOURCE}.yaml OUTFILE=`grep obsdataout ${OBSYAML} | cut -d'"' -f2` diff --git a/test/soca/test_bufr2ioda.sh b/test/soca/test_bufr2ioda.sh index 370e8639e..291694698 100755 --- a/test/soca/test_bufr2ioda.sh +++ b/test/soca/test_bufr2ioda.sh @@ -4,7 +4,7 @@ set -x PROJECT_BINARY_DIR=${1} OBSSOURCE=${2} -CMD=${PROJECT_BINARY_DIR}/bin/bufr2ioda.x +CMD=${CMAKE_BINARY_DIR}/bin/bufr2ioda.x OBSYAML=${PROJECT_BINARY_DIR}/test/testinput/${OBSSOURCE}.yaml OUTFILE=`grep obsdataout ${OBSYAML} | cut -d'"' -f2` From c103f29ed9358782220cbf048a4abbadd63fe8ee Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 20 Dec 2023 16:52:16 +0000 Subject: [PATCH 27/37] some cmake changes --- test/atm/CMakeLists.txt | 2 +- test/land/CMakeLists.txt | 2 +- test/land/test_bufr2ioda.sh | 3 ++- test/soca/CMakeLists.txt | 16 ++++++++-------- test/soca/test_bufr2ioda.sh | 3 ++- utils/test/CMakeLists.txt | 6 +++--- utils/test/testinput/gdas_meanioda.yaml | 2 +- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/test/atm/CMakeLists.txt b/test/atm/CMakeLists.txt index dc6f5cdf8..a7b582f78 100644 --- a/test/atm/CMakeLists.txt +++ b/test/atm/CMakeLists.txt @@ -4,7 +4,7 @@ if (BUILD_GDASBUNDLE) # link input file from iodaconv to test directory - file(CREATE_LINK ${PROJECT_SOURCE_DIR}/iodaconv/test/testinput/gdas.t06z.adpsfc.tm00.bufr_d ${PROJECT_BINARY_DIR}/test/testdata/gdas.t06z.adpsfc.tm00.bufr_d SYMBOLIC) + file(CREATE_LINK ${CMAKE_SOURCE_DIR}/sorc/iodaconv/test/testinput/gdas.t06z.adpsfc.tm00.bufr_d ${PROJECT_BINARY_DIR}/test/testdata/gdas.t06z.adpsfc.tm00.bufr_d SYMBOLIC) # test convert BUFR to IODA add_test(NAME test_gdasapp_convert_bufr_adpsfc COMMAND ${CMAKE_BINARY_DIR}/bin/bufr2ioda.x ${PROJECT_BINARY_DIR}/test/testinput/bufr_adpsfc.yaml diff --git a/test/land/CMakeLists.txt b/test/land/CMakeLists.txt index edaff5a9f..5bd63a522 100644 --- a/test/land/CMakeLists.txt +++ b/test/land/CMakeLists.txt @@ -32,6 +32,6 @@ set_tests_properties(test_gdasapp_land_letkfoi_snowda # Test convert BUFR to IODA add_test(NAME test_gdasapp_convert_bufr_adpsfc_snow - COMMAND ${PROJECT_SOURCE_DIR}/test/land/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} bufr_adpsfc_snow + COMMAND ${PROJECT_SOURCE_DIR}/test/land/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} bufr_adpsfc_snow WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) diff --git a/test/land/test_bufr2ioda.sh b/test/land/test_bufr2ioda.sh index bdb5ecd75..8931e8da5 100755 --- a/test/land/test_bufr2ioda.sh +++ b/test/land/test_bufr2ioda.sh @@ -3,7 +3,8 @@ set -x PROJECT_BINARY_DIR=${1} -OBSSOURCE=${2} +CMAKE_BINARY_DIR=${1} +OBSSOURCE=${3} CMD=${CMAKE_BINARY_DIR}/bin/bufr2ioda.x OBSYAML=${PROJECT_BINARY_DIR}/../ush/land/${OBSSOURCE}.yaml diff --git a/test/soca/CMakeLists.txt b/test/soca/CMakeLists.txt index 44f08cbea..887da8e67 100644 --- a/test/soca/CMakeLists.txt +++ b/test/soca/CMakeLists.txt @@ -6,35 +6,35 @@ set( TESTDATA ${PROJECT_BINARY_DIR}/test/testdata ) # test convert BUFR to IODA add_test(NAME test_gdasapp_convert_bufr_temp_dbuoy - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} temp_bufr_dbuoyprof + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} temp_bufr_dbuoyprof WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) add_test(NAME test_gdasapp_convert_bufr_salt_dbuoy - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} salt_bufr_dbuoyprof + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} salt_bufr_dbuoyprof WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) add_test(NAME test_gdasapp_convert_bufr_temp_mbuoyb - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} temp_bufr_mbuoybprof + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} temp_bufr_mbuoybprof WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) add_test(NAME test_gdasapp_convert_bufr_salt_mbuoyb - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} salt_bufr_mbuoybprof + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} salt_bufr_mbuoybprof WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) add_test(NAME test_gdasapp_convert_bufr_tesacprof - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} bufr_tesacprof + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} bufr_tesacprof WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) add_test(NAME test_gdasapp_convert_bufr_trkobprof - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} bufr_trkobprof + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} bufr_trkobprof WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) add_test(NAME test_gdasapp_convert_bufr_sfcships - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} bufr_sfcships + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} bufr_sfcships WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) add_test(NAME test_gdasapp_convert_bufr_sfcshipsu - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} bufr_sfcshipsu + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} bufr_sfcshipsu WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) # Symlink test input yaml files diff --git a/test/soca/test_bufr2ioda.sh b/test/soca/test_bufr2ioda.sh index 291694698..a6a9c0dbd 100755 --- a/test/soca/test_bufr2ioda.sh +++ b/test/soca/test_bufr2ioda.sh @@ -3,7 +3,8 @@ set -x PROJECT_BINARY_DIR=${1} -OBSSOURCE=${2} +CMAKE_BINARY_DIR=${2} +OBSSOURCE=${3} CMD=${CMAKE_BINARY_DIR}/bin/bufr2ioda.x OBSYAML=${PROJECT_BINARY_DIR}/test/testinput/${OBSSOURCE}.yaml diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index cf74db8aa..9453705df 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -1,4 +1,4 @@ -# Create Data directory for test input config and symlink all files +#eT Create Data directory for test input config and symlink all files list( APPEND utils_test_input testinput/gdas_meanioda.yaml testinput/gdas_rads2ioda.yaml @@ -45,8 +45,8 @@ ecbuild_add_test( TARGET test_gdasapp_util_ioda_example # Prepare data for the IODA converters file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) ecbuild_add_test( TARGET test_gdasapp_util_prepdata - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/test/prepdata.sh - ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../../utils/test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/../../gdas-utils/test/prepdata.sh + ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../../gdas-utils/test WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) # Test the RADS to IODA converter diff --git a/utils/test/testinput/gdas_meanioda.yaml b/utils/test/testinput/gdas_meanioda.yaml index 555ca25e7..91cdbf5b8 100644 --- a/utils/test/testinput/gdas_meanioda.yaml +++ b/utils/test/testinput/gdas_meanioda.yaml @@ -8,7 +8,7 @@ obs space: obsdatain: engine: type: H5File - obsfile: ../../../soca/test/Data/obs/gmi_gpm_obs.nc + obsfile: ../../../sorc/soca/test/Data/obs/gmi_gpm_obs.nc # the below 2 lines are not used but needed by the IODA obsspace it seems... simulated variables: [brightnessTemperature] observed variables: [brightnessTemperature] From 4e727b17e93f5519fb6713a9eb5b58badd4eaeea Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 20 Dec 2023 13:54:11 -0500 Subject: [PATCH 28/37] Fixed utils ctests (#820) That should fix the tests under gdas-utils --- utils/test/CMakeLists.txt | 4 ++-- utils/test/prepdata.sh | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index dd9e89c95..fcbb26e37 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -45,8 +45,8 @@ ecbuild_add_test( TARGET test_gdasapp_util_ioda_example # Prepare data for the IODA converters file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) ecbuild_add_test( TARGET test_gdasapp_util_prepdata - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/../../gdas-utils/test/prepdata.sh - ARGS ${CMAKE_CURRENT_SOURCE_DIR}/../../gdas-utils/test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/prepdata.sh + ARGS ${CMAKE_CURRENT_SOURCE_DIR} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) # Test the RADS to IODA converter diff --git a/utils/test/prepdata.sh b/utils/test/prepdata.sh index 0aaac084b..b7ca4a96e 100755 --- a/utils/test/prepdata.sh +++ b/utils/test/prepdata.sh @@ -17,17 +17,17 @@ cdl2nc4() { project_source_dir=$1 -cdl2nc4 ADT/rads_adt_3a_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3a_2021181.cdl -cdl2nc4 ADT/rads_adt_3b_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3b_2021181.cdl -cdl2nc4 icec/icec_amsr2_north_1.nc4 ${project_source_dir}/testdata/icec_amsr2_north_1.cdl -cdl2nc4 icec/icec_amsr2_north_2.nc4 ${project_source_dir}/testdata/icec_amsr2_north_2.cdl -cdl2nc4 icec/icec_amsr2_south_1.nc4 ${project_source_dir}/testdata/icec_amsr2_south_1.cdl -cdl2nc4 icec/icec_amsr2_south_2.nc4 ${project_source_dir}/testdata/icec_amsr2_south_2.cdl -cdl2nc4 SSS/sss_smap_1.nc4 ${project_source_dir}/testdata/sss_smap_1.cdl -cdl2nc4 SSS/sss_smap_2.nc4 ${project_source_dir}/testdata/sss_smap_2.cdl -cdl2nc4 SSS/sss_smos_1.nc4 ${project_source_dir}/testdata/sss_smos_1.cdl -cdl2nc4 SSS/sss_smos_2.nc4 ${project_source_dir}/testdata/sss_smos_2.cdl -cdl2nc4 sst/ghrsst_sst_mb_202107010000.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010000.cdl -cdl2nc4 sst/ghrsst_sst_mb_202107010100.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl -cdl2nc4 sst/viirs_aod_1.nc4 ${project_source_dir}/testdata/viirs_aod_1.cdl -cdl2nc4 sst/viirs_aod_2.nc4 ${project_source_dir}/testdata/viirs_aod_2.cdl +cdl2nc4 rads_adt_3a_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3a_2021181.cdl +cdl2nc4 rads_adt_3b_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3b_2021181.cdl +cdl2nc4 icec_amsr2_north_1.nc4 ${project_source_dir}/testdata/icec_amsr2_north_1.cdl +cdl2nc4 icec_amsr2_north_2.nc4 ${project_source_dir}/testdata/icec_amsr2_north_2.cdl +cdl2nc4 icec_amsr2_south_1.nc4 ${project_source_dir}/testdata/icec_amsr2_south_1.cdl +cdl2nc4 icec_amsr2_south_2.nc4 ${project_source_dir}/testdata/icec_amsr2_south_2.cdl +cdl2nc4 sss_smap_1.nc4 ${project_source_dir}/testdata/sss_smap_1.cdl +cdl2nc4 sss_smap_2.nc4 ${project_source_dir}/testdata/sss_smap_2.cdl +cdl2nc4 sss_smos_1.nc4 ${project_source_dir}/testdata/sss_smos_1.cdl +cdl2nc4 sss_smos_2.nc4 ${project_source_dir}/testdata/sss_smos_2.cdl +cdl2nc4 ghrsst_sst_mb_202107010000.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010000.cdl +cdl2nc4 ghrsst_sst_mb_202107010100.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl +cdl2nc4 viirs_aod_1.nc4 ${project_source_dir}/testdata/viirs_aod_1.cdl +cdl2nc4 viirs_aod_2.nc4 ${project_source_dir}/testdata/viirs_aod_2.cdl From b052a8a0ae115ff2b2cc18c684282c6bf9b725a7 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 20 Dec 2023 19:21:36 +0000 Subject: [PATCH 29/37] correct path to Data/fv3files in letkfoi_snowda.sh --- test/land/letkfoi_snowda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/land/letkfoi_snowda.sh b/test/land/letkfoi_snowda.sh index 64086ba07..d21672dd9 100755 --- a/test/land/letkfoi_snowda.sh +++ b/test/land/letkfoi_snowda.sh @@ -66,7 +66,7 @@ fi mkdir -p Data diags mkdir -p Data/fieldmetadata ln -s ${project_source_dir}/parm/io/fv3jedi_fieldmetadata_restart.yaml Data/fieldmetadata/. -ln -s ${project_binary_dir}/fv3-jedi/test/Data/fv3files Data/fv3files +ln -s ${project_source_dir}/sorc/fv3-jedi/test/Data/fv3files Data/fv3files ln -s ${project_source_dir}/test/land/letkfoi_land.yaml letkf_land.yaml ln -s ${OBSDIR}/snow_depth/GTS/202103/adpsfc_snow_2021032318.nc4 adpsfc_snow.nc4 ln -s ${OBSDIR} Data/land From b7640dcb5e8c0ab02630b10ad8ca25aebf556a00 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 21 Dec 2023 15:48:29 +0000 Subject: [PATCH 30/37] updates to get convert_bufr_adpsfc ctests working --- test/atm/CMakeLists.txt | 2 +- test/land/CMakeLists.txt | 2 +- test/land/test_bufr2ioda.sh | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/atm/CMakeLists.txt b/test/atm/CMakeLists.txt index a7b582f78..bb34fa4fb 100644 --- a/test/atm/CMakeLists.txt +++ b/test/atm/CMakeLists.txt @@ -4,7 +4,7 @@ if (BUILD_GDASBUNDLE) # link input file from iodaconv to test directory - file(CREATE_LINK ${CMAKE_SOURCE_DIR}/sorc/iodaconv/test/testinput/gdas.t06z.adpsfc.tm00.bufr_d ${PROJECT_BINARY_DIR}/test/testdata/gdas.t06z.adpsfc.tm00.bufr_d SYMBOLIC) + file(CREATE_LINK ${CMAKE_SOURCE_DIR}/iodaconv/test/testinput/gdas.t06z.adpsfc.tm00.bufr_d ${PROJECT_BINARY_DIR}/test/testdata/gdas.t06z.adpsfc.tm00.bufr_d SYMBOLIC) # test convert BUFR to IODA add_test(NAME test_gdasapp_convert_bufr_adpsfc COMMAND ${CMAKE_BINARY_DIR}/bin/bufr2ioda.x ${PROJECT_BINARY_DIR}/test/testinput/bufr_adpsfc.yaml diff --git a/test/land/CMakeLists.txt b/test/land/CMakeLists.txt index 5bd63a522..2f8879768 100644 --- a/test/land/CMakeLists.txt +++ b/test/land/CMakeLists.txt @@ -32,6 +32,6 @@ set_tests_properties(test_gdasapp_land_letkfoi_snowda # Test convert BUFR to IODA add_test(NAME test_gdasapp_convert_bufr_adpsfc_snow - COMMAND ${PROJECT_SOURCE_DIR}/test/land/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${CMAKE_BINARY_DIR} bufr_adpsfc_snow + COMMAND ${PROJECT_SOURCE_DIR}/test/land/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} ${CMAKE_BINARY_DIR} bufr_adpsfc_snow WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) diff --git a/test/land/test_bufr2ioda.sh b/test/land/test_bufr2ioda.sh index 8931e8da5..e4e3b6235 100755 --- a/test/land/test_bufr2ioda.sh +++ b/test/land/test_bufr2ioda.sh @@ -3,10 +3,11 @@ set -x PROJECT_BINARY_DIR=${1} -CMAKE_BINARY_DIR=${1} -OBSSOURCE=${3} +PROJECT_SOURCE_DIR=${2} +CMAKE_BINARY_DIR=${3} +OBSSOURCE=${4} CMD=${CMAKE_BINARY_DIR}/bin/bufr2ioda.x -OBSYAML=${PROJECT_BINARY_DIR}/../ush/land/${OBSSOURCE}.yaml +OBSYAML=${PROJECT_SOURCE_DIR}/ush/land/${OBSSOURCE}.yaml OUTFILE=`grep obsdataout ${OBSYAML} | cut -d'"' -f2` From 8f4f7e915ef39c705665fce1f8ecca2186281c18 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 22 Dec 2023 20:16:19 +0000 Subject: [PATCH 31/37] modify how HOMEgfs is set for atm var and ens ctests --- test/atm/global-workflow/config.yaml | 1 + test/atm/global-workflow/jjob_ens_final.sh | 5 +++-- test/atm/global-workflow/jjob_ens_init.sh | 5 +++-- test/atm/global-workflow/jjob_ens_run.sh | 5 +++-- test/atm/global-workflow/jjob_var_final.sh | 5 +++-- test/atm/global-workflow/jjob_var_init.sh | 5 +++-- test/atm/global-workflow/jjob_var_run.sh | 5 +++-- test/atm/global-workflow/setup_workflow_exp.sh | 5 ++--- 8 files changed, 21 insertions(+), 15 deletions(-) diff --git a/test/atm/global-workflow/config.yaml b/test/atm/global-workflow/config.yaml index 430d73eda..ec37c4f79 100644 --- a/test/atm/global-workflow/config.yaml +++ b/test/atm/global-workflow/config.yaml @@ -1,4 +1,5 @@ base: + HOMEgfs: "@topdir@" DO_JEDIATMVAR: "YES" DO_JEDIATMENS: "YES" DATAPATH: "@bindir@/test/atm/global-workflow/testrun" diff --git a/test/atm/global-workflow/jjob_ens_final.sh b/test/atm/global-workflow/jjob_ens_final.sh index ae1928a18..d1527f6d6 100755 --- a/test/atm/global-workflow/jjob_ens_final.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -5,7 +5,8 @@ bindir=$1 srcdir=$2 # Set g-w HOMEgfs -export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) +export HOMEgfs=$topdir # Set variables for ctest export PSLOT=gdas_test @@ -25,7 +26,7 @@ export NMEM_ENS=3 export ACCOUNT=da-cpu # Set python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh index 8966f2b69..4d23ad41f 100755 --- a/test/atm/global-workflow/jjob_ens_init.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -5,7 +5,8 @@ bindir=$1 srcdir=$2 # Set g-w HOMEgfs -export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) +export HOMEgfs=$topdir # Set variables for ctest export PSLOT=gdas_test @@ -30,7 +31,7 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/parm/config/gfs/config.com" # Set python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH diff --git a/test/atm/global-workflow/jjob_ens_run.sh b/test/atm/global-workflow/jjob_ens_run.sh index 597afcd43..3da3a3ec1 100755 --- a/test/atm/global-workflow/jjob_ens_run.sh +++ b/test/atm/global-workflow/jjob_ens_run.sh @@ -5,7 +5,8 @@ bindir=$1 srcdir=$2 # Set g-w HOMEgfs -export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) +export HOMEgfs=$topdir # Set variables for ctest export PSLOT=gdas_test @@ -28,7 +29,7 @@ export NMEM_ENS=3 export ACCOUNT=da-cpu # Set python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH diff --git a/test/atm/global-workflow/jjob_var_final.sh b/test/atm/global-workflow/jjob_var_final.sh index f9b64afbe..62cf4a1cf 100755 --- a/test/atm/global-workflow/jjob_var_final.sh +++ b/test/atm/global-workflow/jjob_var_final.sh @@ -5,7 +5,8 @@ bindir=$1 srcdir=$2 # Set g-w HOMEgfs -export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) +export HOMEgfs=$topdir # Set variables for ctest export PSLOT=gdas_test @@ -25,7 +26,7 @@ export NMEM_ENS=0 export ACCOUNT=da-cpu # Set python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH diff --git a/test/atm/global-workflow/jjob_var_init.sh b/test/atm/global-workflow/jjob_var_init.sh index 0d88c2174..9740bf785 100755 --- a/test/atm/global-workflow/jjob_var_init.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -5,7 +5,8 @@ bindir=$1 srcdir=$2 # Set g-w HOMEgfs -export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) +export HOMEgfs=$topdir # Set variables for ctest export PSLOT=gdas_test @@ -30,7 +31,7 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/parm/config/gfs/config.com" # Set python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH diff --git a/test/atm/global-workflow/jjob_var_run.sh b/test/atm/global-workflow/jjob_var_run.sh index a78b1e10e..8828f1b60 100755 --- a/test/atm/global-workflow/jjob_var_run.sh +++ b/test/atm/global-workflow/jjob_var_run.sh @@ -5,7 +5,8 @@ bindir=$1 srcdir=$2 # Set g-w HOMEgfs -export HOMEgfs=$srcdir/../../ # TODO: HOMEgfs had to be hard-coded in config +topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) +export HOMEgfs=$topdir # Set variables for ctest export PSLOT=gdas_test @@ -28,7 +29,7 @@ export NMEM_ENS=0 export ACCOUNT=da-cpu # Set python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index 7eb6751db..64cecc3da 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -3,6 +3,7 @@ set -x # ctest to create an experiment directory for global-workflow bindir=$1 srcdir=$2 +topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) # test experiment variables idate=2021032312 @@ -27,14 +28,12 @@ cp $srcdir/test/atm/global-workflow/config.atmanl config/ cp $srcdir/test/atm/global-workflow/config.yaml . # update paths in config.yaml +sed -i -e "s~@topdir@~${topdir}~g" config.yaml sed -i -e "s~@bindir@~${bindir}~g" config.yaml sed -i -e "s~@srcdir@~${srcdir}~g" config.yaml sed -i -e "s~@dumpdir@~${GDASAPP_TESTDATA}/lowres~g" config.yaml # run the script -ln -sf $srcdir/../../workflow/setup_expt.py . - - echo "Running global-workflow experiment generation script" $srcdir/../../workflow/setup_expt.py gfs cycled --idate $idate \ --edate $edate \ From f19bdd944a2a202f3621cae2b7adb7c09e455e5a Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:46:15 -0500 Subject: [PATCH 32/37] update paths in soca scripts and tests (#845) Update soca paths to be consistent with use of submodules --- scripts/exgdas_global_marine_analysis_prep.py | 24 +++++++++++-------- scripts/exgdas_global_marine_analysis_vrfy.py | 6 ++--- scripts/exglobal_prep_ocean_obs.py | 2 +- test/soca/gw/CMakeLists.txt | 8 +++---- test/soca/gw/prep.sh | 2 +- test/soca/gw/run_jjobs.yaml.test | 12 +++++----- test/soca/gw/static.sh | 4 ++-- 7 files changed, 31 insertions(+), 27 deletions(-) diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py index b5e7117fc..93df813e4 100755 --- a/scripts/exgdas_global_marine_analysis_prep.py +++ b/scripts/exgdas_global_marine_analysis_prep.py @@ -85,9 +85,10 @@ def cice_hist2fms(input_filename, output_filename): """ Simple reformatting utility to allow soca/fms to read CICE's history """ + input_filename_real = os.path.realpath(input_filename) # open the CICE history file - ds = xr.open_dataset(input_filename) + ds = xr.open_dataset(input_filename_real) if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables: logging.info(f"*** Already reformatted, skipping.") @@ -100,7 +101,8 @@ def cice_hist2fms(input_filename, output_filename): ds = ds.rename({'aice_h': 'aicen', 'hi_h': 'hicen', 'hs_h': 'hsnon'}) # Save the new netCDF file - ds.to_netcdf(output_filename, mode='w') + output_filename_real = os.path.realpath(output_filename) + ds.to_netcdf(output_filename_real, mode='w') def test_hist_date(histfile, ref_date): @@ -280,7 +282,8 @@ def find_clim_ens(input_date): # concatenate altimeters into one obs space # TODO (SAMG)temporary, move this into the obs procecing eventually -adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt" +com_obs = os.path.realpath(f"{os.getenv('COM_OBS')}") +adt_obs = f"{com_obs}/{RUN}.t{cyc}z.adt" ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True) # get the list of observations @@ -293,7 +296,7 @@ def find_clim_ens(input_date): for obs_file in obs_files: logging.info(f"******* {obs_file}") obs_src = os.path.join(os.getenv('COM_OBS'), obs_file) - obs_dst = os.path.join(os.path.abspath(obs_in), obs_file) + obs_dst = os.path.join(os.path.realpath(obs_in), obs_file) if os.path.exists(obs_src): logging.info(f"******* fetching {obs_file}") obs_list.append([obs_src, obs_dst]) @@ -319,20 +322,21 @@ def find_clim_ens(input_date): for mem in range(1, nmem_ens+1): for domain in ['ocn', 'ice']: # TODO(Guillaume): make use and define ensemble COM in the j-job + ensroot = os.getenv('COM_OCEAN_HISTORY_PREV') ensdir = os.path.join(os.getenv('COM_OCEAN_HISTORY_PREV'), '..', '..', '..', '..', '..', f'enkf{RUN}.{PDY}', f'{gcyc}', f'mem{str(mem).zfill(3)}', 'model_data', longname[domain], 'history') - ensdir = os.path.normpath(ensdir) + ensdir_real = os.path.realpath(ensdir) f009 = f'enkfgdas.t{gcyc}z.{domain}f009.nc' - fname_in = os.path.abspath(os.path.join(ensdir, f009)) - fname_out = os.path.abspath(os.path.join(static_ens, domain+"."+str(mem)+".nc")) + fname_in = os.path.abspath(os.path.join(ensdir_real, f009)) + fname_out = os.path.realpath(os.path.join(static_ens, domain+"."+str(mem)+".nc")) ens_member_list.append([fname_in, fname_out]) FileHandler({'copy': ens_member_list}).sync() # reformat the cice history output for mem in range(1, nmem_ens+1): - cice_fname = os.path.abspath(os.path.join(static_ens, "ice."+str(mem)+".nc")) + cice_fname = os.path.realpath(os.path.join(static_ens, "ice."+str(mem)+".nc")) cice_hist2fms(cice_fname, cice_fname) else: logging.info("---------------- Stage offline ensemble members") @@ -357,8 +361,8 @@ def find_clim_ens(input_date): # copy yaml for grid generation logging.info(f"---------------- generate gridgen.yaml") -gridgen_yaml_src = os.path.abspath(os.path.join(gdas_home, 'parm', 'soca', 'gridgen', 'gridgen.yaml')) -gridgen_yaml_dst = os.path.abspath(os.path.join(stage_cfg['stage_dir'], 'gridgen.yaml')) +gridgen_yaml_src = os.path.realpath(os.path.join(gdas_home, 'parm', 'soca', 'gridgen', 'gridgen.yaml')) +gridgen_yaml_dst = os.path.realpath(os.path.join(stage_cfg['stage_dir'], 'gridgen.yaml')) FileHandler({'copy': [[gridgen_yaml_src, gridgen_yaml_dst]]}).sync() ################################################################################ diff --git a/scripts/exgdas_global_marine_analysis_vrfy.py b/scripts/exgdas_global_marine_analysis_vrfy.py index 6179c00e7..11a9694d9 100755 --- a/scripts/exgdas_global_marine_analysis_vrfy.py +++ b/scripts/exgdas_global_marine_analysis_vrfy.py @@ -26,9 +26,9 @@ import subprocess from datetime import datetime, timedelta -comout = os.getenv('COM_OCEAN_ANALYSIS') -com_ice_history = os.getenv('COM_ICE_HISTORY_PREV') -com_ocean_history = os.getenv('COM_OCEAN_HISTORY_PREV') +comout = os.path.realpath(os.getenv('COM_OCEAN_ANALYSIS')) +com_ice_history = os.path.realpath(os.getenv('COM_ICE_HISTORY_PREV')) +com_ocean_history = os.path.realpath(os.getenv('COM_OCEAN_HISTORY_PREV')) cyc = os.getenv('cyc') RUN = os.getenv('RUN') gcyc = str((int(cyc) - 6) % 24).zfill(2) diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 1d9bd81f0..ac99ed0bd 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta import logging import os -import prep_marine_obs +from soca import prep_marine_obs import subprocess from wxflow import YAMLFile, save_as_yaml, FileHandler diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index 68a5035a7..532c25217 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -27,7 +27,7 @@ add_test(NAME test_gdasapp_soca_concatioda WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/concatioda) set_tests_properties(test_gdasapp_soca_concatioda PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") + ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") # Clean-up add_test(NAME test_gdasapp_soca_run_clean @@ -78,7 +78,7 @@ foreach(jjob ${jjob_list}) WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun) set_tests_properties(${test_name} PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") + ENVIRONMENT "PYTHONPATH=${PROJECT_SOURCE_DIR}/ush:${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") set(setup "--skip") # Only run the setup of the first test, if not, it will hang @@ -89,7 +89,7 @@ endforeach() set(ctest_list "socahybridweights" "incr_handler" "ens_handler") foreach(ctest ${ctest_list}) set(TEST ${ctest}) - set(EXEC ${PROJECT_BINARY_DIR}/bin/gdas_${ctest}.x) + set(EXEC ${PROJECT_BINARY_DIR}/../bin/gdas_${ctest}.x) set(YAML ${PROJECT_SOURCE_DIR}/test/soca/testinput/${ctest}.yaml) configure_file(${PROJECT_SOURCE_DIR}/test/soca/gw/run_gdas_apps.yaml.test ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/run_gdas_apps_${ctest}.yaml) @@ -102,5 +102,5 @@ foreach(ctest ${ctest_list}) set_tests_properties(${test_name} PROPERTIES - ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") + ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}") endforeach() diff --git a/test/soca/gw/prep.sh b/test/soca/gw/prep.sh index 2aeadf224..424881ef3 100755 --- a/test/soca/gw/prep.sh +++ b/test/soca/gw/prep.sh @@ -35,7 +35,7 @@ done for day in $(seq 1 2 9); do cp ${COM}/06/model_data/ocean/history/gdas.t06z.ocnf003.nc \ ${project_binary_dir}/soca_static/bkgerr/stddev/ocn.ensstddev.fc.2019-04-0${day}T00:00:00Z.PT0S.nc - cp ${project_source_dir}/soca/test/Data/72x35x25/ice.bkgerror.nc \ + cp ${project_source_dir}/sorc/soca/test/Data/72x35x25/ice.bkgerror.nc \ ${project_binary_dir}/soca_static/bkgerr/stddev/ice.ensstddev.fc.2019-04-0${day}T00:00:00Z.PT0S.nc done diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index 683ad9135..97a9f5b10 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -21,12 +21,12 @@ gw environement: assym_freq: 6 backgrounds: - COM_SRC: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/COM + COM_SRC: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/COM working directories: - ROTDIRS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs/ROTDIRS - EXPDIRS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs/experiments - STMP: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs + ROTDIRS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs/ROTDIRS + EXPDIRS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs/experiments + STMP: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs jedi: OOPS_TRACE: 1 @@ -46,10 +46,10 @@ setup_expt config: NMEM_ENS: "4" DOHYBVAR: "YES" ocnanal: - SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static + SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/soca_static CASE_ANL: C48 SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml - COMIN_OBS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/obs/r2d2-shared + COMIN_OBS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/obs/r2d2-shared SOCA_NINNER: 1 R2D2_OBS_SRC: gdasapp R2D2_OBS_DUMP: soca diff --git a/test/soca/gw/static.sh b/test/soca/gw/static.sh index 5fc615a85..05407ea92 100755 --- a/test/soca/gw/static.sh +++ b/test/soca/gw/static.sh @@ -10,12 +10,12 @@ mkdir -p ${soca_static}/INPUT mkdir -p ${soca_static}/bump mkdir -p ${soca_static}/bkgerr/stddev -lowres=${project_source_dir}/soca/test/Data +lowres=${project_source_dir}/sorc/soca/test/Data cp -L ${lowres}/workdir/{diag_table,field_table} ${soca_static} cp -L ${project_source_dir}/test/soca/fix/MOM_input ${soca_static} cp -L ${lowres}/{fields_metadata.yml,godas_sst_bgerr.nc,rossrad.dat} ${soca_static} mv ${soca_static}/fields_metadata.yml ${soca_static}/fields_metadata.yaml -cp -L ${project_source_dir}/soca/test/testinput/obsop_name_map.yml ${soca_static}/obsop_name_map.yaml +cp -L ${project_source_dir}/sorc/soca/test/testinput/obsop_name_map.yml ${soca_static}/obsop_name_map.yaml cp -L ${lowres}/72x35x25/input.nml ${soca_static}/inputnml cp -L ${lowres}/72x35x25/INPUT/{hycom1_25.nc,ocean_mosaic.nc,grid_spec.nc,layer_coord25.nc,ocean_hgrid.nc,ocean_topog.nc} ${soca_static}/INPUT From b69929409641707861917a7599fee5c06bbb7201 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 11 Jan 2024 14:26:16 +0000 Subject: [PATCH 33/37] update hashes for jedi libraries (#811) --- sorc/fv3-jedi | 2 +- sorc/fv3-jedi-lm | 2 +- sorc/ioda | 2 +- sorc/iodaconv | 2 +- sorc/oops | 2 +- sorc/saber | 2 +- sorc/soca | 2 +- sorc/ufo | 2 +- sorc/vader | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sorc/fv3-jedi b/sorc/fv3-jedi index 1ccb1c7bc..b4bc84192 160000 --- a/sorc/fv3-jedi +++ b/sorc/fv3-jedi @@ -1 +1 @@ -Subproject commit 1ccb1c7bcc934c0497259d258aacb0c0df215fbf +Subproject commit b4bc841922609c28457e0899a397d001508a34de diff --git a/sorc/fv3-jedi-lm b/sorc/fv3-jedi-lm index d68f1a859..05cc1ae63 160000 --- a/sorc/fv3-jedi-lm +++ b/sorc/fv3-jedi-lm @@ -1 +1 @@ -Subproject commit d68f1a8595ac62fb945d505e1962a69da11205c2 +Subproject commit 05cc1ae63252ca535f3db0fdca9a8a996329fc8f diff --git a/sorc/ioda b/sorc/ioda index 3194c749f..f395e1d33 160000 --- a/sorc/ioda +++ b/sorc/ioda @@ -1 +1 @@ -Subproject commit 3194c749f999bfd26319792042350327a444961c +Subproject commit f395e1d33b3c6ce7b970da25f589d5fd92f5197b diff --git a/sorc/iodaconv b/sorc/iodaconv index c4f173139..01adb65f6 160000 --- a/sorc/iodaconv +++ b/sorc/iodaconv @@ -1 +1 @@ -Subproject commit c4f173139d2c159ac3474c519a87b445295fd2c4 +Subproject commit 01adb65f60dcbb7e83b50d25a82d81e915bdcb6d diff --git a/sorc/oops b/sorc/oops index 6d4a414f4..60f93924f 160000 --- a/sorc/oops +++ b/sorc/oops @@ -1 +1 @@ -Subproject commit 6d4a414f4017d1ad0df3be90d8a40d095a0556b6 +Subproject commit 60f93924fe446714fcb04d96f6930a760db74b23 diff --git a/sorc/saber b/sorc/saber index 426fbb003..4f1956c38 160000 --- a/sorc/saber +++ b/sorc/saber @@ -1 +1 @@ -Subproject commit 426fbb003c8fb806dd7ea6c30c32fdfa438a63b4 +Subproject commit 4f1956c38267200feff91a8e4231e82df3885637 diff --git a/sorc/soca b/sorc/soca index 15eab9640..5783fd72a 160000 --- a/sorc/soca +++ b/sorc/soca @@ -1 +1 @@ -Subproject commit 15eab96405cd59c12b659d4db76ce239937ba52f +Subproject commit 5783fd72ace301b07a9c264595c82c31e7e872b6 diff --git a/sorc/ufo b/sorc/ufo index c042c2fc4..e41a1c928 160000 --- a/sorc/ufo +++ b/sorc/ufo @@ -1 +1 @@ -Subproject commit c042c2fc473edb719b942971dc4ed694057f5226 +Subproject commit e41a1c928150944795ed1276c84c4f1e37a47c99 diff --git a/sorc/vader b/sorc/vader index 099cb5db5..8dd0ae629 160000 --- a/sorc/vader +++ b/sorc/vader @@ -1 +1 @@ -Subproject commit 099cb5db5a2a16049ef959a4feef463fe23dee7e +Subproject commit 8dd0ae629198f955ac86abf1b26f8cffdcf15bad From 2671b13f75676ce8c2367d785ee4ead99ef1e309 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 12 Jan 2024 19:27:04 +0000 Subject: [PATCH 34/37] update submodule hashes for land-imsproc and vader, add gdas to DMPDIR path (#811) --- sorc/land-imsproc | 2 +- sorc/vader | 2 +- test/soca/gw/run_jjobs.yaml.test | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/land-imsproc b/sorc/land-imsproc index 2ac7c0d90..6373819ca 160000 --- a/sorc/land-imsproc +++ b/sorc/land-imsproc @@ -1 +1 @@ -Subproject commit 2ac7c0d90d3ebd449ad11bce155a71d381b4534a +Subproject commit 6373819ca034d66523cb7852cd7b1b66f3f8ae07 diff --git a/sorc/vader b/sorc/vader index 8dd0ae629..2baeb8dfa 160000 --- a/sorc/vader +++ b/sorc/vader @@ -1 +1 @@ -Subproject commit 8dd0ae629198f955ac86abf1b26f8cffdcf15bad +Subproject commit 2baeb8dfa8781a67bcf386bf152f02619b748298 diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index 860cf608c..2cda89dac 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -56,7 +56,7 @@ setup_expt config: prepoceanobs: SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml OBSPROC_CONFIG: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml - DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/obsproc + DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsproc job options: account: da-cpu From 54ff2af5416324d69e56d85f75615495357e7de4 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 12 Jan 2024 19:41:02 +0000 Subject: [PATCH 35/37] replace resdet and resens with resdetatmos and resensatmos (#811) --- test/aero/global-workflow/setup_workflow_exp.sh | 8 ++++---- test/atm/global-workflow/setup_workflow_exp.sh | 8 ++++---- test/setup_workflow_exp.sh | 6 +++--- test/soca/gw/run_jjobs.yaml.test | 4 ++-- ush/soca/examples/run_jjobs_container.yaml | 2 +- ush/soca/examples/run_jjobs_hera.yaml | 2 +- ush/soca/examples/run_jjobs_hera_025.yaml | 2 +- ush/soca/run_jjobs.py | 4 ++-- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/aero/global-workflow/setup_workflow_exp.sh b/test/aero/global-workflow/setup_workflow_exp.sh index 288021a8d..af2165631 100755 --- a/test/aero/global-workflow/setup_workflow_exp.sh +++ b/test/aero/global-workflow/setup_workflow_exp.sh @@ -10,8 +10,8 @@ edate=2021032318 app=ATMA starttype='warm' gfscyc='4' -resdet='48' -resens='48' +resdetatmos='48' +resensatmos='48' nens=0 pslot='gdas_test' configdir=$srcdir/../../parm/config/gfs @@ -41,8 +41,8 @@ $srcdir/../../workflow/setup_expt.py gfs cycled --idate $idate \ --app $app \ --start $starttype \ --gfs_cyc $gfscyc \ - --resdet $resdet \ - --resens $resens \ + --resdetatmos $resdetatmos \ + --resensatmos $resensatmos \ --nens $nens \ --pslot $pslot \ --configdir $expdir/../config \ diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index 64cecc3da..c283c0b86 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -11,8 +11,8 @@ edate=2021032318 app=ATM starttype='warm' gfscyc='4' -resdet='48' -resens='48' +resdetatmos='48' +resensatmos='48' nens=3 pslot='gdas_test' configdir=$srcdir/../../parm/config/gfs @@ -40,8 +40,8 @@ $srcdir/../../workflow/setup_expt.py gfs cycled --idate $idate \ --app $app \ --start $starttype \ --gfs_cyc $gfscyc \ - --resdet $resdet \ - --resens $resens \ + --resdetatmos $resdetatmos \ + --resensatmos $resensatmos \ --nens $nens \ --pslot $pslot \ --configdir $expdir/../config \ diff --git a/test/setup_workflow_exp.sh b/test/setup_workflow_exp.sh index 5f16b0ecd..b73e8b503 100755 --- a/test/setup_workflow_exp.sh +++ b/test/setup_workflow_exp.sh @@ -9,8 +9,8 @@ edate=2021032418 app=ATM # NOTE make this S2SWA soon starttype='warm' gfscyc='4' -resdet='48' -resens='48' +resdetatmos='48' +resensatmos='48' nens=0 pslot='gdas_test' configdir=$srcdir/../../parm/config/gfs @@ -24,6 +24,6 @@ rm -rf "${comrot}" "${expdir}" cd $srcdir/../../workflow echo "Running global-workflow experiment generation script" -./setup_expt.py gfs cycled --idate $idate --edate $edate --app $app --start $starttype --gfs_cyc $gfscyc --resdet $resdet --resens $resens --nens $nens --pslot $pslot --configdir $configdir --comrot $comrot --expdir $expdir +./setup_expt.py gfs cycled --idate $idate --edate $edate --app $app --start $starttype --gfs_cyc $gfscyc --resdetatmos $resdetatmos --resensatmos $resensatmos --nens $nens --pslot $pslot --configdir $configdir --comrot $comrot --expdir $expdir exit $? diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index 2cda89dac..38d1bcdca 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -1,6 +1,6 @@ machine: @MACHINE@ -resdet: 48 -resens: 48 +resdetatmos: 48 +resensatmos: 48 jjobs: - @JJOB@ diff --git a/ush/soca/examples/run_jjobs_container.yaml b/ush/soca/examples/run_jjobs_container.yaml index 13cf2a76c..92901fdc8 100644 --- a/ush/soca/examples/run_jjobs_container.yaml +++ b/ush/soca/examples/run_jjobs_container.yaml @@ -1,5 +1,5 @@ machine: container -resdet: 48 +resdetatmos: 48 jjobs: - JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP diff --git a/ush/soca/examples/run_jjobs_hera.yaml b/ush/soca/examples/run_jjobs_hera.yaml index b4bb5d448..6b2edb863 100644 --- a/ush/soca/examples/run_jjobs_hera.yaml +++ b/ush/soca/examples/run_jjobs_hera.yaml @@ -1,5 +1,5 @@ machine: hera -resdet: 48 +resdetatmos: 48 jjobs: - JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP diff --git a/ush/soca/examples/run_jjobs_hera_025.yaml b/ush/soca/examples/run_jjobs_hera_025.yaml index ba2552ef9..5686af150 100644 --- a/ush/soca/examples/run_jjobs_hera_025.yaml +++ b/ush/soca/examples/run_jjobs_hera_025.yaml @@ -1,5 +1,5 @@ machine: hera -resdet: 48 +resdetatmos: 48 jjobs: - JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index f9b73c845..8324afab8 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -125,8 +125,8 @@ def setupexpt(self): "app": "ATM", "start": "warm", "gfs_cyc": "0", - "resdet": self.config['resdet'], - "resens": self.config['resens'], + "resdetatmos": self.config['resdetatmos'], + "resensatmos": self.config['resensatmos'], "nens": "0", "pslot": "${PSLOT}", "configdir": "${PWD}/config/gfs", From b9379ba9531ce4f207f82f121d897efb540b0c4b Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 12 Jan 2024 20:49:45 +0000 Subject: [PATCH 36/37] update comrot to be consistent with g-w develop (#811) --- test/aero/global-workflow/setup_workflow_exp.sh | 6 +++--- test/atm/global-workflow/setup_workflow_exp.sh | 6 +++--- test/setup_workflow_exp.sh | 6 +++--- test/soca/gw/CMakeLists.txt | 2 +- test/soca/gw/run_jjobs.yaml.test | 2 +- ush/soca/run_jjobs.py | 8 ++++---- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/aero/global-workflow/setup_workflow_exp.sh b/test/aero/global-workflow/setup_workflow_exp.sh index af2165631..b9f495cc0 100755 --- a/test/aero/global-workflow/setup_workflow_exp.sh +++ b/test/aero/global-workflow/setup_workflow_exp.sh @@ -15,11 +15,11 @@ resensatmos='48' nens=0 pslot='gdas_test' configdir=$srcdir/../../parm/config/gfs -comrot=$bindir/test/aero/global-workflow/testrun/ROTDIRS +comroot=$bindir/test/aero/global-workflow/testrun/ROTDIRS expdir=$bindir/test/aero/global-workflow/testrun/experiments # clean previous experiment -rm -rf $comrot $expdir config +rm -rf $comroot $expdir config # copy config.yaml to local config cp -r $configdir config @@ -46,7 +46,7 @@ $srcdir/../../workflow/setup_expt.py gfs cycled --idate $idate \ --nens $nens \ --pslot $pslot \ --configdir $expdir/../config \ - --comrot $comrot \ + --comroot $comroot \ --expdir $expdir \ --yaml $expdir/../config.yaml diff --git a/test/atm/global-workflow/setup_workflow_exp.sh b/test/atm/global-workflow/setup_workflow_exp.sh index c283c0b86..67bfd3b03 100755 --- a/test/atm/global-workflow/setup_workflow_exp.sh +++ b/test/atm/global-workflow/setup_workflow_exp.sh @@ -16,11 +16,11 @@ resensatmos='48' nens=3 pslot='gdas_test' configdir=$srcdir/../../parm/config/gfs -comrot=$bindir/test/atm/global-workflow/testrun/ROTDIRS +comroot=$bindir/test/atm/global-workflow/testrun/ROTDIRS expdir=$bindir/test/atm/global-workflow/testrun/experiments # clean previous experiment -rm -rf $comrot $expdir config +rm -rf $comroot $expdir config # copy config.yaml to local config cp -r $configdir config @@ -45,7 +45,7 @@ $srcdir/../../workflow/setup_expt.py gfs cycled --idate $idate \ --nens $nens \ --pslot $pslot \ --configdir $expdir/../config \ - --comrot $comrot \ + --comroot $comroot \ --expdir $expdir \ --yaml $expdir/../config.yaml diff --git a/test/setup_workflow_exp.sh b/test/setup_workflow_exp.sh index b73e8b503..3123a7570 100755 --- a/test/setup_workflow_exp.sh +++ b/test/setup_workflow_exp.sh @@ -14,16 +14,16 @@ resensatmos='48' nens=0 pslot='gdas_test' configdir=$srcdir/../../parm/config/gfs -comrot=$bindir/test/testrun/ROTDIRS +comroot=$bindir/test/testrun/ROTDIRS expdir=$bindir/test/testrun/experiments # clean previous experiment -rm -rf "${comrot}" "${expdir}" +rm -rf "${comroot}" "${expdir}" # run the script cd $srcdir/../../workflow echo "Running global-workflow experiment generation script" -./setup_expt.py gfs cycled --idate $idate --edate $edate --app $app --start $starttype --gfs_cyc $gfscyc --resdetatmos $resdetatmos --resensatmos $resensatmos --nens $nens --pslot $pslot --configdir $configdir --comrot $comrot --expdir $expdir +./setup_expt.py gfs cycled --idate $idate --edate $edate --app $app --start $starttype --gfs_cyc $gfscyc --resdetatmos $resdetatmos --resensatmos $resensatmos --nens $nens --pslot $pslot --configdir $configdir --comroot $comroot --expdir $expdir exit $? diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index 38c12eadd..bd6616744 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -1,7 +1,7 @@ # test for creating an experiment directory within the global-workflow file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun) -# Prepare COMROT and static dir +# Prepare ROTDIR and static dir add_test(NAME test_gdasapp_soca_prep COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/prep.sh ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw) diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index 38d1bcdca..c4f4e12f3 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -55,7 +55,7 @@ setup_expt config: NICAS_GRID_SIZE: 150 prepoceanobs: SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml - OBSPROC_CONFIG: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml + OBSPROC_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsproc job options: diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index 8324afab8..b83b24d32 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -130,7 +130,7 @@ def setupexpt(self): "nens": "0", "pslot": "${PSLOT}", "configdir": "${PWD}/config/gfs", - "comrot": self.rotdir, + "comroot": self.rotdir, "expdir": "${EXPDIRS}", "yaml": "overwrite_defaults.yaml"} @@ -164,7 +164,7 @@ def precom(self, com, tmpl): def copy_bkgs(self): """ Fill the ROTDIR with backgrounds - TODO: replace by fill comrot? + TODO: replace by fill comroot? """ print(f"gPDY: {self.gPDY}") print(f"gcyc: {self.gcyc}") @@ -254,7 +254,7 @@ def execute(self, submit=False): def main(): - epilog = ["Make sure the comrot, experiment and config directories are removed before running this script", + epilog = ["Make sure the comroot, experiment and config directories are removed before running this script", "Examples:", " ./run_jjobs.py -y run_jjobs_orion.yaml", " ./run_jjobs.py -h"] @@ -272,7 +272,7 @@ def main(): exp_config = yaml.safe_load(file) if not args.skip: - # Write a setup card (prepare COMROT, configs, ...) + # Write a setup card (prepare COMROOT, configs, ...) setup_card = JobCard("setup_expt.sh", exp_config) setup_card.export_env_vars_script() setup_card.setupexpt() From 695447babefa817432fe3f19d8c565df92f05054 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 13 Jan 2024 14:14:40 +0000 Subject: [PATCH 37/37] use realpath for HOMEgfs in config.base (#811) --- ush/soca/run_jjobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index b83b24d32..2008c3042 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -213,7 +213,8 @@ def fixconfigs(self): print(f"Probably does not work for {machine} yet") # swap a few variables in config.base - var2replace = {'HOMEgfs': self.homegfs, + self.homegfs_real = os.path.realpath(self.homegfs) + var2replace = {'HOMEgfs': self.homegfs_real, 'STMP': self.stmp, 'ROTDIR': self.rotdir, 'EXPDIRS': self.expdirs}