Update the sprint branch with as much of develop branch as possible for November code sprint

.gitattributes

@@ -13,3 +13,5 @@ src/ncep/xx120 filter=lfs diff=lfs merge=lfs -text
 *.grib2 filter=lfs diff=lfs merge=lfs -text
 test/testinput/bufr_tables/** filter=lfs diff=lfs merge=lfs -text
 *.radiosonde filter=lfs diff=lfs merge=lfs -text
+*.dump filter=lfs diff=lfs merge=lfs -text
+*.prepbufr filter=lfs diff=lfs merge=lfs -text

CI/CMakeLists.txt

@@ -19,24 +19,14 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 
 ecbuild_bundle_initialize()
 
-ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA-internal/jedi-cmake.git" )
-include( jedicmake/cmake/Functions/git_functions.cmake )
-
-# ECMWF libs
-# ----------
-option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
-option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Build fckit unless user passes -DBUNDLE_SKIP_FCKIT=OFF
-option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Build atlas unless user passes -DBUNDLE_SKIP_ATLAS=OFF
-
-ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 )
-ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 )
-ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 )
+#ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA-internal/jedi-cmake.git" )
+#include( jedicmake/cmake/Functions/git_functions.cmake )
+include( /opt/view/share/jedicmake/Functions/git_functions.cmake )
 
 # Core JEDI repositories
 ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA-internal/oops.git" )
 ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA-internal/ioda.git" )
 
-
 # If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch.
 # If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case,
 # IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned.
@@ -52,7 +42,6 @@ endif()
 branch_checkout (REPO_DIR_NAME ioda-data
                  BRANCH ${GIT_BRANCH_FUNC} )
 
-
 # Build IODA converters
 
 ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" )

CI/buildspec_clang.yml

@@ -15,11 +15,6 @@ phases:
       - echo $CODEBUILD_WEBHOOK_ACTOR_ACCOUNT_ID
       - echo $CODEBUILD_WEBHOOK_EVENT
       - echo $CODEBUILD_RESOLVED_SOURCE_VERSION
-      - pip install xarray
-      - echo $PATH
-      - echo $LD_LIBRARY_PATH
-      - echo $PYTHONPATH
-      - export PYTHONPATH=/usr/local/lib:/usr/local/lib/python3.8/site-packages
 
       # read cdash url from s3
       - wget https://ci-test-cdash-url.s3.amazonaws.com/cdash_url.txt
@@ -28,7 +23,6 @@ phases:
       # Codebuild only runs on PUSH events if HEAD_REF
       # is refs/heads/develop (merge to develop). In this
       # case CODEBUILD_GIT_BRANCH="develop"
-
       - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ];
         then export CODEBUILD_GIT_BRANCH="develop";
         echo "Merging to develop";
@@ -46,17 +40,12 @@ phases:
   pre_build:
     commands:
       - echo Executing pre_build phase
-      - git lfs install
-      - mkdir /build_container
       - mkdir -p /jcsda/ioda-bundle
+      - mkdir /build_container
 
+      - git lfs install
       - cd CI
 
-      # Upload branch name and commit sha as CodeBuild artifact to S3
-      - mkdir -p /jcsda/artifacts
-      - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
-      - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.tx
-
       #  Setting git credentials
       - sed -i '/ssh/d' ~/.gitconfig
       - sed '/instead/d' ~/.gitconfig
@@ -65,50 +54,59 @@ phases:
       - chmod 0700 ~/.git-credentials
       - echo "https://${GIT_USER}:${GIT_PASS}@github.com" >~/.git-credentials
 
+
+      # Upload branch name and commit sha as CodeBuild artifact to S3
+      - mkdir -p /jcsda/artifacts
+      - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
+      - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt
+
       # ioda-converters (testing repo)
       - ./clone.sh jcsda-internal/ioda-converters $CODEBUILD_GIT_BRANCH  /jcsda/ioda-bundle iodaconv ${GIT_BASE_BRANCH} develop
 
-      # jedi-cmake
-      - ./clone.sh jcsda-internal/jedi-cmake $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle jedicmake ${GIT_BASE_BRANCH} develop
-
-     # oops
+      # oops
       - ./clone.sh jcsda-internal/oops $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle oops ${GIT_BASE_BRANCH} develop
 
-     # ioda
+      # ioda
       - ./clone.sh jcsda-internal/ioda $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda ${GIT_BASE_BRANCH} develop
 
       # ioda-data
       - ./clone.sh jcsda-internal/ioda-data $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda-data ${GIT_BASE_BRANCH} develop
 
+      # move CMakeLists.txt from ioda-converters/CI to bundle directory
       - cp CMakeLists.txt /jcsda/ioda-bundle
       - cp -r cmake /jcsda/ioda-bundle/
       - cp /jcsda/ioda-bundle/cmake/CTestConfig.cmake /jcsda/ioda-bundle/
       - sed -i "s@CDASH_URL@$CDASH_URL@g" /jcsda/ioda-bundle/CTestConfig.cmake
       - ls /jcsda/ioda-bundle/
+
   build:
     on-failure: CONTINUE
     commands:
+      - echo Executing build phase
+      - export BUILD_STATUS="0"
+      - echo $BUILD_STATUS
+      - echo $CODEBUILD_BUILD_SUCCEEDING
+
       - cd /build_container
-      - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_SITE=CodeBuild -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DUSE_ECCODES=ON /jcsda/ioda-bundle
+      - source /etc/profile.d/z10_spack_environment.sh
+      - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_SITE=CodeBuild -DCDASH_OVERRIDE_SYSTEM_NAME=ClangContainer -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DUSE_ECCODES=ON /jcsda/ioda-bundle
+
+      # Build ioda
       - cd /build_container/ioda
       - cp ../DartConfiguration.tcl .
       - sed -i 's/ioda-bundle/ioda-bundle\/ioda/' DartConfiguration.tcl
       - sed -i 's/build_container/build_container\/ioda/' DartConfiguration.tcl
       - cat DartConfiguration.tcl
       - ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4
 
+      # Build ioda-converters
       - cd /build_container/iodaconv
       - cp ../DartConfiguration.tcl .
       - sed -i 's/ioda-bundle/ioda-bundle\/iodaconv/' DartConfiguration.tcl
       - sed -i 's/build_container/build_container\/iodaconv/' DartConfiguration.tcl
       - cat DartConfiguration.tcl
-
       - ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4
 
-      - export BUILD_STATUS="0"
-      - echo $BUILD_STATUS
-      - echo $CODEBUILD_BUILD_SUCCEEDING
-
       - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ];
         then export BUILD_STATUS="1";
         echo "Build passed";

CI/buildspec_gnu.yml

@@ -14,16 +14,10 @@ phases:
       - echo $CODEBUILD_WEBHOOK_ACTOR_ACCOUNT_ID
       - echo $CODEBUILD_WEBHOOK_EVENT
       - echo $CODEBUILD_RESOLVED_SOURCE_VERSION
-      - pip install xarray
-      - echo $PATH
-      - echo $LD_LIBRARY_PATH
-      - echo $LIBRARY_PATH
-      - echo $PYTHONPATH
 
       # Codebuild only runs on PUSH events if HEAD_REF
       # is refs/heads/develop (merge to develop). In this
       # case CODEBUILD_GIT_BRANCH="develop"
-
       - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ];
         then export CODEBUILD_GIT_BRANCH="develop";
         echo "Merging to develop";
@@ -46,14 +40,9 @@ phases:
     on-failure: CONTINUE
     commands:
       - echo Executing pre_build phase
-      - git lfs install # creates .gitconfig
       - mkdir -p /jcsda/ioda-bundle
 
-      # Upload branch name and commit sha as CodeBuild artifact to S3
-      - mkdir -p /jcsda/artifacts
-      - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
-      - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt
-
+      - git lfs install
       - cd CI
 
       #  Setting git credentials
@@ -64,21 +53,24 @@ phases:
       - chmod 0700 ~/.git-credentials
       - echo "https://${GIT_USER}:${GIT_PASS}@github.com" >~/.git-credentials
 
+      # Upload branch name and commit sha as CodeBuild artifact to S3
+      - mkdir /jcsda/artifacts
+      - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt
+      - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt
+
       # ioda-converters (testing repo)
       - ./clone.sh jcsda-internal/ioda-converters $CODEBUILD_GIT_BRANCH  /jcsda/ioda-bundle iodaconv ${GIT_BASE_BRANCH} develop
 
-      # jedi-cmake
-      - ./clone.sh jcsda-internal/jedi-cmake $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle jedicmake ${GIT_BASE_BRANCH} develop
-
-     # oops
+      # oops
       - ./clone.sh jcsda-internal/oops $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle oops ${GIT_BASE_BRANCH} develop
 
-     # ioda
+      # ioda
       - ./clone.sh jcsda-internal/ioda $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda ${GIT_BASE_BRANCH} develop
 
       # ioda-data
       - ./clone.sh jcsda-internal/ioda-data $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda-data ${GIT_BASE_BRANCH} develop
 
+       # move CMakeLists.txt from ioda-converters/CI to bundle directory
       - cp CMakeLists.txt /jcsda/ioda-bundle
       - cp -r cmake /jcsda/ioda-bundle/
       - cp /jcsda/ioda-bundle/cmake/CTestConfig.cmake /jcsda/ioda-bundle/
@@ -87,51 +79,57 @@ phases:
 
   build:
     commands:
-      - su - jedi -c "cd /home/jedi
-        && echo $CC
-        && echo $CXX
-        && echo $FC
-        && CC=mpicc CXX=mpicxx FC=mpifort ecbuild -Wno-dev -DCDASH_OVERRIDE_SITE=CodeBuild -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DENABLE_GPROF=ON -DUSE_ECCODES=ON /jcsda/ioda-bundle/"
-
-      - su - jedi -c "cd /home/jedi/ioda
-        && echo $CC
-        && echo $CXX
-        && echo $FC
+      - echo Executing build phase
+      - export BUILD_STATUS="0"
+      - echo $BUILD_STATUS
+      - echo $CODEBUILD_BUILD_SUCCEEDING
+
+      # configure and build
+      - su - nonroot -c "cd /home/nonroot
+        && ls
+        && export FC=mpifort
+        && export CC=mpicc
+        && export CXX=mpicxx
+        && ecbuild -Wno-dev -DCDASH_OVERRIDE_SITE=CodeBuild -DCDASH_OVERRIDE_SYSTEM_NAME=GNUContainer -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DENABLE_GPROF=ON -DUSE_ECCODES=ON /jcsda/ioda-bundle/"
+
+      - su - nonroot -c "cd /home/nonroot/ioda
+        && export FC=mpifort
+        && export CC=mpicc
+        && export CXX=mpicxx
         && cp ../DartConfiguration.tcl .
         && sed -i 's/ioda-bundle/ioda-bundle\/ioda/' DartConfiguration.tcl
-        && sed -i 's/jedi/jedi\/ioda/' DartConfiguration.tcl
+        && sed -i 's/nonroot/nonroot\/ioda/' DartConfiguration.tcl
         && cat DartConfiguration.tcl
         && ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4"
 
-      - su - jedi -c "cd /home/jedi/iodaconv
-        && echo $CC
-        && echo $CXX
-        && echo $FC
+      - su - nonroot -c "cd /home/nonroot/iodaconv
+        && export FC=mpifort
+        && export CC=mpicc
+        && export CXX=mpicxx
         && cp ../DartConfiguration.tcl .
         && sed -i 's/ioda-bundle/ioda-bundle\/iodaconv/' DartConfiguration.tcl
-        && sed -i 's/jedi/jedi\/iodaconv/' DartConfiguration.tcl
+        && sed -i 's/nonroot/nonroot\/iodaconv/' DartConfiguration.tcl
         && cat DartConfiguration.tcl
         && ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4"
 
-
       - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ];
         then export BUILD_STATUS="1";
         echo "Build passed";
         fi
       - echo $BUILD_STATUS
 
       # run ctest
-      - su - jedi -c "CC=mpicc CXX=mpicxx FC=mpifort
-        && export PATH=/usr/local/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-        && export LD_LIBRARY_PATH=/usr/local/lib:/jedi/home/lib
-        && export PYTHONPATH=/usr/local/lib:/usr/local/lib/python3.8/site-packages:$PYTHONPATH
-        && cd /home/jedi/iodaconv
+      - su - nonroot -c "cd /home/nonroot/iodaconv
+        && export FC=mpifort
+        && export CC=mpicc
+        && export CXX=mpicxx
         && ctest -C RelWithDebInfo -D ExperimentalTest"
 
     finally:
-      # upload ctest report to CDASH
-      - su - jedi -c "CC=mpicc CXX=mpicxx FC=mpifort
-        && cd /home/jedi/iodaconv
+      - su - nonroot -c "cd /home/nonroot/iodaconv
+        && export FC=mpifort
+        && export CC=mpicc
+        && export CXX=mpicxx
         && ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous"
 
   post_build:
@@ -143,7 +141,7 @@ phases:
       # upload find cdash url and upload it as CodeBuild artifact to S3
       - if [ "$BUILD_STATUS" = "1" ];
         then echo "Build & tests passed, find cdash url";
-        url=$(bash /jcsda/ioda-bundle/iodaconv/CI/cdash-url.sh /home/jedi/iodaconv/Testing $CDASH_URL);
+        url=$(bash /jcsda/ioda-bundle/iodaconv/CI/cdash-url.sh /home/nonroot/iodaconv/Testing $CDASH_URL);
         echo $url;
         echo ${url} > /jcsda/artifacts/cdash-url.txt;
         cat /jcsda/artifacts/cdash-url.txt;