diff --git a/.gitattributes b/.gitattributes index f1cc5484d..1204bb208 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,3 +13,5 @@ src/ncep/xx120 filter=lfs diff=lfs merge=lfs -text *.grib2 filter=lfs diff=lfs merge=lfs -text test/testinput/bufr_tables/** filter=lfs diff=lfs merge=lfs -text *.radiosonde filter=lfs diff=lfs merge=lfs -text +*.dump filter=lfs diff=lfs merge=lfs -text +*.prepbufr filter=lfs diff=lfs merge=lfs -text diff --git a/CI/CMakeLists.txt b/CI/CMakeLists.txt index 29aab3ce0..e7c9cff40 100644 --- a/CI/CMakeLists.txt +++ b/CI/CMakeLists.txt @@ -19,24 +19,14 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) ecbuild_bundle_initialize() -ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA-internal/jedi-cmake.git" ) -include( jedicmake/cmake/Functions/git_functions.cmake ) - -# ECMWF libs -# ---------- -option("BUNDLE_SKIP_ECKIT" "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF -option("BUNDLE_SKIP_FCKIT" "Don't build fckit" "ON") # Build fckit unless user passes -DBUNDLE_SKIP_FCKIT=OFF -option("BUNDLE_SKIP_ATLAS" "Don't build atlas" "ON") # Build atlas unless user passes -DBUNDLE_SKIP_ATLAS=OFF - -ecbuild_bundle( PROJECT eckit GIT "https://github.com/ecmwf/eckit.git" TAG 1.16.0 ) -ecbuild_bundle( PROJECT fckit GIT "https://github.com/ecmwf/fckit.git" TAG 0.9.2 ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/ecmwf/atlas.git" TAG 0.24.1 ) +#ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA-internal/jedi-cmake.git" ) +#include( jedicmake/cmake/Functions/git_functions.cmake ) +include( /opt/view/share/jedicmake/Functions/git_functions.cmake ) # Core JEDI repositories ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA-internal/oops.git" ) ecbuild_bundle( PROJECT ioda GIT "https://github.com/JCSDA-internal/ioda.git" ) - # If IODA branch is being built set GIT_BRANCH_FUNC to IODA's current branch. # If a tagged version of IODA is being built set GIT_TAG_FUNC to ioda's current tag. In this case, # IODA test files will be download from UCAR DASH and ioda-data repo will not be cloned. @@ -52,7 +42,6 @@ endif() branch_checkout (REPO_DIR_NAME ioda-data BRANCH ${GIT_BRANCH_FUNC} ) - # Build IODA converters ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/JCSDA-internal/ioda-converters.git" ) diff --git a/CI/buildspec_clang.yml b/CI/buildspec_clang.yml index 73a49400c..2dd607ec2 100644 --- a/CI/buildspec_clang.yml +++ b/CI/buildspec_clang.yml @@ -15,11 +15,6 @@ phases: - echo $CODEBUILD_WEBHOOK_ACTOR_ACCOUNT_ID - echo $CODEBUILD_WEBHOOK_EVENT - echo $CODEBUILD_RESOLVED_SOURCE_VERSION - - pip install xarray - - echo $PATH - - echo $LD_LIBRARY_PATH - - echo $PYTHONPATH - - export PYTHONPATH=/usr/local/lib:/usr/local/lib/python3.8/site-packages # read cdash url from s3 - wget https://ci-test-cdash-url.s3.amazonaws.com/cdash_url.txt @@ -28,7 +23,6 @@ phases: # Codebuild only runs on PUSH events if HEAD_REF # is refs/heads/develop (merge to develop). In this # case CODEBUILD_GIT_BRANCH="develop" - - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ]; then export CODEBUILD_GIT_BRANCH="develop"; echo "Merging to develop"; @@ -46,17 +40,12 @@ phases: pre_build: commands: - echo Executing pre_build phase - - git lfs install - - mkdir /build_container - mkdir -p /jcsda/ioda-bundle + - mkdir /build_container + - git lfs install - cd CI - # Upload branch name and commit sha as CodeBuild artifact to S3 - - mkdir -p /jcsda/artifacts - - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt - - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.tx - # Setting git credentials - sed -i '/ssh/d' ~/.gitconfig - sed '/instead/d' ~/.gitconfig @@ -65,31 +54,44 @@ phases: - chmod 0700 ~/.git-credentials - echo "https://${GIT_USER}:${GIT_PASS}@github.com" >~/.git-credentials + + # Upload branch name and commit sha as CodeBuild artifact to S3 + - mkdir -p /jcsda/artifacts + - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt + - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt + # ioda-converters (testing repo) - ./clone.sh jcsda-internal/ioda-converters $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle iodaconv ${GIT_BASE_BRANCH} develop - # jedi-cmake - - ./clone.sh jcsda-internal/jedi-cmake $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle jedicmake ${GIT_BASE_BRANCH} develop - - # oops + # oops - ./clone.sh jcsda-internal/oops $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle oops ${GIT_BASE_BRANCH} develop - # ioda + # ioda - ./clone.sh jcsda-internal/ioda $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda ${GIT_BASE_BRANCH} develop # ioda-data - ./clone.sh jcsda-internal/ioda-data $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda-data ${GIT_BASE_BRANCH} develop + # move CMakeLists.txt from ioda-converters/CI to bundle directory - cp CMakeLists.txt /jcsda/ioda-bundle - cp -r cmake /jcsda/ioda-bundle/ - cp /jcsda/ioda-bundle/cmake/CTestConfig.cmake /jcsda/ioda-bundle/ - sed -i "s@CDASH_URL@$CDASH_URL@g" /jcsda/ioda-bundle/CTestConfig.cmake - ls /jcsda/ioda-bundle/ + build: on-failure: CONTINUE commands: + - echo Executing build phase + - export BUILD_STATUS="0" + - echo $BUILD_STATUS + - echo $CODEBUILD_BUILD_SUCCEEDING + - cd /build_container - - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_SITE=CodeBuild -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DUSE_ECCODES=ON /jcsda/ioda-bundle + - source /etc/profile.d/z10_spack_environment.sh + - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_SITE=CodeBuild -DCDASH_OVERRIDE_SYSTEM_NAME=ClangContainer -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DUSE_ECCODES=ON /jcsda/ioda-bundle + + # Build ioda - cd /build_container/ioda - cp ../DartConfiguration.tcl . - sed -i 's/ioda-bundle/ioda-bundle\/ioda/' DartConfiguration.tcl @@ -97,18 +99,14 @@ phases: - cat DartConfiguration.tcl - ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4 + # Build ioda-converters - cd /build_container/iodaconv - cp ../DartConfiguration.tcl . - sed -i 's/ioda-bundle/ioda-bundle\/iodaconv/' DartConfiguration.tcl - sed -i 's/build_container/build_container\/iodaconv/' DartConfiguration.tcl - cat DartConfiguration.tcl - - ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4 - - export BUILD_STATUS="0" - - echo $BUILD_STATUS - - echo $CODEBUILD_BUILD_SUCCEEDING - - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ]; then export BUILD_STATUS="1"; echo "Build passed"; diff --git a/CI/buildspec_gnu.yml b/CI/buildspec_gnu.yml index a78b1e860..bf7424e75 100644 --- a/CI/buildspec_gnu.yml +++ b/CI/buildspec_gnu.yml @@ -14,16 +14,10 @@ phases: - echo $CODEBUILD_WEBHOOK_ACTOR_ACCOUNT_ID - echo $CODEBUILD_WEBHOOK_EVENT - echo $CODEBUILD_RESOLVED_SOURCE_VERSION - - pip install xarray - - echo $PATH - - echo $LD_LIBRARY_PATH - - echo $LIBRARY_PATH - - echo $PYTHONPATH # Codebuild only runs on PUSH events if HEAD_REF # is refs/heads/develop (merge to develop). In this # case CODEBUILD_GIT_BRANCH="develop" - - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ]; then export CODEBUILD_GIT_BRANCH="develop"; echo "Merging to develop"; @@ -46,14 +40,9 @@ phases: on-failure: CONTINUE commands: - echo Executing pre_build phase - - git lfs install # creates .gitconfig - mkdir -p /jcsda/ioda-bundle - # Upload branch name and commit sha as CodeBuild artifact to S3 - - mkdir -p /jcsda/artifacts - - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt - - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt - + - git lfs install - cd CI # Setting git credentials @@ -64,21 +53,24 @@ phases: - chmod 0700 ~/.git-credentials - echo "https://${GIT_USER}:${GIT_PASS}@github.com" >~/.git-credentials + # Upload branch name and commit sha as CodeBuild artifact to S3 + - mkdir /jcsda/artifacts + - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt + - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt + # ioda-converters (testing repo) - ./clone.sh jcsda-internal/ioda-converters $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle iodaconv ${GIT_BASE_BRANCH} develop - # jedi-cmake - - ./clone.sh jcsda-internal/jedi-cmake $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle jedicmake ${GIT_BASE_BRANCH} develop - - # oops + # oops - ./clone.sh jcsda-internal/oops $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle oops ${GIT_BASE_BRANCH} develop - # ioda + # ioda - ./clone.sh jcsda-internal/ioda $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda ${GIT_BASE_BRANCH} develop # ioda-data - ./clone.sh jcsda-internal/ioda-data $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda-data ${GIT_BASE_BRANCH} develop + # move CMakeLists.txt from ioda-converters/CI to bundle directory - cp CMakeLists.txt /jcsda/ioda-bundle - cp -r cmake /jcsda/ioda-bundle/ - cp /jcsda/ioda-bundle/cmake/CTestConfig.cmake /jcsda/ioda-bundle/ @@ -87,33 +79,39 @@ phases: build: commands: - - su - jedi -c "cd /home/jedi - && echo $CC - && echo $CXX - && echo $FC - && CC=mpicc CXX=mpicxx FC=mpifort ecbuild -Wno-dev -DCDASH_OVERRIDE_SITE=CodeBuild -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DENABLE_GPROF=ON -DUSE_ECCODES=ON /jcsda/ioda-bundle/" - - - su - jedi -c "cd /home/jedi/ioda - && echo $CC - && echo $CXX - && echo $FC + - echo Executing build phase + - export BUILD_STATUS="0" + - echo $BUILD_STATUS + - echo $CODEBUILD_BUILD_SUCCEEDING + + # configure and build + - su - nonroot -c "cd /home/nonroot + && ls + && export FC=mpifort + && export CC=mpicc + && export CXX=mpicxx + && ecbuild -Wno-dev -DCDASH_OVERRIDE_SITE=CodeBuild -DCDASH_OVERRIDE_SYSTEM_NAME=GNUContainer -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DENABLE_GPROF=ON -DUSE_ECCODES=ON /jcsda/ioda-bundle/" + + - su - nonroot -c "cd /home/nonroot/ioda + && export FC=mpifort + && export CC=mpicc + && export CXX=mpicxx && cp ../DartConfiguration.tcl . && sed -i 's/ioda-bundle/ioda-bundle\/ioda/' DartConfiguration.tcl - && sed -i 's/jedi/jedi\/ioda/' DartConfiguration.tcl + && sed -i 's/nonroot/nonroot\/ioda/' DartConfiguration.tcl && cat DartConfiguration.tcl && ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4" - - su - jedi -c "cd /home/jedi/iodaconv - && echo $CC - && echo $CXX - && echo $FC + - su - nonroot -c "cd /home/nonroot/iodaconv + && export FC=mpifort + && export CC=mpicc + && export CXX=mpicxx && cp ../DartConfiguration.tcl . && sed -i 's/ioda-bundle/ioda-bundle\/iodaconv/' DartConfiguration.tcl - && sed -i 's/jedi/jedi\/iodaconv/' DartConfiguration.tcl + && sed -i 's/nonroot/nonroot\/iodaconv/' DartConfiguration.tcl && cat DartConfiguration.tcl && ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4" - - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ]; then export BUILD_STATUS="1"; echo "Build passed"; @@ -121,17 +119,17 @@ phases: - echo $BUILD_STATUS # run ctest - - su - jedi -c "CC=mpicc CXX=mpicxx FC=mpifort - && export PATH=/usr/local/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - && export LD_LIBRARY_PATH=/usr/local/lib:/jedi/home/lib - && export PYTHONPATH=/usr/local/lib:/usr/local/lib/python3.8/site-packages:$PYTHONPATH - && cd /home/jedi/iodaconv + - su - nonroot -c "cd /home/nonroot/iodaconv + && export FC=mpifort + && export CC=mpicc + && export CXX=mpicxx && ctest -C RelWithDebInfo -D ExperimentalTest" finally: - # upload ctest report to CDASH - - su - jedi -c "CC=mpicc CXX=mpicxx FC=mpifort - && cd /home/jedi/iodaconv + - su - nonroot -c "cd /home/nonroot/iodaconv + && export FC=mpifort + && export CC=mpicc + && export CXX=mpicxx && ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous" post_build: @@ -143,7 +141,7 @@ phases: # upload find cdash url and upload it as CodeBuild artifact to S3 - if [ "$BUILD_STATUS" = "1" ]; then echo "Build & tests passed, find cdash url"; - url=$(bash /jcsda/ioda-bundle/iodaconv/CI/cdash-url.sh /home/jedi/iodaconv/Testing $CDASH_URL); + url=$(bash /jcsda/ioda-bundle/iodaconv/CI/cdash-url.sh /home/nonroot/iodaconv/Testing $CDASH_URL); echo $url; echo ${url} > /jcsda/artifacts/cdash-url.txt; cat /jcsda/artifacts/cdash-url.txt; diff --git a/CI/buildspec_intel.yml b/CI/buildspec_intel.yml index ecb2cac60..e069bf3c7 100644 --- a/CI/buildspec_intel.yml +++ b/CI/buildspec_intel.yml @@ -10,6 +10,7 @@ phases: install: commands: - echo Executing install phase + - echo $CODEBUILD_RESOLVED_SOURCE_VERSION - echo $CODEBUILD_SOURCE_REPO_URL - echo $CODEBUILD_SOURCE_VERSION @@ -22,18 +23,6 @@ phases: - echo $CODEBUILD_WEBHOOK_TRIGGER - echo $CODEBUILD_WEBHOOK_BASE_REF - - - echo $PATH - - echo $LD_LIBRARY_PATH - - echo $PYTHONPATH - - . /etc/profile.d/intel.sh - - # read cdash url from s3 - - wget https://ci-test-cdash-url.s3.amazonaws.com/cdash_url.txt - - CDASH_URL=$(cat cdash_url.txt) - - - pip install xarray - pre_build: commands: - echo Executing pre_build phase @@ -42,7 +31,7 @@ phases: # Codebuild only runs on PUSH events if HEAD_REF # is refs/heads/develop (merge to develop). In this # case CODEBUILD_GIT_BRANCH="develop" - + # - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ]; then export CODEBUILD_GIT_BRANCH="develop"; echo "Merging to develop"; @@ -62,6 +51,12 @@ phases: - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt + - cd CI + + # read cdash url from s3 + - wget https://ci-test-cdash-url.s3.amazonaws.com/cdash_url.txt + - CDASH_URL=$(cat cdash_url.txt) + # Setting git credentials - sed -i '/ssh/d' ~/.gitconfig - sed '/instead/d' ~/.gitconfig @@ -70,18 +65,13 @@ phases: - chmod 0700 ~/.git-credentials - echo "https://${GIT_USER}:${GIT_PASS}@github.com" >~/.git-credentials - - cd CI - # ioda-converters (testing repo) - ./clone.sh jcsda-internal/ioda-converters $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle iodaconv ${GIT_BASE_BRANCH} develop - # jedi-cmake - - ./clone.sh jcsda-internal/jedi-cmake $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle jedicmake ${GIT_BASE_BRANCH} develop - - # oops + # oops - ./clone.sh jcsda-internal/oops $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle oops ${GIT_BASE_BRANCH} develop - # ioda + # ioda - ./clone.sh jcsda-internal/ioda $CODEBUILD_GIT_BRANCH /jcsda/ioda-bundle ioda ${GIT_BASE_BRANCH} develop # ioda-data @@ -95,26 +85,42 @@ phases: build: on-failure: CONTINUE commands: - - cd /home/jedi - - echo $PYTHONPATH - - export PYTHONPATH=/usr/local/lib:/usr/local/lib/python3.8/site-packages:$PYTHONPATH - - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_SITE=CodeBuild -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DUSE_ECCODES=ON /jcsda/ioda-bundle/ + - echo Executing build phase + - export BUILD_STATUS="0" + + - mkdir /build_container + - chmod 777 /build_container + + # These are not permitted + #- ulimit -S -s unlimited + #- ulimit -S -v unlimited + - source /etc/profile.d/z10_spack_environment.sh + - cat /etc/profile.d/z10_spack_environment.sh + - source /opt/intel/oneapi/compiler/latest/env/vars.sh + - source /opt/intel/oneapi/mpi/latest/env/vars.sh - - cd /home/jedi/ioda + # for debugging + #- which icc || true + #- which mpiicc || true + #- export | grep CC + + - cd /build_container + - ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_SITE=CodeBuild -DCDASH_OVERRIDE_SYSTEM_NAME=IntelContainer -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DUSE_ECCODES=ON /jcsda/ioda-bundle/ + + - cd /build_container/ioda - cp ../DartConfiguration.tcl . - sed -i 's/ioda-bundle/ioda-bundle\/ioda/' DartConfiguration.tcl - - sed -i 's/jedi/jedi\/ioda/' DartConfiguration.tcl + - sed -i 's/build_container/build_container\/ioda/' DartConfiguration.tcl - cat DartConfiguration.tcl - - ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4 + - ctest -C RelWithDebInfo -V -D ExperimentalBuild -j8 - - cd /home/jedi/iodaconv + - cd /build_container/iodaconv - cp ../DartConfiguration.tcl . - sed -i 's/ioda-bundle/ioda-bundle\/iodaconv/' DartConfiguration.tcl - - sed -i 's/jedi/jedi\/iodaconv/' DartConfiguration.tcl + - sed -i 's/build_container/build_container\/iodaconv/' DartConfiguration.tcl - cat DartConfiguration.tcl - - ctest -C RelWithDebInfo -V -D ExperimentalBuild -j4 + - ctest -C RelWithDebInfo -V -D ExperimentalBuild -j8 - - export BUILD_STATUS="0" - echo $BUILD_STATUS - echo $CODEBUILD_BUILD_SUCCEEDING @@ -124,10 +130,12 @@ phases: fi - echo $BUILD_STATUS + - cd /build_container/iodaconv - ctest -C RelWithDebInfo -D ExperimentalTest finally: # upload ctest report to CDASH + - cd /build_container/iodaconv - ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous post_build: @@ -137,7 +145,7 @@ phases: - if [ "$BUILD_STATUS" = "1" ]; then echo "Build & tests passed, find cdash url"; - url=$(bash /jcsda/ioda-bundle/iodaconv/CI/cdash-url.sh /home/jedi/iodaconv/Testing $CDASH_URL); + url=$(bash /jcsda/ioda-bundle/iodaconv/CI/cdash-url.sh /build_container/iodaconv/Testing $CDASH_URL); echo $url; echo ${url} > /jcsda/artifacts/cdash-url.txt; cat /jcsda/artifacts/cdash-url.txt; diff --git a/README.md b/README.md index 34a02eff1..78e284683 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,22 @@ For method option (-m) of bias and uncertainty calculation (default/nesdis), dea The land converters include all converter scripts for snowpack, soil, vegeation, and the other surface related land variables. +For snow cover fraction(scf), IMS grib2 files are supported with `ims_scf2ioda.py`. +``` +Usage: ims_scf2ioda.py -i input_ims_file.grib2 -o output_ioda_file.nc -m maskout +``` +For -i you can specify an input file and the converter will write it to one output file. For maskout option (-m) default/maskout, default means to keep all missing values and maskout means to not write out missing values. + + +For snow depth (snod), afwa grib1 files are supported with `afwa_snod2ioda.py`. +``` +Usage: afwa_snod2ioda.py -i input_afwa_file.grb -o output_ioda_file.nc -m maskout +``` +For -i you can specify an input file and the converter will write it to one output file. For maskout option (-m) default/maskout, default means to keep all missing values and maskout means to not write out missing values. + +It should be noted that both ims_scf2ioda.py and afwa_snod2ioda.py are depending on the python pygrib module. To enable the testing of these two scripts when the user has a pygrib module available, during the ecbuild process, please add -DUSE_PYGRIB=True to the ecbuild command line. + + For snow depth (snod), GHCN csv files are supported with `ghcn_snod2ioda.py`. ``` Usage: ghcn_snod2ioda.py -i input_ghcn_file.csv -o output_ioda_file.nc -f ghcn_station.txt -d YYYYMMDD -m maskout @@ -115,9 +131,15 @@ In the test case, YYYYMMDD is set 20200228. For -i you can specify an input file For surface volumetric soil moisture (ssm), SMAP NRT h5 files are supported with `smap_ssm2ioda.py`. ``` -Usage: smap_ssm2ioda.py -i input_smap_file.h5 -o output_ioda_file.nc -m maskout +Usage: smap_ssm2ioda.py -i input_smap_file.h5 -o output_ioda_file.nc --maskMissing +``` +For -i you can specify an input file and the converter will write it to one output file. --maskMissing means to not write out missing values. It should be noted that SMAP NRT h5 filename contains date and time which has been transferred to the datetime in smap_ssm2ioda.py because the data in the file does not have date and time variables. The h5 file is read with the netCDF4 module rather than the h5py module generally used. + +For surface volumetric soil moisture (ssm), SMAP 9km h5 files are supported with `smap9km_ssm2ioda.py`. +``` +Usage: smap9km_ssm2ioda.py -i input_smap9km_file.h5 -o output_ioda_file.nc --maskMissing ``` -For -i you can specify an input file and the converter will write it to one output file. For maskout option (-m) default/maskout, default means to keep all missing values and maskout means to not write out missing values. It should be noted that SMAP NRT h5 filename contains date and time which has been transferred to the datetime in smap_ssm2ioda.py because the data in the file does not have date and time variables. The h5 file is read with the netCDF4 module rather than the h5py module generally used. +For -i you can specify an input file and the converter will write it to one output file. --maskMissin means to not write out missing values. The h5 file is read with the netCDF4 module rather than the h5py module generally used. For surface volumetric soil moisture (ssm), SMOS L2 NRT Netcdf files are supported with `smos_ssm2ioda.py`. ``` diff --git a/cmake/compiler_flags_Clang_CXX.cmake b/cmake/compiler_flags_Clang_CXX.cmake index 44b7dd9bf..661dd5850 100644 --- a/cmake/compiler_flags_Clang_CXX.cmake +++ b/cmake/compiler_flags_Clang_CXX.cmake @@ -7,7 +7,7 @@ # FLAGS COMMON TO ALL BUILD TYPES #################################################################### -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Wno-c++11-extensions") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14 -g -Wall -Wno-c++11-extensions") #################################################################### # RELEASE FLAGS diff --git a/cmake/compiler_flags_GNU_CXX.cmake b/cmake/compiler_flags_GNU_CXX.cmake index 716544fac..73dc5f45a 100644 --- a/cmake/compiler_flags_GNU_CXX.cmake +++ b/cmake/compiler_flags_GNU_CXX.cmake @@ -7,7 +7,7 @@ # FLAGS COMMON TO ALL BUILD TYPES #################################################################### -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Wno-deprecated-declarations") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14 -g -Wall -Wno-deprecated-declarations") #################################################################### # RELEASE FLAGS diff --git a/cmake/compiler_flags_Intel_CXX.cmake b/cmake/compiler_flags_Intel_CXX.cmake index 484df8ac0..90844ba02 100644 --- a/cmake/compiler_flags_Intel_CXX.cmake +++ b/cmake/compiler_flags_Intel_CXX.cmake @@ -7,7 +7,7 @@ # FLAGS COMMON TO ALL BUILD TYPES #################################################################### -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -traceback") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++14 -g -traceback") #################################################################### # RELEASE FLAGS diff --git a/doc/bufr_table_samples/NCEP_NC021XXX_NormalFeed_Radiance_BUFRTable.txt b/doc/bufr_table_samples/NCEP_NC021XXX_NormalFeed_Radiance_BUFRTable.txt new file mode 100755 index 000000000..208a88cf6 --- /dev/null +++ b/doc/bufr_table_samples/NCEP_NC021XXX_NormalFeed_Radiance_BUFRTable.txt @@ -0,0 +1,658 @@ +.------------------------------------------------------------------------------. +| ------------ USER DEFINITIONS FOR TABLE-A TABLE-B TABLE D -------------- | +|------------------------------------------------------------------------------| +| MNEMONIC | NUMBER | DESCRIPTION | +|----------|--------|----------------------------------------------------------| +| | | | +| NC021021 | A61221 | MTYP 021-021 PROCESSED HIRS-2 1B Tb (NOAA-14) | +| NC021022 | A61222 | MTYP 021-022 PROCESSED MSU 1B Tb (NOAA-14) | +| NC021023 | A61223 | MTYP 021-023 PROC AMSU-A 1B Tb (NOAA-15-19, METOP-1,2) | +| NC021024 | A61224 | MTYP 021-024 PROCESSED AMSU-B 1B Tb (NOAA-15-17) | +| NC021025 | A61225 | MTYP 021-025 PROCESSED HIRS-3 1B Tb (NOAA-15-17) | +| NC021027 | A61234 | MTYP 021-027 PROCESSED MHS Tb (NOAA-18-19, METOP-1,2) | +| NC021028 | A61245 | MTYP 021-028 PROC HIRS-4 1B Tb (NOAA-18-19, METOP-1,2) | +| NC021041 | A61244 | MTYP 021-041 PROCESSED GOES IMAGER Tb | +| NC021045 | A10200 | MSG TYPE 021-045 PROC. GOES-16 ALL SKY RADIANCES (ASR) | +| NC021046 | A10199 | MSG TYPE 021-046 PROC. GOES-16 CLEAR SKY RADIANCES (CSR | +| NC021051 | A61246 | MTYP 021-051 PROC AVHRR(GAC) 1B Tb-CLR & SEA (N-17,M-2) | +| NC021052 | A61247 | MTYP 021-052 PROC AVHRR(GAC) 1B Tb-CLD OR LND(N-17,M-2) | +| NC021053 | A61248 | MTYP 021-053 PROC AVHRR(GAC) 1B Tb-CLR & SEA (NOAA-18) | +| NC021054 | A61249 | MTYP 021-054 PROC AVHRR(GAC) 1B Tb-CLD OR LND(NOAA-18) | +| NC021123 | A61227 | MTYP 021-123 PROC AMSU-A 1B Ta (NOAA-15-18, METOP-1,2) | +| NC021201 | A50242 | MTYP 021-201 DMSP SSM/IS Tb (UNIFIED PRE-PROCESSOR) | +| NC021202 | A10060 | MTYP 021-202 CrIS APODIZED RADIANCE DATA (399 CHANNEL) | +| NC021203 | A10061 | MTYP 021-203 ATMS BRIGHTNESS TEMPERATURE DATA | +| NC021205 | A10196 | MTYP 021-205 CrIS FULL SPECTRAL RADIANCE DATA (2211 CHN | +| NC021206 | A10198 | MTYP 021-206 CrIS FULL SPCTRL RADIANCE (431 CHN SUBSET | +| NC021241 | A61207 | MTYP 021-241 IASI 1C RADIANCES (VARIABLE CHNS) (METOP) | +| NC021242 | A10197 | MTYP 021-242 SAPHIR L1A2 BRIGHT. TEMPS(MEGHA-TROPIQUES) | +| NC021246 | A40192 | MTYP 021-246 GMI L1C-R BRIGHT. TEMPERATURES (GPM-CORE) | +| NC021248 | A50239 | MTYP 021-248 AMSR2 1B BRIGHTNESS TEMPERATURES (GCOM-W) | +| NC021249 | A50243 | MTYP 021-249 EVERY FOV AIRS/AMSU-A/HSB 1B BTEMPS(AQUA) | +| NC021250 | A50250 | MTYP 021-250 CENTER FOV AIRS/AMSU-A/HSB 1B BTEMPS(AQUA) | +| NC021251 | A50251 | MTYP 021-251 ATOVS AMSU-A RADIANCES | +| NC021252 | A50252 | MTYP 021-252 ATOVS AMSU-B RADIANCES | +| NC021253 | A50253 | MTYP 021-253 AIRS PRINCIPAL COMPONENTS | +| NC021254 | A50254 | MTYP 021-254 AMSR-E BRIGHTNESS TEMPERATURES (AQUA) | +| NC021255 | A61233 | MTYP 021-255 WARMEST FOV AIRS/AMSU-A/HSB 1B BTMPS(AQUA) | +| | | | +| YYMMDD | 301011 | DATE -- YEAR, MONTH, DAY | +| HHMM | 301012 | TIME -- HOUR, MINUTE | +| HHMMSS | 301013 | TIME -- HOUR, MINUTE, SECOND | +| LTLONH | 301021 | HIGH ACCURACY LATITUDE/LONGITUDE POSITION | +| SIDGRSEQ | 301071 | SATELLITE IDENTIFIER/GENERATING RESOLUTION | +| SIDENSEQ | 301072 | SATELLITE IDENTIFICATION | +| LOCPLAT | 304030 | LOCATION OF PLATFORM (SATELLITE) SEQUENCE | +| CLFRASEQ | 304032 | CLOUD FRACTION | +| CSRADSEQ | 304033 | CLEAR SKY RADIANCE | +| CLOUDCOV | 304036 | CLOUD COVERAGE | +| ALLSKYRC | 304037 | ALL SKY RADIANCE DATA | +| ATOVAMUA | 310009 | ATOVS AMSU-A REPORT | +| ATOVAMUB | 310010 | ATOVS AMSU-B REPORT | +| ATFOV | 310011 | ATOVS FIELD OF VIEW VARIABLES | +| ATCHV | 310012 | ATOVS CHANNEL VARIABLES | +| SSMISTEM | 310025 | SSM/IS TEMPERATURE DATA RECORD | +| SCO1C3IN | 310050 | SATELLITE COLOCATED 1C REPORTS WITH 3 INSTRUMENTS | +| SPITSEQN | 310051 | SATELLITE POSITION AND INSTRUMENT TEMPERATURES | +| SITPSEQN | 310052 | SATELLITE INSTRUMENT TYPE AND POSITION | +| SCBTSEQN | 310053 | SAT CHANNELS AND BRIGHT TEMPS WITH EXPANDED CHANNEL SET | +| SVCASEQN | 310054 | SAT VISB CHANNELS AND ALBEDOS WITH EXPANDED CHANNEL SET | +| SATRCPRC | 310055 | SATELLITE RADIANCE/CHANNEL PRINCIPAL COMPONENTS | +| AMSR2CHN | 310193 | AMSR2 CHANNEL SEQUENCE | +| IASIL1CB | 340002 | IASI LEVEL 1C BAND DESCRIPTION SEQUENCE | +| IASIL1CS | 340004 | IASI LEVEL 1C AVHRR SINGLE SCENE SEQUENCE | +| CRISCS | 340193 | CrIS LEVEL 1B VIIRS SINGLE SCENE SEQUENCE | +| VIIRCH | 340194 | CrIS LEVEL 1B MEAN AND STD DEV RADIANCE SEQUENCE | +| BCFQFSQ | 350200 | NPP CrIS BAND CALIBRATION & F-O-V QUALITY FLAG SEQUENCE | +| CRCHN | 350201 | NPP CrIS CHANNEL DATA | +| CRCHNM | 350216 | NPP CrIS CHANNEL DATA EXTENDED | +| GCRCHN | 350220 | NPP CrIS GUARD CHANNEL DATA | +| AMSUSPOT | 350202 | AMSU-A SPOT SEQUENCE (AQUA SATELLITE) | +| AMSUCHAN | 350203 | AMSU-A CHANNEL SEQUENCE (AQUA SATELLITE) | +| HSBSPOT | 350204 | HSB (HUMIDITY) SPOT SEQUENCE (AQUA SATELLITE) | +| HSBCHAN | 350205 | HSB (HUMIDITY) CHANNEL SEQUENCE (AQUA SATELLITE) | +| ATMSCH | 350206 | NPP ATMS CHANNEL DATA | +| INTMS | 350207 | SATELLITE INSTRUMENT TEMPERATURES | +| AIRSPC | 350208 | AIRS PRINCIPAL COMPONENT SEQUENCE | +| AIRSCORE | 350209 | AIRS PRINCIPAL COMPONENT FITS TO DATA SEQUENCE | +| AVCSEQ | 350212 | AVHRR (GAC) CHANNEL SEQUENCE | +| SDRADSQ | 360237 | STANDARD DEVIATION SEQUENCE | +| SATEPHEM | 361118 | SSM/IS SATELLITE EPHEMERIS INFORMATION | +| SCLINGEO | 361119 | SSM/IS SCAN LINE GEOMETRY | +| SSMISCHN | 361135 | SSM/IS CHANNEL SEQUENCE | +| IASICHN | 361137 | IASI LEVEL 1C SCALED RADIANCE SEQUENCE | +| AVHRCHN | 361139 | IASI LEVEL 1C MEAN AND STANDARD DEVIATION RADIANCE SEQ | +| BRITCSTC | 361171 | BRIGHTNESS TEMPERATURE SEQUENCE #2 | +| BRIT | 361182 | BRIGHTNESS TEMPERATURE SEQUENCE | +| RADS | 361191 | RADIANCE SEQUENCE | +| AMSRCHAN | 361193 | AMSR-E CHANNEL SEQUENCE (AQUA SATELLITE) | +| AMSRDICE | 361194 | AMSR-E 2 CHANNEL 4 SPOT SOUNDING SEQUENCE (AQUA SAT) | +| RPSEQ9 | 362070 | BRIGHTNESS TEMPERATURE SEQUENCE | +| SASEQF | 362072 | MEGHA-TROPIQUES SAPHIR L1A2 CHANNEL DATA | +| CHINFO | 362075 | GPM GMI CHANNEL INFORMATION | +| PIXELS | 362076 | GPM GMI PIXEL INFORMATION | +| GMICHT | 362077 | GPM GMI BRIGHTNESS TEMPERATURE DATA SEQUENCE | +| GMIANG | 362078 | GPM GMI GEOLOCATION DATA SEQUENCE | +| RPSEQ3 | 362083 | BITMAP (DATA PRESENT INDICATOR SEQUENCE) | +| RPSEQ4 | 362084 | PERCENT CONFIDENCE SEQUENCE | +| WLTMSEQN | 362087 | WARM LOAD TEMPERATURE SEQUENCE | +| MUHOSEQN | 362088 | MULTIPLEXER HOUSEKEEPING SEQUENCE | +| | | | +| SAID | 001007 | SATELLITE IDENTIFIER | +| GCLONG | 001031 | ORIGINATING/GENERATING CENTER | +| GNAP | 001032 | GENERATING APPLICATION | +| OGCE | 001033 | IDENTIFICATION OF ORIGINATING/GENERATING CENTER | +| GSES | 001034 | IDENTIFICATION OF ORIGINATING/GENERATING SUB-CENTER | +| SIID | 002019 | SATELLITE INSTRUMENTS | +| SCLF | 002020 | SATELLITE CLASSIFICATION | +| IMHC | 002024 | INTEGRATED MEAN HUMIDITY COMPUTATIONAL METHOD | +| SSNX | 002028 | SEGMENT SIZE AT NADIR IN X DIRECTION | +| SSNY | 002029 | SEGMENT SIZE AT NADIR IN Y DIRECTION | +| SSIN | 002048 | SATELLITE SENSOR INDICATOR | +| ANPO | 002104 | ANTENNA POLARIZATION | +| RAIA | 002111 | RADAR INCIDENCE ANGLE | +| MTYP | 002141 | MEASUREMENT TYPE | +| INCN | 002150 | TOVS/ATOVS/AVHRR INSTRUMENTATION CHANNEL NUMBER | +| RAID | 002151 | RADIOMETER IDENTIFIER | +| SIDP | 002152 | SATELLITE INSTRUMENT DATA USED IN PROCESSING | +| SCCF | 002153 | SATELLITE CHANNEL CENTER FREQUENCY | +| SCBW | 002154 | SATELLITE CHANNEL BAND WIDTH | +| RDTF | 002165 | RADIANCE TYPE FLAGS | +| RDTP | 002166 | RADIANCE TYPE | +| RDCM | 002167 | RADIANCE COMPUTATIONAL METHOD | +| YEAR | 004001 | YEAR | +| MNTH | 004002 | MONTH | +| DAYS | 004003 | DAY | +| HOUR | 004004 | HOUR | +| MINU | 004005 | MINUTE | +| SECO | 004006 | SECOND | +| TPSE | 004026 | TIME PERIOD OR DISPLACEMENT | +| CLATH | 005001 | LATITUDE (HIGH ACCURACY) | +| CLAT | 005002 | LATITUDE (COARSE ACCURACY) | +| BEARAZ | 005021 | BEARING OR AZIMUTH | +| SOLAZI | 005022 | SOLAR AZIMUTH | +| ORBN | 005040 | ORBIT NUMBER | +| SLNM | 005041 | SCAN LINE NUMBER | +| CHNM | 005042 | CHANNEL NUMBER | +| FOVN | 005043 | FIELD OF VIEW NUMBER | +| FORN | 005045 | FIELD OF REGARD NUMBER | +| YAPCG | 005060 | Y ANGULAR POSITION OF CENTER OF GRAVITY | +| ZAPCG | 005061 | Z ANGULAR POSITION OF CENTER OF GRAVITY | +| CLONH | 006001 | LONGITUDE (HIGH ACCURACY) | +| CLON | 006002 | LONGITUDE (COARSE ACCURACY) | +| WVNM | 006029 | WAVE NUMBER | +| SELV | 007001 | HEIGHT OF STATION | +| HMSL | 007002 | HEIGHT OR ALTITUDE | +| PRLC | 007004 | PRESSURE | +| SOEL | 007022 | SOLAR ELEVATION | +| SAZA | 007024 | SATELLITE ZENITH ANGLE | +| SOZA | 007025 | SOLAR ZENITH ANGLE | +| SSGA | 007192 | SATELLITE-SUN GLINT ANGLE | +| VSAT | 008003 | VERTICAL SIGNIFICANCE (SATELLITE OBSERVATIONS) | +| DIMS | 008007 | DIMENSIONAL SIGNIFICANCE | +| METFET | 008011 | METEOROLOGICAL FEATURE | +| LSQL | 008012 | LAND/SEA QUALIFIER | +| TSIG | 008021 | TIME SIGNIFICANCE | +| FOST | 008023 | FIRST ORDER STATISTICS | +| MDPC | 008033 | METHOD OF DEVIATION OF PERCENTAGE CONFIDENCE | +| SGIN | 008065 | SUN-GLINT INDICATOR | +| TAPQ | 008070 | TOVS/ATOVS PRODUCT IDENTIFIER | +| STKO | 008075 | ASCENDING/DESCENDING ORBIT QUALIFIER | +| TOBD | 008076 | TYPE OF BAND | +| HOLS | 010001 | HEIGHT OF LAND SURFACE | +| HITE | 010002 | GEOPOTENTIAL HEIGHT | +| PDNP | 010031 | IN DIR. OF NORTH POLE, DISTANCE FROM THE EARTH'S CENTER | +| TMBRST | 012063 | BRIGHTNESS TEMPERATURE | +| TMINST | 012064 | INSTRUMENT TEMPERATURE | +| SDTB | 012065 | STANDARD DEVIATION BRIGHTNESS TEMPERATURE | +| TMANT | 012066 | ANTENNA TEMPERATURE | +| WLTM | 012070 | WARM LOAD TEMPERATURE | +| SPRD | 012075 | SPECTRAL RADIANCE | +| RDNE | 012076 | RADIANCE | +| NEDTCO | 012158 | NOISE-EQUIVALENT DELTA TEMPERATURE VIEWING COLD TARGET | +| NEDTWA | 012159 | NOISE-EQUIVALENT DELTA TEMPERATURE VIEWING WARM TARGET | +| TMBR | 012163 | BRIGHTNESS TEMPERATURE (HIGH ACCURACY) | +| CSTC | 012206 | COLD SPACE TEMPERATURE CORRECTION | +| REHU | 013003 | RELATIVE HUMIDITY | +| SFLG | 013040 | SURFACE FLAG | +| ALBD | 014027 | ALBEDO | +| SCHRAD | 014043 | CHANNEL RADIANCE | +| SRAD | 014044 | CHANNEL RADIANCE | +| SCRA | 014046 | SCALED IASI RADIANCE | +| SMRA | 014047 | SCALED MEAN AVHRR RADIANCE | +| SSDR | 014048 | SCALED STANDARD DEVIATION OF AVHRR RADIANCE | +| TOCC | 020010 | CLOUD COVER (TOTAL) | +| CLTP | 020012 | CLOUD TYPE | +| HOCT | 020014 | HEIGHT OF TOP OF CLOUD | +| RFLAG | 020029 | RAIN FLAG | +| CLDMNT | 020081 | CLOUD AMOUNT IN SEGMENT | +| NCLDMNT | 020082 | AMOUNT SEGMENT CLOUD FREE | +| CLAVR | 020199 | CLOUD FROM AVHRR (CLAVR) CLOUD MASK | +| WTCA | 021083 | WARM TARGET CALIBRATION | +| CTCA | 021084 | COLD TARGET CALIBRATION | +| ALFR | 021166 | LAND FRACTION | +| APCOMP | 025050 | PRINCIPAL COMPONENT SCORE | +| AVHCST | 025051 | AVHRR CHANNEL COMBINATION | +| APCFIT | 025052 | LOG OF PRINCIPAL COMPONENTS NORMALIZED FIT TO DATA | +| SSID | 025054 | SSMIS SUBFRAME ID NUMBER | +| MUHO | 025055 | MULTIPLEXOR HOUSEKEEPING | +| MJFC | 025070 | MAJOR FRAME COUNT | +| SACV | 025075 | SATELLITE ANTENNA CORRECTIONS VERSION NUMBER | +| LOGRCW | 025076 | LOG-10 OF (TEMPERATURE-RADIANCE CENTRAL WAVENUMBER) | +| BWCC1 | 025077 | BANDWIDTH CORRECTION COEFFICIENT 1 FOR ATOVS | +| BWCC2 | 025078 | BANDWIDTH CORRECTION COEFFICIENT 2 FOR ATOVS | +| IANG | 025081 | INCIDENCE ANGLE | +| AANG | 025082 | AZIMUTH ANGLE | +| FCPH | 025085 | FRACTION OF CLEAR PIXELS IN HIRS FOV | +| STCH | 025140 | START CHANNEL | +| ENCH | 025141 | END CHANNEL | +| CHSF | 025142 | CHANNEL SCALE FACTOR | +| PD00 | 027031 | IN DIRECTION OF 0 DEG E, DISTANCE FROM EARTH'S CENTER | +| PD90 | 028031 | IN DIRECTION OF 90 DEG E, DISTANCE FROM EARTH'S CENTER | +| NPPR | 030021 | NUMBER OF PIXELS PER ROW | +| NPPC | 030022 | NUMBER OF PIXELS PER COLUMN | +| DPRI | 031031 | DATA PRESENT INDICATOR | +| QMRKH | 033003 | QUALITY INFORMATION | +| PCCF | 033007 | PERCENT CONFIDENCE | +| SLSF | 033030 | SCAN LINE STATUS FLAGS FOR ATOVS | +| SLQF | 033031 | SCAN LINE QUALITY FLAGS FOR ATOVS | +| ACQF | 033032 | CHANNEL QUALITY FLAGS FOR ATOVS | +| FOVQ | 033033 | FIELD OF VIEW QUALITY FLAGS FOR ATOVS | +| QGFQ | 033060 | INDIVIDUAL IASI-SYSTEM QUALITY FLAG | +| QGQI | 033061 | INSTR. NOISE PERF. INDICATOR (SPECTRAL & RADIOMETRIC) | +| QGQIL | 033062 | GEOMETRIC QUALITY INDEX INDICATOR | +| QGQIR | 033063 | INSTR. NOISE PERF. INDICATOR (RADIOMETRIC CALIBRATION) | +| QGQIS | 033064 | INSTRUMENT NOISE PERF. INDICATOR (SPECTRAL CALIBRATION) | +| QGSSQ | 033065 | OUTPUT OF TEC FUNCTION | +| NSQF | 033075 | SCAN LEVEL QUALITY FLAGS | +| NCQF | 033076 | CALIBRATION QUALITY FLAGS | +| NFQF | 033077 | FIELD OF VIEW QUALITY FLAGS | +| NGQI | 033078 | GEOLOCATION QUALITY | +| ATMSGQ | 033079 | GRANULE LEVEL QUALITY FLAGS | +| ATMSSQ | 033080 | SCAN LEVEL QUALITY FLAGS | +| ATMSCHQ | 033081 | CHANNEL DATA QUALITY FLAGS | +| VIIRSQ | 033083 | RADIANCE DATA QUALITY FLAGS | +| TPQC2 | 033254 | 2-BIT INDICATOR OF QUALITY | +| RSRD | 035200 | RESTRICTIONS ON REDISTRIBUTION | +| EXPRSRD | 035201 | EXPIRATION OF RESTRICTIONS ON REDISTRIBUTION | +| SLQFS | 050210 | SCAN LINE QUALITY FLAGS FOR FOR SAPHIR/MADRAS | +| PLMD | 050211 | PAYLOAD MODE FOR SAPHIR | +| SMODE | 050213 | SATELLITE MODE FOR SAPHIR/MADRAS | +| CHQF | 050214 | CHANNEL QUALITY FLAG FOR SAPHIR/MADRAS | +| CLFG | 050215 | CALIBRATION FLAGS FOR SAPHIR/MADRAS | +| | | | +|------------------------------------------------------------------------------| +| MNEMONIC | SEQUENCE | +|----------|-------------------------------------------------------------------| +| | | +| NC021021 | YEAR MNTH DAYS HOUR MINU SECO CLAT CLON SAID SIID FOVN | +| NC021021 | LSQL SAZA SOZA HOLS 202127 HMSL 202000 "BRIT"20 | +| | | +| NC021022 | YEAR MNTH DAYS HOUR MINU SECO CLAT CLON SAID SIID FOVN | +| NC021022 | LSQL SAZA SOZA HOLS 202127 HMSL 202000 "BRIT"4 | +| | | +| NC021023 | YEAR MNTH DAYS HOUR MINU SECO 207002 CLAT CLON 207000 | +| NC021023 | SAID SIID FOVN LSQL SAZA SOZA HOLS 202127 HMSL 202000 | +| NC021023 | SOLAZI BEARAZ "BRITCSTC"15 | +| | | +| NC021024 | YEAR MNTH DAYS HOUR MINU SECO 207002 CLAT CLON 207000 | +| NC021024 | SAID SIID FOVN LSQL SAZA SOZA HOLS 202127 HMSL 202000 | +| NC021024 | SOLAZI BEARAZ "BRITCSTC"5 | +| | | +| NC021025 | YEAR MNTH DAYS HOUR MINU SECO 207002 CLAT CLON 207000 | +| NC021025 | SAID SIID FOVN LSQL SAZA SOZA HOLS 202127 HMSL 202000 | +| NC021025 | SOLAZI BEARAZ "BRIT"20 | +| | | +| NC021027 | YEAR MNTH DAYS HOUR MINU SECO 207002 CLAT CLON 207000 | +| NC021027 | SAID SIID FOVN LSQL SAZA SOZA HOLS 202127 HMSL 202000 | +| NC021027 | SOLAZI BEARAZ "BRITCSTC"5 | +| | | +| NC021028 | YEAR MNTH DAYS HOUR MINU SECO 207002 CLAT CLON 207000 | +| NC021028 | SAID SIID FOVN LSQL SAZA SOZA HOLS 202127 HMSL 202000 | +| NC021028 | SOLAZI BEARAZ "BRIT"20 | +| | | +| NC021041 | SAID GCLONG SCLF SSNX SSNY NPPR NPPC YEAR MNTH DAYS | +| NC021041 | HOUR MINU SECO CLAT CLON SAZA SOZA LSQL "RADS"6 | +| | | +| NC021045 | SIDENSEQ NPPR NPPC SAZA BEARAZ SOZA SOLAZI HITE CLOUDCOV | +| NC021045 | SIDP RDCM "ALLSKYRC"10 "RPSEQ3"174 GCLONG GNAP "RPSEQ4"60 | +| NC021045 | GCLONG GNAP FOST "RPSEQ9"60 | +| | | +| NC021046 | SIDENSEQ NPPR NPPC LSQL SAZA BEARAZ SOZA SOLAZI HITE | +| NC021046 | "CLFRASEQ"10 SIDP IMHC PRLC PRLC REHU SIDP IMHC PRLC | +| NC021046 | PRLC REHU "CSRADSEQ"10 "RPSEQ3"161 GCLONG GNAP "RPSEQ4"30 | +| NC021046 | GCLONG GNAP MDPC "RPSEQ4"30 GCLONG GNAP MDPC "RPSEQ4"30 | +| NC021046 | GCLONG GNAP MDPC "RPSEQ4"30 GCLONG GNAP MDPC "RPSEQ4"30 | +| NC021046 | GCLONG GNAP FOST "SDRADSQ"10 GCLONG GNAP FOST "SDRADSQ"10 | +| | | +| NC021051 | YEAR MNTH DAYS HOUR MINU 202129 201132 SECO 201000 | +| NC021051 | 202000 CLATH CLONH SAID 201129 FOVN 201000 SAZA SOZA | +| NC021051 | CLAVR "AVCSEQ"5 | +| | | +| NC021052 | YEAR MNTH DAYS HOUR MINU 202129 201132 SECO 201000 | +| NC021052 | 202000 CLATH CLONH SAID 201129 FOVN 201000 SAZA SOZA | +| NC021052 | CLAVR "AVCSEQ"5 | +| | | +| NC021053 | YEAR MNTH DAYS HOUR MINU 202129 201132 SECO 201000 | +| NC021053 | 202000 CLATH CLONH SAID 201129 FOVN 201000 SAZA SOZA | +| NC021053 | CLAVR "AVCSEQ"5 | +| | | +| NC021054 | YEAR MNTH DAYS HOUR MINU 202129 201132 SECO 201000 | +| NC021054 | 202000 CLATH CLONH SAID 201129 FOVN 201000 SAZA SOZA | +| NC021054 | CLAVR "AVCSEQ"5 | +| | | +| NC021123 | YEAR MNTH DAYS HOUR MINU SECO 207002 CLAT CLON 207000 | +| NC021123 | SAID SIID FOVN LSQL SAZA SOZA HOLS 202127 HMSL 202000 | +| NC021123 | SOLAZI BEARAZ "BRITCSTC"15 | +| | | +| NC021201 | SSMISTEM | +| | | +| NC021202 | SAID OGCE SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021202 | LOCPLAT LTLONH SAZA BEARAZ SOZA SOLAZI STKO 201133 SLNM | +| NC021202 | 201000 FORN FOVN ORBN HOLS 201129 HMSL 201000 202127 | +| NC021202 | 201125 ALFR 201000 202000 LSQL TOCC HOCT RDTF NSQF | +| NC021202 | "BCFQFSQ"3 TOBD NGQI QMRKH (CRCHN) | +| | | +| NC021203 | SAID OGCE GSES SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021203 | ORBN SLNM FOVN ATMSGQ ATMSSQ NGQI LTLONH 201129 HMSL | +| NC021203 | 201000 SAZA BEARAZ SOZA SOLAZI SACV (ATMSCH) | +| | | +| NC021205 | SAID OGCE SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021205 | LOCPLAT LTLONH SAZA BEARAZ SOZA SOLAZI STKO 201133 SLNM | +| NC021205 | 201000 FORN FOVN ORBN HOLS 201129 HMSL 201000 202127 | +| NC021205 | 201125 ALFR 201000 202000 LSQL TOCC HOCT RDTF NSQF | +| NC021205 | "BCFQFSQ"3 TOBD NGQI QMRKH (CRCHNM) MTYP TOBD {GCRCHN} | +| NC021205 | TOBD SIID "CRISCS"7 | +| | | +| NC021206 | SAID OGCE SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021206 | LOCPLAT LTLONH SAZA BEARAZ SOZA SOLAZI STKO 201133 SLNM | +| NC021206 | 201000 FORN FOVN ORBN HOLS 201129 HMSL 201000 202127 | +| NC021206 | 201125 ALFR 201000 202000 LSQL TOCC HOCT RDTF NSQF | +| NC021206 | "BCFQFSQ"3 TOBD NGQI QMRKH (CRCHNM) MTYP TOBD {GCRCHN} | +| NC021206 | TOBD SIID "CRISCS"7 | +| | | +| NC021241 | SAID GCLONG SIID SCLF YEAR MNTH DAYS HOUR MINU 202131 | +| NC021241 | 201138 SECO 201000 202000 CLATH CLONH SAZA BEARAZ SOZA | +| NC021241 | SOLAZI FOVN ORBN 201133 SLNM 201000 201132 MJFC 201000 | +| NC021241 | 202126 SELV 202000 QGFQ QGQI QGQIL QGQIR QGQIS QGSSQ | +| NC021241 | "IASIL1CB"10 (IASICHN) SIID AVHCST "IASIL1CS"7 | +| | | +| NC021242 | RSRD EXPRSRD TAPQ OGCE GSES TAPQ OGCE GSES SAID SIID | +| NC021242 | ORBN SACV 201133 SLNM 201000 FOVN MJFC SLQFS PLMD YEAR | +| NC021242 | MNTH DAYS HOUR MINU 202131 201138 SECO 201000 202000 | +| NC021242 | CLATH CLONH 202126 SELV 202000 SAZA BEARAZ SOZA SOLAZI | +| NC021242 | IANG SMODE RAID TMINST RAID TMINST RAID TMINST RAID | +| NC021242 | TMINST "SASEQF"6 | +| | | +| NC021246 | SAID SIID OGCE GSES "CHINFO"13 SLNM YYMMDD HHMM 207003 | +| NC021246 | SECO 207000 201129 HMSL 201000 (PIXELS) | +| | | +| NC021248 | SAID OGCE GSES SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021248 | ORBN 201133 SLNM FOVN 201000 LTLONH SOLAZI SOEL IANG | +| NC021248 | AANG ACQF 208006 MTYP 208000 {AMSR2CHN} | +| | | +| NC021249 | SCO1C3IN | +| | | +| NC021250 | SCO1C3IN | +| | | +| NC021251 | ATOVAMUA | +| | | +| NC021252 | ATOVAMUB | +| | | +| NC021253 | SATRCPRC | +| | | +| NC021254 | SPITSEQN SITPSEQN "AMSRCHAN"12 "AMSRDICE"4 | +| | | +| NC021255 | SCO1C3IN | +| | | +| YYMMDD | YEAR MNTH DAYS | +| | | +| HHMM | HOUR MINU | +| | | +| HHMMSS | HOUR MINU SECO | +| | | +| LTLONH | CLATH CLONH | +| | | +| SIDGRSEQ | SAID GCLONG SCLF SSNX SSNY | +| | | +| SIDENSEQ | SIDGRSEQ YYMMDD HHMMSS LTLONH | +| | | +| LOCPLAT | PD00 PD90 PDNP | +| | | +| CLFRASEQ | SCCF SCBW CLDMNT NCLDMNT CLTP | +| | | +| CSRADSEQ | SIDP RDTP RDCM SCCF SCBW SPRD RDNE TMBRST | +| | | +| CLOUDCOV | NCLDMNT LSQL NCLDMNT LSQL CLDMNT VSAT CLDMNT VSAT CLDMNT | +| CLOUDCOV | VSAT CLDMNT VSAT | +| | | +| ALLSKYRC | SCCF SCBW TMBRST METFET TMBRST METFET TMBRST METFET VSAT | +| ALLSKYRC | TMBRST VSAT TMBRST VSAT TMBRST | +| | | +| ATOVAMUA | ATFOV "ATCHV"15 | +| | | +| ATOVAMUB | ATFOV "ATCHV"5 | +| | | +| ATFOV | TAPQ OGCE GSES TAPQ OGCE GSES SAID SSIN ORBN SACV | +| ATFOV | 201133 SLNM 201000 FOVN MJFC SLSF SLQF YEAR MNTH DAYS | +| ATFOV | HOUR MINU 202131 201138 SECO 201000 202000 CLATH CLONH | +| ATFOV | 202126 SELV 202000 SAZA BEARAZ SOZA SOLAZI FOVQ RAID | +| ATFOV | TMINST RAID TMINST RAID TMINST RAID TMINST | +| | | +| ATCHV | INCN LOGRCW BWCC1 BWCC2 ACQF 201132 202129 TMBRST 202000 | +| ATCHV | 201000 | +| | | +| SSMISTEM | SAID TSIG YEAR MNTH DAYS HOUR MINU 201138 202131 SECO | +| SSMISTEM | 202000 201000 201132 SLNM 201000 201129 FOVN 201000 CLAT | +| SSMISTEM | CLON SFLG RFLAG "SSMISCHN"24 "SATEPHEM"3 TSIG YEAR MNTH | +| SSMISTEM | DAYS HOUR MINU ORBN "WLTMSEQN"3 SSID "MUHOSEQN"4 DIMS | +| SSMISTEM | "SCLINGEO"28 | +| | | +| SCO1C3IN | SPITSEQN SITPSEQN (SCBTSEQN) "SVCASEQN"4 TOCC AMSUSPOT | +| SCO1C3IN | "AMSUCHAN"15 HSBSPOT "HSBCHAN"5 | +| | | +| SPITSEQN | SAID ORBN 201133 SLNM 201000 201132 MJFC 201000 202126 | +| SPITSEQN | SELV 202000 SOZA SOLAZI "INTMS"9 | +| | | +| SITPSEQN | SIID YEAR MNTH DAYS HOUR MINU 202131 201138 SECO 201000 | +| SITPSEQN | 202000 CLATH CLONH SAZA BEARAZ FOVN | +| | | +| SCBTSEQN | 201134 CHNM 201000 LOGRCW ACQF TMBR | +| | | +| SVCASEQN | 201134 CHNM 201000 LOGRCW ACQF 201131 202129 FOST ALBD | +| SVCASEQN | FOST ALBD FOST 202000 201000 | +| | | +| SATRCPRC | SPITSEQN SITPSEQN "AIRSCORE"20 (AIRSPC) | +| | | +| AMSR2CHN | SCCF ALFR VIIRSQ ANPO TMBR | +| | | +| IASIL1CB | STCH ENCH CHSF | +| | | +| IASIL1CS | YAPCG ZAPCG FCPH "AVHRCHN"6 | +| | | +| CRISCS | YAPCG ZAPCG FCPH "VIIRCH"16 | +| | | +| VIIRCH | CHNM FOST SCHRAD FOST SCHRAD | +| | | +| BCFQFSQ | TOBD WVNM WVNM STCH ENCH NCQF NFQF | +| | | +| CRCHN | 201133 CHNM 201000 SRAD | +| | | +| CRCHNM | 201134 CHNM 201000 SRAD | +| | | +| GCRCHN | 201134 CHNM 201000 SRAD | +| | | +| AMSUSPOT | SIID YEAR MNTH DAYS HOUR MINU 202131 201138 SECO 201000 | +| AMSUSPOT | 202000 CLATH CLONH SAZA BEARAZ FOVN | +| | | +| AMSUCHAN | 201134 CHNM 201000 LOGRCW ACQF TMBR | +| | | +| HSBSPOT | SIID YEAR MNTH DAYS HOUR MINU 202131 201138 SECO 201000 | +| HSBSPOT | 202000 CLATH CLONH SAZA BEARAZ FOVN | +| | | +| HSBCHAN | 201134 CHNM 201000 LOGRCW ACQF TMBR | +| | | +| ATMSCH | CHNM 202131 SCCF SCBW 202000 ANPO TMANT TMBR NEDTCO | +| ATMSCH | NEDTWA ATMSCHQ | +| | | +| INTMS | RAID TMINST | +| | | +| AIRSPC | APCOMP | +| | | +| AIRSCORE | LOGRCW APCFIT | +| | | +| AVCSEQ | INCN ALBD TMBR | +| | | +| SDRADSQ | SPRD RDNE SDTB | +| | | +| SATEPHEM | YEAR MNTH DAYS 201142 202131 TPSE 202000 201000 CLATH | +| SATEPHEM | CLONH 201138 202129 SELV 202000 201000 | +| | | +| SCLINGEO | CLAT CLON RAIA BEARAZ | +| | | +| SSMISCHN | CHNM TMBR WTCA CTCA | +| | | +| IASICHN | 201136 CHNM 201000 SCRA | +| | | +| AVHRCHN | CHNM CHSF SMRA CHSF SSDR | +| | | +| BRITCSTC | CHNM TMBR CSTC | +| | | +| BRIT | CHNM TMBR | +| | | +| RADS | SIDP RDTP RDCM SCCF SCBW SPRD RDNE 201132 202129 TMBRST | +| RADS | 202000 201000 CLDMNT NCLDMNT CLTP 202129 SDTB 202000 PCCF | +| | | +| AMSRCHAN | 201134 CHNM 201000 LOGRCW ACQF TMBR | +| | | +| AMSRDICE | SITPSEQN 201129 FOVN 201000 "AMSRCHAN"2 | +| | | +| RPSEQ9 | SDTB | +| | | +| SASEQF | INCN LOGRCW BWCC1 BWCC2 CHQF CLFG SGIN SFLG 201132 | +| SASEQF | 202129 TMBRST 202000 201000 | +| | | +| CHINFO | CHNM SCCF SCBW ANPO | +| | | +| PIXELS | LTLONH FOVN "GMIANG"2 "GMICHT"13 | +| | | +| GMICHT | CHNM TPQC2 TMBR VIIRSQ | +| | | +| GMIANG | SAZA BEARAZ SOZA SOLAZI SSGA | +| | | +| RPSEQ3 | DPRI | +| | | +| RPSEQ4 | PCCF | +| | | +| WLTMSEQN | WLTM | +| | | +| MUHOSEQN | MUHO | +| | | +|------------------------------------------------------------------------------| +| MNEMONIC | SCAL | REFERENCE | BIT | UNITS |-------------| +|----------|------|-------------|-----|--------------------------|-------------| +| | | | | |-------------| +| SAID | 0 | 0 | 10 | CODE TABLE |-------------| +| GCLONG | 0 | 0 | 16 | CODE TABLE |-------------| +| GNAP | 0 | 0 | 8 | CODE TABLE |-------------| +| OGCE | 0 | 0 | 8 | CODE TABLE |-------------| +| GSES | 0 | 0 | 8 | CODE TABLE |-------------| +| SIID | 0 | 0 | 11 | CODE TABLE |-------------| +| SCLF | 0 | 0 | 9 | CODE TABLE |-------------| +| IMHC | 0 | 0 | 4 | CODE TABLE |-------------| +| SSNX | 0 | 0 | 18 | M |-------------| +| SSNY | 0 | 0 | 18 | M |-------------| +| SSIN | 0 | 0 | 4 | CODE TABLE |-------------| +| ANPO | 0 | 0 | 4 | CODE TABLE |-------------| +| RAIA | 1 | 0 | 10 | DEGREE |-------------| +| MTYP | 0 | 0 | 24 | CCITT IA5 |-------------| +| INCN | 0 | 0 | 6 | CODE TABLE |-------------| +| RAID | 0 | 0 | 11 | CODE TABLE |-------------| +| SIDP | 0 | 0 | 31 | FLAG TABLE |-------------| +| SCCF | -8 | 0 | 26 | HZ |-------------| +| SCBW | -8 | 0 | 26 | HZ |-------------| +| RDTF | 0 | 0 | 15 | FLAG TABLE |-------------| +| RDTP | 0 | 0 | 4 | CODE TABLE |-------------| +| RDCM | 0 | 0 | 4 | CODE TABLE |-------------| +| YEAR | 0 | 0 | 12 | YEAR |-------------| +| MNTH | 0 | 0 | 4 | MONTH |-------------| +| DAYS | 0 | 0 | 6 | DAY |-------------| +| HOUR | 0 | 0 | 5 | HOUR |-------------| +| MINU | 0 | 0 | 6 | MINUTE |-------------| +| SECO | 0 | 0 | 6 | SECOND |-------------| +| TPSE | 0 | -4096 | 13 | SECOND |-------------| +| CLATH | 5 | -9000000 | 25 | DEGREE |-------------| +| CLAT | 2 | -9000 | 15 | DEGREE |-------------| +| BEARAZ | 2 | 0 | 16 | DEGREE TRUE |-------------| +| SOLAZI | 2 | 0 | 16 | DEGREE TRUE |-------------| +| ORBN | 0 | 0 | 24 | NUMERIC |-------------| +| SLNM | 0 | 0 | 8 | NUMERIC |-------------| +| CHNM | 0 | 0 | 6 | NUMERIC |-------------| +| FOVN | 0 | 0 | 8 | NUMERIC |-------------| +| FORN | 0 | 0 | 8 | NUMERIC |-------------| +| YAPCG | 6 | -8000000 | 24 | DEGREE |-------------| +| ZAPCG | 6 | -8000000 | 24 | DEGREE |-------------| +| CLONH | 5 | -18000000 | 26 | DEGREE |-------------| +| CLON | 2 | -18000 | 16 | DEGREE |-------------| +| WVNM | 1 | 0 | 22 | M**-1 |-------------| +| SELV | 0 | -400 | 15 | M |-------------| +| HMSL | -1 | -40 | 16 | M |-------------| +| PRLC | -1 | 0 | 14 | PASCALS |-------------| +| SOEL | 2 | -9000 | 15 | DEGREE |-------------| +| SAZA | 2 | -9000 | 15 | DEGREE |-------------| +| SOZA | 2 | -9000 | 15 | DEGREE |-------------| +| SSGA | 1 | 0 | 11 | DEGREE |-------------| +| VSAT | 0 | 0 | 6 | CODE TABLE |-------------| +| DIMS | 0 | 0 | 4 | CODE TABLE |-------------| +| METFET | 0 | 0 | 6 | CODE TABLE |-------------| +| LSQL | 0 | 0 | 2 | CODE TABLE |-------------| +| TSIG | 0 | 0 | 5 | CODE TABLE |-------------| +| FOST | 0 | 0 | 6 | CODE TABLE |-------------| +| MDPC | 0 | 0 | 7 | CODE TABLE |-------------| +| SGIN | 0 | 0 | 2 | CODE TABLE |-------------| +| TAPQ | 0 | 0 | 4 | CODE TABLE |-------------| +| STKO | 0 | 0 | 2 | CODE TABLE |-------------| +| TOBD | 0 | 0 | 6 | CODE TABLE |-------------| +| HOLS | 0 | -400 | 15 | M |-------------| +| HITE | -1 | -40 | 16 | M |-------------| +| PDNP | 2 | -1073741824 | 31 | M |-------------| +| TMBRST | 1 | 0 | 12 | KELVIN |-------------| +| TMINST | 1 | 0 | 12 | KELVIN |-------------| +| SDTB | 1 | 0 | 12 | KELVIN |-------------| +| TMANT | 2 | 0 | 16 | K |-------------| +| WLTM | 2 | 0 | 16 | KELVIN |-------------| +| SPRD | -3 | 0 | 16 | WM**(-3)SR**(-1) |-------------| +| RDNE | 3 | 0 | 16 | WM**(-2)SR**(-1) |-------------| +| NEDTCO | 2 | 0 | 12 | K |-------------| +| NEDTWA | 2 | 0 | 12 | K |-------------| +| TMBR | 2 | 0 | 16 | KELVIN |-------------| +| CSTC | 2 | 0 | 10 | KELVIN |-------------| +| REHU | 0 | 0 | 7 | % |-------------| +| SFLG | 0 | 0 | 4 | CODE TABLE |-------------| +| ALBD | 0 | 0 | 7 | % |-------------| +| SCHRAD | 4 | 0 | 23 | W M**-2 SR**-1 UM**-1 |-------------| +| SRAD | 7 | -100000 | 22 | W M**-2 SR**-1 CM |-------------| +| SCRA | 0 | 0 | 16 | W M**-2 SR**-1 M |-------------| +| SMRA | 0 | 0 | 31 | W M**-2 SR**-1 M |-------------| +| SSDR | 0 | 0 | 31 | W M**-2 SR**-1 M |-------------| +| TOCC | 0 | 0 | 7 | % |-------------| +| CLTP | 0 | 0 | 6 | CODE TABLE |-------------| +| HOCT | -1 | -40 | 11 | M |-------------| +| RFLAG | 0 | 0 | 2 | CODE TABLE |-------------| +| CLDMNT | 0 | 0 | 7 | % |-------------| +| NCLDMNT | 0 | 0 | 7 | % |-------------| +| CLAVR | 0 | 0 | 3 | CODE TABLE |-------------| +| WTCA | 0 | 0 | 16 | NUMERIC |-------------| +| CTCA | 0 | 0 | 16 | NUMERIC |-------------| +| ALFR | 3 | 0 | 10 | NUMERIC |-------------| +| APCOMP | 4 | -131072 | 18 | NUMERIC |-------------| +| AVHCST | 0 | 0 | 7 | FLAG TABLE |-------------| +| APCFIT | 4 | 0 | 15 | NUMERIC |-------------| +| SSID | 0 | 0 | 5 | NUMERIC |-------------| +| MUHO | 2 | 0 | 16 | KELVIN |-------------| +| MJFC | 0 | 0 | 4 | NUMERIC |-------------| +| SACV | 0 | 0 | 5 | NUMERIC |-------------| +| LOGRCW | 8 | 0 | 30 | LOG/M |-------------| +| BWCC1 | 5 | -100000 | 18 | NUMERIC |-------------| +| BWCC2 | 5 | 0 | 17 | NUMERIC |-------------| +| IANG | 3 | 0 | 17 | DEGREE |-------------| +| AANG | 3 | 0 | 19 | DEGREE |-------------| +| FCPH | 0 | 0 | 7 | NUMERIC |-------------| +| STCH | 0 | 0 | 14 | NUMERIC |-------------| +| ENCH | 0 | 0 | 14 | NUMERIC |-------------| +| CHSF | 0 | 0 | 6 | NUMERIC |-------------| +| PD00 | 2 | -1073741824 | 31 | M |-------------| +| PD90 | 2 | -1073741824 | 31 | M |-------------| +| NPPR | 0 | 0 | 12 | NUMERIC |-------------| +| NPPC | 0 | 0 | 12 | NUMERIC |-------------| +| DPRI | 0 | 0 | 1 | FLAG TABLE |-------------| +| QMRKH | 0 | 0 | 3 | CODE TABLE |-------------| +| PCCF | 0 | 0 | 7 | PERCENT |-------------| +| SLSF | 0 | 0 | 24 | FLAG TABLE |-------------| +| SLQF | 0 | 0 | 24 | FLAG TABLE |-------------| +| ACQF | 0 | 0 | 24 | FLAG TABLE |-------------| +| FOVQ | 0 | 0 | 24 | FLAG TABLE |-------------| +| QGFQ | 0 | 0 | 2 | CODE TABLE |-------------| +| QGQI | 0 | 0 | 7 | % |-------------| +| QGQIL | 0 | 0 | 7 | % |-------------| +| QGQIR | 0 | 0 | 7 | % |-------------| +| QGQIS | 0 | 0 | 7 | % |-------------| +| QGSSQ | 0 | 0 | 24 | NUMERIC |-------------| +| NSQF | 0 | 0 | 13 | FLAG TABLE |-------------| +| NCQF | 0 | 0 | 9 | FLAG TABLE |-------------| +| NFQF | 0 | 0 | 19 | FLAG TABLE |-------------| +| NGQI | 0 | 0 | 4 | CODE TABLE |-------------| +| ATMSGQ | 0 | 0 | 16 | FLAG TABLE |-------------| +| ATMSSQ | 0 | 0 | 20 | FLAG TABLE |-------------| +| ATMSCHQ | 0 | 0 | 12 | FLAG TABLE |-------------| +| VIIRSQ | 0 | 0 | 16 | FLAG TABLE |-------------| +| TPQC2 | 0 | 0 | 2 | CODE TABLE |-------------| +| RSRD | 0 | 0 | 9 | FLAG TABLE |-------------| +| EXPRSRD | 0 | 0 | 8 | HOURS |-------------| +| SLQFS | 0 | 0 | 10 | FLAG TABLE |-------------| +| PLMD | 0 | 0 | 3 | CODE TABLE |-------------| +| SMODE | 0 | 0 | 4 | CODE TABLE |-------------| +| CHQF | 0 | 0 | 11 | FLAG TABLE |-------------| +| CLFG | 0 | 0 | 3 | CODE TABLE |-------------| +| | | | | |-------------| +`------------------------------------------------------------------------------' diff --git a/doc/bufr_table_samples/NCEP_NC021XXX_RARS-DB_Radiance_BUFRTable.txt b/doc/bufr_table_samples/NCEP_NC021XXX_RARS-DB_Radiance_BUFRTable.txt new file mode 100755 index 000000000..78ad8ea19 --- /dev/null +++ b/doc/bufr_table_samples/NCEP_NC021XXX_RARS-DB_Radiance_BUFRTable.txt @@ -0,0 +1,447 @@ +.------------------------------------------------------------------------------. +| ------------ USER DEFINITIONS FOR TABLE-A TABLE-B TABLE D -------------- | +|------------------------------------------------------------------------------| +| MNEMONIC | NUMBER | DESCRIPTION | +|----------|--------|----------------------------------------------------------| +| | | | +| NC021033 | A56033 | MTYP 021-033 RARS(EARS,AP,SA) AMSU-A 1C Tb DATA(N15-19) | +| NC021034 | A56034 | MTYP 021-034 RARS(EARS,AP,SA) AMSU-B 1C Tb DATA(N15-17) | +| NC021035 | A56035 | MTYP 021-035 RARS(EARS,AP,SA) HIRS 1C Tb DATA(N15-19) | +| NC021036 | A56036 | MTYP 021-036 RARS(EARS,AP,SA) MHS 1C Tb DATA (N18-19) | +| NC021037 | A56037 | MTYP 021-037 RARS(EARS,AP,SA) CRIS RADIANCE DATA (NPP) | +| NC021038 | A56038 | MTYP 021-038 RARS(EARS,AP,SA) ATMS Tb DATA (NPP) | +| NC021039 | A56039 | MTYP 021-039 RARS(EARS,AP,SA) IASI RADIANCE DATA (METOP | +| NC021042 | A63194 | MTYP 021-042 PROC. MSG SEVIRI ALL SKY RADIANCES (ASR) | +| NC021043 | A63195 | MTYP 021-043 PROC. MSG SEVIRI CLEAR SKY RADIANCES (CSR) | +| NC021044 | A63188 | MTYP 021-044 JMA HIMAWARI-8 CLEAR SKY RADIANCES (CSR) | +| NC021212 | A10205 | MTYP 021-212 CrIS RADIANCES (direct broadcast) | +| NC021213 | A10204 | MTYP 021-213 ATMS BRIGHT. TEMPS (direct broadcast) | +| NC021239 | A56040 | MTYP 021-239 IASI 1C RAD.(VBL CHNS)(dir. b-cst)(M) | +| | | | +| YYMMDD | 301011 | DATE -- YEAR, MONTH, DAY | +| HHMM | 301012 | TIME -- HOUR, MINUTE | +| HHMMSS | 301013 | TIME -- HOUR, MINUTE, SECOND | +| LTLONH | 301021 | HIGH ACCURACY LATITUDE/LONGITUDE POSITION | +| LOCPLAT | 304030 | LOCATION OF PLATFORM (SATELLITE) SEQUENCE | +| CLOUDCOV | 304036 | CLOUD COVERAGE | +| SIDGRSEQ | 301071 | SATELLITE IDENTIFIER/GENERATING RESOLUTION | +| SIDENSEQ | 301072 | SATELLITE IDENTIFICATION | +| ATFOV | 310011 | ATOVS FIELD OF VIEW VARIABLES | +| ATCHV | 310012 | ATOVS CHANNEL VARIABLES | +| IASIL1CB | 340002 | IASI LEVEL 1C BAND DESCRIPTION | +| IASIL1CS | 340004 | IASI LEVEL 1C AVHRR SINGLE SCENE | +| BCFQFSQ | 350200 | NPP CrIS BAND CALIBRATION & F-O-V QUALITY FLAG SEQUENCE | +| CRCHN | 350201 | NPP CrIS CHANNEL DATA | +| ATMSCH | 350206 | NPP ATMS CHANNEL DATA | +| CRCHNM | 350216 | NPP CrIS CHANNEL DATA EXTENDED | +| IASIQF | 350217 | IASI CHANNEL QUALITY FLAGS | +| IASISC | 350218 | IASI CHANNEL SCORES | +| IASIPCS | 350219 | IASI PRINCIPAL COMPONENT SCORES | +| GCRCHN | 350220 | NPP CrIS GUARD CHANNEL DATA | +| CRCHFS | 350221 | N20 CrIS FIRST ORDER STAT CHANNEL DATA | +| CRISL1C | 350222 | N20 CrIS LEVEL 1C | +| CRISN20 | 350223 | N20 CrIS DESCRIPTORS | +| RPSEQ7 | 362068 | CSR SEVIRI CLOUD & BRIGHTNESS TEMPERATURE DATA SEQUENCE | +| RPSEQ8 | 362069 | PERCENT CONFIDENCE W/MDPC REPLICATED DATA SEQUENCE | +| BID | 352001 | BULLETIN HEADER DATA | +| RCPTIM | 352003 | REPORT RECEIPT TIME DATA | +| IASICHN | 361137 | IASI LEVEL 1C SCALED RADIANCE SEQUENCE | +| AVHRCHN | 361139 | IASI LEVEL 1C MEAN AND STANDARD DEVIATION RADIANCE SEQ | +| RPSEQ10 | 362071 | ASR SEVIRI BRIGHTNESS TEMPERATURE REPLICATED SEQUENCE | +| RPSEQ11 | 362073 | CSR HIMAWARI-8 CLOUD & BRGHTNS TEMPERATURE DATA SEQUENC | +| | | | +| SAID | 001007 | SATELLITE IDENTIFIER | +| GCLONG | 001031 | ORIGINATING/GENERATING CENTER | +| GNAP | 001032 | GENERATING APPLICATION | +| OGCE | 001033 | IDENTIFICATION OF ORIGINATING/GENERATING CENTER | +| GSES | 001034 | IDENTIFICATION OF ORIGINATING/GENERATING SUB-CENTER | +| SIID | 002019 | SATELLITE INSTRUMENTS | +| SCLF | 002020 | SATELLITE CLASSIFICATION | +| SSNX | 002028 | SEGMENT SIZE AT NADIR IN X DIRECTION | +| SSNY | 002029 | SEGMENT SIZE AT NADIR IN Y DIRECTION | +| SSIN | 002048 | SATELLITE SENSOR INDICATOR | +| ANPO | 002104 | ANTENNA POLARIZATION | +| MTYP | 002141 | MEASUREMENT TYPE | +| INCN | 002150 | TOVS/ATOVS/AVHRR INSTRUMENTATION CHANNEL NUMBER | +| RAID | 002151 | RADIOMETER IDENTIFIER | +| SIDP | 002152 | SATELLITE INSTRUMENT DATA USED IN PROCESSING | +| SCCF | 002153 | SATELLITE CHANNEL CENTER FREQUENCY | +| SCBW | 002154 | SATELLITE CHANNEL BAND WIDTH | +| RDTF | 002165 | RADIANCE TYPE FLAGS | +| RDTP | 002166 | RADIANCE TYPE | +| RDCM | 002167 | RADIANCE COMPUTATIONAL METHOD | +| YEAR | 004001 | YEAR | +| MNTH | 004002 | MONTH | +| DAYS | 004003 | DAY | +| HOUR | 004004 | HOUR | +| MINU | 004005 | MINUTE | +| SECO | 004006 | SECOND | +| RCYR | 004200 | YEAR - TIME OF RECEIPT | +| RCMO | 004201 | MONTH - TIME OF RECEIPT | +| RCDY | 004202 | DAY - TIME OF RECEIPT | +| RCHR | 004203 | HOUR - TIME OF RECEIPT | +| RCMI | 004204 | MINUTE - TIME OF RECEIPT | +| CLATH | 005001 | LATITUDE (HIGH ACCURACY) | +| BEARAZ | 005021 | BEARING OR AZIMUTH | +| SOLAZI | 005022 | SOLAR AZIMUTH | +| ORBN | 005040 | ORBIT NUMBER | +| SLNM | 005041 | SCAN LINE NUMBER | +| CHNM | 005042 | CHANNEL NUMBER | +| FOVN | 005043 | FIELD OF VIEW NUMBER | +| FORN | 005045 | FIELD OF REGARD NUMBER | +| YAPCG | 005060 | Y ANGULAR POSITION FROM CENTRE OF GRAVITY | +| ZAPCG | 005061 | Z ANGULAR POSITION FROM CENTRE OF GRAVITY | +| CLONH | 006001 | LONGITUDE (HIGH ACCURACY) | +| WVNM | 006029 | WAVE NUMBER | +| SELV | 007001 | HEIGHT OF STATION | +| HMSL | 007002 | HEIGHT OR ALTITUDE | +| SAZA | 007024 | SATELLITE ZENITH ANGLE | +| SOZA | 007025 | SOLAR ZENITH ANGLE | +| VSAT | 008003 | VERTICAL SIGNIFICANCE (SATELLITE OBSERVATIONS) | +| METFET | 008011 | METEOROLOGICAL FEATURE | +| LSQL | 008012 | LAND/SEA QUALIFIER | +| FOST | 008023 | First-order statistics | +| RSST | 008029 | SURFACE TYPE | +| MDPC | 008033 | METHOD OF DEVIATION OF PERCENTAGE CONFIDENCE | +| TAPQ | 008070 | TOVS/ATOVS PRODUCT IDENTIFIER | +| STKO | 008075 | ASCENDING/DESCENDING ORBIT QUALIFIER | +| TOBD | 008076 | TYPE OF BAND | +| RCTS | 008202 | RECEIPT TIME SIGNIFICANCE | +| HOLS | 010001 | HEIGHT OF LAND SURFACE | +| HITE | 010002 | GEOPOTENTIAL HEIGHT | +| PDNP | 010031 | IN DIR. OF NORTH POLE, DISTANCE FROM THE EARTH'S CENTER | +| TMBRST | 012063 | BRIGHTNESS TEMPERATURE | +| TMINST | 012064 | INSTRUMENT TEMPERATURE | +| SDTB | 012065 | STANDARD DEVIATION BRIGHTNESS TEMPERATURE | +| TMANT | 012066 | ANTENNA TEMPERATURE | +| NEDTCO | 012158 | NOISE-EQUIV. DELTA TEMPERATURE WHILE VIEWING COLD TARGE | +| NEDTWA | 012159 | NOISE-EQUIV. DELTA TEMPERATURE WHILE VIEWING WARM TARGE | +| TMBR | 012163 | BRIGHTNESS TEMPERATURE | +| SCHRAD | 014043 | Channel radiance | +| SRAD | 014044 | CHANNEL RADIANCE | +| CHRAD | 014045 | CHANNEL RADIANCE | +| SCRA | 014046 | SCALED IASI RADIANCE | +| SMRA | 014047 | SCALED MEAN AVHRR RADIANCE | +| SSDR | 014048 | SCALED STANDARD DEVIATION AVHRR RADIANCE | +| TOCC | 020010 | CLOUD COVER (TOTAL) | +| CLTP | 020012 | CLOUD TYPE | +| HOCT | 020014 | HEIGHT OF TOP OF CLOUD | +| CLDMNT | 020081 | CLOUD AMOUNT IN SEGMENT | +| NCLDMNT | 020082 | AMOUNT SEGMENT CLOUD FREE | +| ASCS | 020083 | AMOUNT OF SEGMENT COVERED BY SCENE | +| ALFR | 021166 | LAND FRACTION | +| AVHCST | 025051 | AVHRR CHANNEL COMBINATION | +| DBID | 025062 | DATABASE IDENTIFICATION | +| MJFC | 025070 | MAJOR FRAME COUNT | +| SACV | 025075 | SATELLITE ANTENNA CORRECTIONS VERSION NUMBER | +| LOGRCW | 025076 | LOG-10 OF (TEMPERATURE-RADIANCE CENTRAL WAVENUMBER) | +| BWCC1 | 025077 | BANDWIDTH CORRECTION COEFFICIENT 1 FOR ATOVS | +| BWCC2 | 025078 | BANDWIDTH CORRECTION COEFFICIENT 2 FOR ATOVS | +| ASFI | 025079 | ALBEDO-RADIANCE SOLAR FILTERED IRRADIANCE FOR ATOVS | +| AEFW | 025080 | ALBEDO-RADIANCE EQUIVALENT FILTER WIDTH FOR ATOVS | +| FCPH | 025085 | FRACTION OF CLEAR PIXELS IN HIRS FOV | +| STCH | 025140 | START CHANNEL | +| ENCH | 025141 | END CHANNEL | +| CHSF | 025142 | CHANNEL SCALE FACTOR | +| PD00 | 027031 | IN DIRECTION OF 0 DEG E, DISTANCE FROM EARTH'S CENTER | +| PD90 | 028031 | IN DIRECTION OF 90 DEG E, DISTANCE FROM EARTH'S CENTER | +| NPPR | 030021 | NUMBER OF PIXELS PER ROW | +| NPPC | 030022 | NUMBER OF PIXELS PER COLUMN | +| QMRKH | 033003 | QUALITY INFORMATION | +| PCCF | 033007 | PERCENT CONFIDENCE | +| SLSF | 033030 | SCAN LINE STATUS FLAGS FOR ATOVS | +| SLQF | 033031 | SCAN LINE QUALITY FLAGS FOR ATOVS | +| ACQF | 033032 | CHANNEL QUALITY FLAGS FOR ATOVS | +| FOVQ | 033033 | FIELD OF VIEW QUALITY FLAGS FOR ATOVS | +| QGFQ | 033060 | GqisFlagQual - individual IASI-System quality flag | +| QGQI | 033061 | GqisQualIndex - indicator for instrument noise performa | +| QGQIL | 033062 | GqisQualIndexLoc - indicator for geometric quality inde | +| QGQIR | 033063 | GqisQualIndexRad - indicator for instrument noise perfo | +| QGQIS | 033064 | GqisQualIndexSpect - indicator for instrument noise per | +| QGSSQ | 033065 | GqisSysTecSondQual - output of system TEC (Technical Ex | +| NSQF | 033075 | SCAN LEVEL QUALITY FLAGS | +| NCQF | 033076 | CALIBRATION QUALITY FLAGS | +| NFQF | 033077 | FIELD OF VIEW QUALITY FLAGS | +| NGQI | 033078 | GEOLOCATION QUALITY | +| ATMSGQ | 033079 | GRANULE LEVEL QUALITY FLAGS | +| ATMSSQ | 033080 | SCAN LEVEL QUALITY FLAGS | +| ATMSCHQ | 033081 | CHANNEL DATA QUALITY FLAGS | +| CORN | 033215 | CORRECTED REPORT INDICATOR | +| BUHD | 035021 | BULLETIN BEING MONITORED (TTAAii) | +| BULTIM | 035022 | BULLETIN BEING MONITORED (YYGGgg) | +| BORG | 035023 | BULLETIN BEING MONITORED (CCCC) | +| BBB | 035194 | BULLETIN BEING MONITORED (BBB) | +| SEQNUM | 035195 | CHANNEL SEQUENCE NUMBER | +| RRIB | 040016 | RESIDUAL RMS IN BAND | +| NNPCS | 040017 | NON-NORMALISED PRINCIPAL COMPONENT SCORE | +| AOIM | 040018 | GIacAvgImagIIS - average of imager measurements | +| VOIM | 040019 | GIacVarImagIIS - variance of imager measurements | +| FICSI | 040021 | FRACTION OF WEIGHTED AVHRR PIXEL IN IASI FOV COVERED | +| NMBFA | 040022 | NUMBER OF MISSING, BAD OR FAILED AVHRR PIXELS | +| QFFS | 040020 | GqisFlagQualDetailed - quality flag for the system | +| SQFA | 040026 | SCORE QUANTIZATION FACTOR | +| | | | +|------------------------------------------------------------------------------| +| MNEMONIC | SEQUENCE | +|----------|-------------------------------------------------------------------| +| | | +| NC021033 | ATFOV "ATCHV"15 BID RCPTIM | +| | | +| NC021034 | ATFOV "ATCHV"5 BID RCPTIM | +| | | +| NC021035 | ATFOV "ATCHV"19 INCN ASFI AEFW ACQF CHRAD BID RCPTIM | +| | | +| NC021036 | ATFOV "ATCHV"5 BID RCPTIM | +| | | +| NC021037 | SAID OGCE SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021037 | LOCPLAT LTLONH SAZA BEARAZ SOZA SOLAZI STKO 201133 SLNM | +| NC021037 | 201000 FORN FOVN ORBN HOLS 201129 HMSL 201000 202127 | +| NC021037 | 201125 ALFR 201000 202000 LSQL TOCC HOCT RDTF NSQF | +| NC021037 | "BCFQFSQ"3 TOBD NGQI QMRKH (CRCHN) BID RCPTIM | +| | | +| NC021038 | SAID OGCE GSES SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021038 | ORBN SLNM FOVN ATMSGQ ATMSSQ NGQI LTLONH 201129 HMSL | +| NC021038 | 201000 SAZA BEARAZ SOZA SOLAZI SACV (ATMSCH) BID RCPTIM | +| | | +| NC021039 | SAID GCLONG SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021039 | LTLONH SAZA BEARAZ SOZA SOLAZI FOVN ORBN 201133 SLNM | +| NC021039 | 201000 201132 MJFC 201000 202126 SELV 202000 "IASIQF"3 | +| NC021039 | QGQI QGQIL QGQIR QGQIS QGSSQ QFFS "IASIL1CB"10 (IASICHN) | +| NC021039 | "IASISC"3 SIID AVHCST "IASIL1CS"7 CLDMNT RSST ASCS RSST | +| NC021039 | AOIM VOIM FICSI NMBFA BID RCPTIM | +| | | +| NC021042 | SIDENSEQ NPPR NPPC SAZA SOZA HITE CLOUDCOV SIDP RDCM | +| NC021042 | GNAP "RPSEQ10"11 BID RCPTIM | +| | | +| NC021043 | SIDENSEQ NPPR NPPC LSQL SAZA SOZA HITE "RPSEQ7"12 BID | +| NC021043 | RCPTIM | +| | | +| NC021044 | SIDENSEQ NPPR NPPC LSQL SAZA SOZA HITE "RPSEQ11"12 CORN | +| NC021044 | RCPTIM | +| | | +| NC021212 | SAID OGCE SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021212 | LOCPLAT LTLONH SAZA BEARAZ SOZA SOLAZI STKO 201133 SLNM | +| NC021212 | 201000 FORN FOVN ORBN HOLS 201129 HMSL 201000 202127 | +| NC021212 | 201125 ALFR 201000 202000 LSQL TOCC HOCT RDTF NSQF | +| NC021212 | "BCFQFSQ"3 TOBD NGQI QMRKH (CRCHN) (CRCHNM) BID | +| NC021212 | RCPTIM | +| | | +| NC021213 | SAID OGCE GSES SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021213 | ORBN SLNM FOVN ATMSGQ ATMSSQ NGQI LTLONH 201129 HMSL | +| NC021213 | 201000 SAZA BEARAZ SOZA SOLAZI SACV (ATMSCH) BID RCPTIM | +| | | +| NC021239 | SAID GCLONG SIID SCLF YYMMDD HHMM 207003 SECO 207000 | +| NC021239 | LTLONH SAZA BEARAZ SOZA SOLAZI FOVN ORBN 201133 SLNM | +| NC021239 | 201000 201132 MJFC 201000 202126 SELV 202000 "IASIQF"3 | +| NC021239 | QGQI QGQIL QGQIR QGQIS QGSSQ QFFS "IASIL1CB"10 (IASICHN) | +| NC021239 | "IASISC"3 SIID AVHCST "IASIL1CS"7 CLDMNT RSST ASCS RSST | +| NC021239 | AOIM VOIM FICSI NMBFA BID RCPTIM | +| | | +| YYMMDD | YEAR MNTH DAYS | +| | | +| HHMM | HOUR MINU | +| | | +| HHMMSS | HOUR MINU SECO | +| | | +| LTLONH | CLATH CLONH | +| | | +| LOCPLAT | PD00 PD90 PDNP | +| | | +| CLOUDCOV | NCLDMNT LSQL NCLDMNT LSQL CLDMNT VSAT CLDMNT VSAT CLDMNT | +| CLOUDCOV | VSAT CLDMNT VSAT | +| | | +| SIDGRSEQ | SAID GCLONG SCLF SSNX SSNY | +| | | +| SIDENSEQ | SIDGRSEQ YYMMDD HHMMSS LTLONH | +| | | +| ATFOV | TAPQ OGCE GSES TAPQ OGCE GSES SAID SSIN ORBN SACV | +| ATFOV | 201133 SLNM 201000 FOVN MJFC SLSF SLQF YEAR MNTH DAYS | +| ATFOV | HOUR MINU 202131 201138 SECO 201000 202000 CLATH CLONH | +| ATFOV | 202126 SELV 202000 SAZA BEARAZ SOZA SOLAZI FOVQ RAID | +| ATFOV | TMINST RAID TMINST RAID TMINST RAID TMINST | +| | | +| ATCHV | INCN LOGRCW BWCC1 BWCC2 ACQF 201132 202129 TMBRST 202000 | +| ATCHV | 201000 | +| | | +| IASIL1CB | STCH ENCH CHSF | +| | | +| IASIL1CS | YAPCG ZAPCG FCPH "AVHRCHN"6 | +| | | +| BCFQFSQ | TOBD WVNM WVNM STCH ENCH NCQF NFQF | +| | | +| CRCHN | 201134 CHNM 201000 SRAD | +| | | +| ATMSCH | CHNM 202131 SCCF SCBW 202000 ANPO TMANT TMBR NEDTCO | +| ATMSCH | NEDTWA ATMSCHQ | +| | | +| CRCHNM | 201134 CHNM 201000 SRAD | +| | | +| IASIQF | STCH ENCH QGFQ | +| | | +| IASISC | STCH ENCH SQFA RRIB DBID (IASIPCS) | +| | | +| IASIPCS | NNPCS | +| | | +| GCRCHN | 201134 CHNM 201000 SRAD | +| | | +| CRCHFS | CHNM FOST SCHRAD FOST SCHRAD | +| | | +| CRISL1C | YAPCG ZAPCG FCPH "CRCHFS"16 | +| | | +| CRISN20 | MTYP TOBD {GCRCHN} TOBD SIID "CRISL1C"7 | +| | | +| RPSEQ7 | SCCF SCBW CLDMNT NCLDMNT CLTP SIDP RDTP RDCM TMBRST SDTB | +| RPSEQ7 | PCCF "RPSEQ8"4 | +| | | +| RPSEQ8 | MDPC PCCF | +| | | +| BID | SEQNUM BUHD BORG BULTIM BBB | +| | | +| RCPTIM | RCTS RCYR RCMO RCDY RCHR RCMI | +| | | +| IASICHN | 201136 CHNM 201000 SCRA | +| | | +| AVHRCHN | CHNM CHSF SMRA CHSF SSDR | +| | | +| RPSEQ10 | SCCF SCBW TMBRST SDTB PCCF METFET TMBRST SDTB PCCF | +| RPSEQ10 | METFET TMBRST SDTB PCCF METFET VSAT TMBRST SDTB PCCF | +| RPSEQ10 | VSAT TMBRST SDTB PCCF VSAT TMBRST SDTB PCCF VSAT | +| | | +| RPSEQ11 | SCCF SCBW CLDMNT NCLDMNT CLTP SIDP RDTP RDCM TMBRST SDTB | +| | | +|------------------------------------------------------------------------------| +| MNEMONIC | SCAL | REFERENCE | BIT | UNITS |-------------| +|----------|------|-------------|-----|--------------------------|-------------| +| | | | | |-------------| +| SAID | 0 | 0 | 10 | CODE TABLE |-------------| +| GCLONG | 0 | 0 | 16 | CODE TABLE |-------------| +| GNAP | 0 | 0 | 8 | CODE TABLE |-------------| +| OGCE | 0 | 0 | 8 | CODE TABLE |-------------| +| GSES | 0 | 0 | 8 | CODE TABLE |-------------| +| SIID | 0 | 0 | 11 | CODE TABLE |-------------| +| SCLF | 0 | 0 | 9 | CODE TABLE |-------------| +| SSNX | 0 | 0 | 18 | M |-------------| +| SSNY | 0 | 0 | 18 | M |-------------| +| SSIN | 0 | 0 | 4 | CODE TABLE |-------------| +| ANPO | 0 | 0 | 4 | CODE TABLE |-------------| +| MTYP | 0 | 0 | 24 | CCITT IA5 |-------------| +| INCN | 0 | 0 | 6 | CODE TABLE |-------------| +| RAID | 0 | 0 | 11 | CODE TABLE |-------------| +| SIDP | 0 | 0 | 31 | FLAG TABLE |-------------| +| SCCF | -8 | 0 | 26 | HZ |-------------| +| SCBW | -8 | 0 | 26 | HZ |-------------| +| RDTF | 0 | 0 | 15 | FLAG TABLE |-------------| +| RDTP | 0 | 0 | 4 | CODE TABLE |-------------| +| RDCM | 0 | 0 | 4 | CODE TABLE |-------------| +| YEAR | 0 | 0 | 12 | YEAR |-------------| +| MNTH | 0 | 0 | 4 | MONTH |-------------| +| DAYS | 0 | 0 | 6 | DAY |-------------| +| HOUR | 0 | 0 | 5 | HOUR |-------------| +| MINU | 0 | 0 | 6 | MINUTE |-------------| +| SECO | 0 | 0 | 6 | SECOND |-------------| +| RCYR | 0 | 0 | 12 | YEAR |-------------| +| RCMO | 0 | 0 | 4 | MONTH |-------------| +| RCDY | 0 | 0 | 6 | DAY |-------------| +| RCHR | 0 | 0 | 5 | HOUR |-------------| +| RCMI | 0 | 0 | 6 | MINUTE |-------------| +| CLATH | 5 | -9000000 | 25 | DEGREE |-------------| +| BEARAZ | 2 | 0 | 16 | DEGREE TRUE |-------------| +| SOLAZI | 2 | 0 | 16 | DEGREE TRUE |-------------| +| ORBN | 0 | 0 | 24 | NUMERIC |-------------| +| SLNM | 0 | 0 | 8 | NUMERIC |-------------| +| CHNM | 0 | 0 | 6 | NUMERIC |-------------| +| FOVN | 0 | 0 | 8 | NUMERIC |-------------| +| FORN | 0 | 0 | 8 | NUMERIC |-------------| +| YAPCG | 6 | -8000000 | 24 | DEGREE |-------------| +| ZAPCG | 6 | -8000000 | 24 | DEGREE |-------------| +| CLONH | 5 | -18000000 | 26 | DEGREE |-------------| +| WVNM | 1 | 0 | 22 | M**-1 |-------------| +| SELV | 0 | -400 | 15 | M |-------------| +| HMSL | -1 | -40 | 16 | M |-------------| +| SAZA | 2 | -9000 | 15 | DEGREE |-------------| +| SOZA | 2 | -9000 | 15 | DEGREE |-------------| +| VSAT | 0 | 0 | 6 | CODE TABLE |-------------| +| METFET | 0 | 0 | 6 | CODE TABLE |-------------| +| LSQL | 0 | 0 | 2 | CODE TABLE |-------------| +| FOST | 0 | 0 | 6 | CODE TABLE |-------------| +| RSST | 0 | 0 | 8 | CODE TABLE |-------------| +| MDPC | 0 | 0 | 7 | CODE TABLE |-------------| +| TAPQ | 0 | 0 | 4 | CODE TABLE |-------------| +| STKO | 0 | 0 | 2 | CODE TABLE |-------------| +| TOBD | 0 | 0 | 6 | CODE TABLE |-------------| +| RCTS | 0 | 0 | 6 | CODE TABLE |-------------| +| HOLS | 0 | -400 | 15 | M |-------------| +| HITE | -1 | -40 | 16 | M |-------------| +| PDNP | 2 | -1073741824 | 31 | M |-------------| +| TMBRST | 1 | 0 | 12 | KELVIN |-------------| +| TMINST | 1 | 0 | 12 | KELVIN |-------------| +| SDTB | 1 | 0 | 12 | KELVIN |-------------| +| TMANT | 2 | 0 | 16 | K |-------------| +| NEDTCO | 2 | 0 | 12 | K |-------------| +| NEDTWA | 2 | 0 | 12 | K |-------------| +| TMBR | 2 | 0 | 16 | K |-------------| +| SCHRAD | 4 | 0 | 23 | W M**-2 SR**-1 UM**-1 |-------------| +| SRAD | 7 | -100000 | 22 | W M**-2 SR**-1 CM |-------------| +| CHRAD | 0 | 0 | 11 | W M**-2 SR**-1 CM |-------------| +| SCRA | 0 | -5000 | 16 | W M**-2 SR**-1 M |-------------| +| SMRA | 0 | 0 | 31 | W M**-2 SR**-1 M |-------------| +| SSDR | 0 | 0 | 31 | W M**-2 SR**-1 M |-------------| +| TOCC | 0 | 0 | 7 | % |-------------| +| CLTP | 0 | 0 | 6 | CODE TABLE |-------------| +| HOCT | -1 | -40 | 11 | M |-------------| +| CLDMNT | 0 | 0 | 7 | % |-------------| +| NCLDMNT | 0 | 0 | 7 | % |-------------| +| ASCS | 0 | 0 | 7 | % |-------------| +| ALFR | 3 | 0 | 10 | NUMERIC |-------------| +| AVHCST | 0 | 0 | 7 | FLAG TABLE |-------------| +| DBID | 0 | 0 | 14 | NUMERIC |-------------| +| MJFC | 0 | 0 | 4 | NUMERIC |-------------| +| SACV | 0 | 0 | 5 | NUMERIC |-------------| +| LOGRCW | 8 | 0 | 30 | LOG/M |-------------| +| BWCC1 | 5 | -100000 | 18 | NUMERIC |-------------| +| BWCC2 | 5 | 0 | 17 | NUMERIC |-------------| +| ASFI | 4 | 0 | 24 | W M**-2 |-------------| +| AEFW | 10 | 0 | 14 | M |-------------| +| FCPH | 0 | 0 | 7 | NUMERIC |-------------| +| STCH | 0 | 0 | 14 | NUMERIC |-------------| +| ENCH | 0 | 0 | 14 | NUMERIC |-------------| +| CHSF | 0 | 0 | 6 | NUMERIC |-------------| +| PD00 | 2 | -1073741824 | 31 | M |-------------| +| PD90 | 2 | -1073741824 | 31 | M |-------------| +| NPPR | 0 | 0 | 12 | NUMERIC |-------------| +| NPPC | 0 | 0 | 12 | NUMERIC |-------------| +| QMRKH | 0 | 0 | 3 | CODE TABLE |-------------| +| PCCF | 0 | 0 | 7 | PERCENT |-------------| +| SLSF | 0 | 0 | 24 | FLAG TABLE |-------------| +| SLQF | 0 | 0 | 24 | FLAG TABLE |-------------| +| ACQF | 0 | 0 | 24 | FLAG TABLE |-------------| +| FOVQ | 0 | 0 | 24 | FLAG TABLE |-------------| +| QGFQ | 0 | 0 | 2 | CODE TABLE |-------------| +| QGQI | 0 | 0 | 7 | % |-------------| +| QGQIL | 0 | 0 | 7 | % |-------------| +| QGQIR | 0 | 0 | 7 | % |-------------| +| QGQIS | 0 | 0 | 7 | % |-------------| +| QGSSQ | 0 | 0 | 24 | NUMERIC |-------------| +| NSQF | 0 | 0 | 13 | FLAG TABLE |-------------| +| NCQF | 0 | 0 | 9 | FLAG TABLE |-------------| +| NFQF | 0 | 0 | 19 | FLAG TABLE |-------------| +| NGQI | 0 | 0 | 4 | CODE TABLE |-------------| +| ATMSGQ | 0 | 0 | 16 | FLAG TABLE |-------------| +| ATMSSQ | 0 | 0 | 20 | FLAG TABLE |-------------| +| ATMSCHQ | 0 | 0 | 12 | FLAG TABLE |-------------| +| CORN | 0 | 0 | 3 | CODE TABLE |-------------| +| BUHD | 0 | 0 | 48 | CCITT IA5 |-------------| +| BULTIM | 0 | 0 | 48 | CCITT IA5 |-------------| +| BORG | 0 | 0 | 32 | CCITT IA5 |-------------| +| BBB | 0 | 0 | 48 | CCITT IA5 |-------------| +| SEQNUM | 0 | 0 | 32 | CCITT IA5 |-------------| +| RRIB | 3 | 0 | 14 | NUMERIC |-------------| +| NNPCS | 0 | -1073741824 | 31 | NUMERIC |-------------| +| AOIM | 6 | 0 | 24 | W M**-2 SR**-1 M |-------------| +| VOIM | 6 | 0 | 24 | W M**-2 SR**-1 M |-------------| +| FICSI | 0 | 0 | 7 | % |-------------| +| NMBFA | 0 | 0 | 7 | NUMERIC |-------------| +| QFFS | 0 | 0 | 17 | FLAG TABLE |-------------| +| SQFA | 2 | 0 | 16 | NUMERIC |-------------| +| | | | | |-------------| +`------------------------------------------------------------------------------' diff --git a/share/raob_stations.json b/share/raob_stations.json new file mode 100644 index 000000000..3ad1f3ff1 --- /dev/null +++ b/share/raob_stations.json @@ -0,0 +1,2374 @@ +{ +"01001": {"id": "ENJA", "name": "JAN MAYEN", "state": "NO", "lat": 70.930, "lon": -8.670, "elev": 9}, +"01004": {"id": "ENAS", "name": "NY-ALESUND II", "state": "NO", "lat": 78.920, "lon": 11.930, "elev": 8}, +"01007": {"id": "", "name": "NY-ALESUND", "state": "NO", "lat": 78.920, "lon": 11.930, "elev": 18}, +"01008": {"id": "ENSB", "name": "SVALBARD/LONGYE", "state": "NO", "lat": 78.250, "lon": 15.470, "elev": 29}, +"01010": {"id": "ENAN", "name": "ANDOYA", "state": "NO", "lat": 69.300, "lon": 16.150, "elev": 14}, +"01011": {"id": "", "name": "KVITOYA", "state": "NO", "lat": 80.100, "lon": 31.400, "elev": 16}, +"01025": {"id": "ENTC", "name": "TROMSOE", "state": "NO", "lat": 69.680, "lon": 18.920, "elev": 10}, +"01028": {"id": "ENBJ", "name": "BJORNOYA ISLAND", "state": "NO", "lat": 74.520, "lon": 19.020, "elev": 14}, +"01088": {"id": "ENVD", "name": "VADSO", "state": "NO", "lat": 70.070, "lon": 29.850, "elev": 39}, +"01152": {"id": "ENBO", "name": "BODO VI (CIV/MIL)", "state": "NO", "lat": 67.250, "lon": 14.370, "elev": 13}, +"01240": {"id": "", "name": "HALTEN", "state": "NO", "lat": 64.170, "lon": 9.420, "elev": 16}, +"01241": {"id": "ENOL", "name": "ORLAND III (NOR-AFB)", "state": "NO", "lat": 63.700, "lon": 9.600, "elev": 7}, +"01384": {"id": "ENGM", "name": "OSLO/GARDERMOEN", "state": "NO", "lat": 60.200, "lon": 11.080, "elev": 204}, +"01400": {"id": "ENEK", "name": "EKOFISK", "state": "NO", "lat": 56.530, "lon": 3.220, "elev": 46}, +"01415": {"id": "ENZV", "name": "STAVANGER/SOLA (AFB)", "state": "NO", "lat": 58.880, "lon": 5.630, "elev": 9}, +"01465": {"id": "", "name": "TORUNGEN", "state": "NO", "lat": 58.400, "lon": 8.800, "elev": 15}, +"01492": {"id": "", "name": "OSLO/BLINDERN", "state": "NO", "lat": 59.570, "lon": 10.430, "elev": 96}, +"02120": {"id": "", "name": "KVIKKJOKK", "state": "SW", "lat": 66.950, "lon": 17.750, "elev": 337}, +"02185": {"id": "ESPA", "name": "LULEA/KALLAX (AFB)", "state": "SW", "lat": 65.550, "lon": 22.130, "elev": 34}, +"02188": {"id": "", "name": "RODKALLEN", "state": "SW", "lat": 65.320, "lon": 22.380, "elev": 1}, +"02225": {"id": "", "name": "OSTERSUND/FROSON", "state": "SW", "lat": 63.180, "lon": 14.500, "elev": 370}, +"02365": {"id": "ESNN", "name": "SUNDSVALL/HARNOSAND", "state": "SW", "lat": 62.530, "lon": 17.470, "elev": 6}, +"02527": {"id": "ESGG", "name": "GOTEBORG/LANDVETTER", "state": "SW", "lat": 57.670, "lon": 12.500, "elev": 164}, +"02591": {"id": "ESQV", "name": "VISBY AERO STATION", "state": "SW", "lat": 57.650, "lon": 18.350, "elev": 47}, +"02836": {"id": "EFSO", "name": "SODANKYLA", "state": "FI", "lat": 67.370, "lon": 26.650, "elev": 179}, +"02935": {"id": "EFJY", "name": "JYVASKYLA (MIL/CIV)", "state": "FI", "lat": 62.400, "lon": 25.680, "elev": 145}, +"02963": {"id": "", "name": "JOKIOINEN", "state": "FI", "lat": 60.820, "lon": 23.500, "elev": 103}, +"02982": {"id": "", "name": "RUSSARO", "state": "FI", "lat": 59.770, "lon": 22.950, "elev": 11}, +"03001": {"id": "", "name": "MUCKLE FLUGGA (LH)", "state": "UK", "lat": 60.850, "lon": -0.880, "elev": 53}, +"03005": {"id": "", "name": "LERWICK/SHETLAND IS", "state": "UK", "lat": 60.130, "lon": -1.180, "elev": 84}, +"03008": {"id": "", "name": "FAIR ISLE", "state": "UK", "lat": 59.530, "lon": -1.630, "elev": 59}, +"03023": {"id": "", "name": "WEST GEIRINISH", "state": "UK", "lat": 57.350, "lon": -7.370, "elev": 3}, +"03026": {"id": "EGPO", "name": "STORNOWAY", "state": "UK", "lat": 58.220, "lon": -6.320, "elev": 13}, +"03066": {"id": "EGQK", "name": "KINLOSS RAF", "state": "UK", "lat": 57.650, "lon": -3.570, "elev": 7}, +"03085": {"id": "", "name": "INCHMARLO", "state": "UK", "lat": 57.070, "lon": -2.530, "elev": 80}, +"03091": {"id": "EGPD", "name": "ABERDEEN/DYCE", "state": "UK", "lat": 57.200, "lon": -2.220, "elev": 65}, +"03121": {"id": "", "name": "KILDONAN", "state": "UK", "lat": 55.430, "lon": -5.100, "elev": 18}, +"03130": {"id": "EGOY", "name": "WEST FREUGH", "state": "UK", "lat": 54.850, "lon": -4.950, "elev": 16}, +"03170": {"id": "", "name": "SHANWELL", "state": "UK", "lat": 56.430, "lon": -2.870, "elev": 5}, +"03171": {"id": "EGQL", "name": "LEUCHARS RAF", "state": "UK", "lat": 56.380, "lon": -2.870, "elev": 12}, +"03213": {"id": "", "name": "ESKMEALS", "state": "UK", "lat": 54.320, "lon": -3.400, "elev": 9}, +"03238": {"id": "", "name": "ALBERMARLE", "state": "UK", "lat": 55.100, "lon": -1.930, "elev": 146}, +"03240": {"id": "EGQM", "name": "BOULMER", "state": "UK", "lat": 55.420, "lon": -1.600, "elev": 23}, +"03241": {"id": "", "name": "LONG FRAMLINGTO", "state": "UK", "lat": 55.300, "lon": -1.800, "elev": 158}, +"03257": {"id": "EGXE", "name": "LEEMING RAF", "state": "UK", "lat": 54.300, "lon": -1.530, "elev": 40}, +"03322": {"id": "", "name": "AUGHTON", "state": "UK", "lat": 53.550, "lon": -2.920, "elev": 56}, +"03334": {"id": "EGCC", "name": "MANCHESTER INTL", "state": "UK", "lat": 53.340, "lon": -2.280, "elev": 78}, +"03345": {"id": "", "name": "EMLEY MOOR", "state": "UK", "lat": 53.620, "lon": -1.670, "elev": 259}, +"03346": {"id": "EGNM", "name": "LEEDS AND BRADFORD", "state": "UK", "lat": 53.860, "lon": -1.650, "elev": 208}, +"03354": {"id": "", "name": "NOTTINGHAM WX CTR", "state": "UK", "lat": 53.000, "lon": -1.250, "elev": 117}, +"03377": {"id": "EGXW", "name": "WADDINGTON RAF", "state": "UK", "lat": 53.170, "lon": -0.520, "elev": 70}, +"03414": {"id": "EGOS", "name": "SHAWBURY RAF", "state": "UK", "lat": 52.800, "lon": -2.670, "elev": 76}, +"03496": {"id": "", "name": "HEMSBY", "state": "UK", "lat": 52.680, "lon": 1.680, "elev": 14}, +"03501": {"id": "", "name": "CAPE DEWI", "state": "UK", "lat": 55.420, "lon": -4.000, "elev": 92}, +"03502": {"id": "EGUC", "name": "ABERPORTH", "state": "UK", "lat": 52.130, "lon": -4.570, "elev": 134}, +"03559": {"id": "", "name": "CARDINGTON", "state": "UK", "lat": 52.100, "lon": -0.420, "elev": 30}, +"03590": {"id": "EGUW", "name": "WATTISHAM RAF", "state": "UK", "lat": 52.120, "lon": 0.970, "elev": 87}, +"03614": {"id": "", "name": "CILFYNYDD", "state": "UK", "lat": 51.630, "lon": -3.300, "elev": 194}, +"03649": {"id": "EGVN", "name": "BRIZE NORTON RAF", "state": "UK", "lat": 51.750, "lon": -1.580, "elev": 88}, +"03693": {"id": "", "name": "SHOEBURYNESS", "state": "UK", "lat": 51.550, "lon": 0.850, "elev": 3}, +"03715": {"id": "EGFF", "name": "CARDIFF-WALES", "state": "UK", "lat": 51.400, "lon": -3.350, "elev": 67}, +"03743": {"id": "", "name": "LARKHILL", "state": "UK", "lat": 51.200, "lon": -1.800, "elev": 133}, +"03774": {"id": "", "name": "CRAWLEY", "state": "UK", "lat": 51.080, "lon": -0.220, "elev": 144}, +"03808": {"id": "", "name": "CAMBORNE", "state": "UK", "lat": 50.220, "lon": -5.320, "elev": 88}, +"03810": {"id": "", "name": "PENDENNIS POINT", "state": "UK", "lat": 50.150, "lon": -5.070, "elev": 35}, +"03815": {"id": "", "name": "LIZARD LIGHTHOU", "state": "UK", "lat": 49.970, "lon": -5.200, "elev": 60}, +"03872": {"id": "", "name": "THORNEY ISLAND", "state": "UK", "lat": 50.820, "lon": -0.920, "elev": 4}, +"03882": {"id": "", "name": "HERSTOMONCEUX WEST", "state": "UK", "lat": 50.900, "lon": 0.320, "elev": 54}, +"03894": {"id": "EGJB", "name": "GUERNSEY AIRPORT", "state": "UK", "lat": 49.430, "lon": -2.600, "elev": 102}, +"03918": {"id": "", "name": "CASTOR BAY", "state": "UK", "lat": 54.500, "lon": -6.330, "elev": 17}, +"03920": {"id": "", "name": "HILLSBOROUGH", "state": "UK", "lat": 54.800, "lon": -6.170, "elev": 38}, +"03953": {"id": "", "name": "VALENTIA OBSERVATORY", "state": "IE", "lat": 51.930, "lon": -10.250, "elev": 14}, +"04001": {"id": "", "name": "HVALLATUR", "state": "IL", "lat": 65.530, "lon": -24.470, "elev": 17}, +"04018": {"id": "BIKF", "name": "KEFLAVIK", "state": "IL", "lat": 63.970, "lon": -22.600, "elev": 52}, +"04089": {"id": "BIEG", "name": "EGILSSTADIR", "state": "IL", "lat": 65.170, "lon": -14.400, "elev": 23}, +"04168": {"id": "", "name": "SANDBUDIR", "state": "GL", "lat": 64.930, "lon": -17.980, "elev": 821}, +"04202": {"id": "BGTL", "name": "THULE AB", "state": "GL", "lat": 76.530, "lon": -68.750, "elev": 77}, +"04220": {"id": "BGEM", "name": "EGEDESMINDE/AUSIAT", "state": "GL", "lat": 68.700, "lon": -52.850, "elev": 41}, +"04230": {"id": "BGHB", "name": "HOLSTEINSBORG", "state": "GL", "lat": 66.920, "lon": -53.670, "elev": 9}, +"04231": {"id": "BGSF", "name": "SONDRESTROM", "state": "GL", "lat": 67.000, "lon": -50.800, "elev": 46}, +"04255": {"id": "BMKA", "name": "MARRAK POINT", "state": "GL", "lat": 63.430, "lon": -51.180, "elev": 41}, +"04270": {"id": "BGBW", "name": "NARSARSSUAK", "state": "GL", "lat": 61.180, "lon": -45.420, "elev": 26}, +"04310": {"id": "", "name": "NORD", "state": "GL", "lat": 81.600, "lon": -16.700, "elev": 40}, +"04320": {"id": "BGDH", "name": "DANMARKSHAVN (PORT)", "state": "GL", "lat": 76.770, "lon": -18.670, "elev": 12}, +"04330": {"id": "BGDB", "name": "DANEBORG (AUT)", "state": "GL", "lat": 74.300, "lon": -20.220, "elev": 44}, +"04339": {"id": "BGSC", "name": "SCORESBYSUND", "state": "GL", "lat": 70.480, "lon": -21.950, "elev": 69}, +"04342": {"id": "", "name": "WALRUS BAY", "state": "GL", "lat": 70.500, "lon": -21.970, "elev": 7}, +"04353": {"id": "BIKA", "name": "IKATEQ", "state": "GL", "lat": 65.930, "lon": -36.680, "elev": 56}, +"04360": {"id": "BGAM", "name": "ANGMAGSSALIK", "state": "GL", "lat": 65.600, "lon": -37.630, "elev": 52}, +"04417": {"id": " ", "name": "GEOSUMMIT", "state": "GL", "lat": 72.580, "lon": -38.460, "elev": 3255}, +"06010": {"id": "EKVG", "name": "SOERVAAG/VAGAR", "state": "FA", "lat": 62.070, "lon": -7.270, "elev": 85}, +"06011": {"id": "", "name": "THORSHAVN (PORT)", "state": "FA", "lat": 62.020, "lon": -6.770, "elev": 39}, +"06030": {"id": "EKYT", "name": "AALBORG", "state": "DN", "lat": 57.100, "lon": 9.870, "elev": 3}, +"06060": {"id": "", "name": "KARUP", "state": "DN", "lat": 56.300, "lon": 9.120, "elev": 53}, +"06080": {"id": "EKEB", "name": "ESBJERG", "state": "DN", "lat": 55.520, "lon": 8.550, "elev": 29}, +"06173": {"id": "", "name": "STEVNS", "state": "DN", "lat": 55.280, "lon": 31.400, "elev": 40}, +"06181": {"id": "", "name": "KOEBENHAVN/JAEGERSBORG", "state": "DN", "lat": 55.770, "lon": 12.530, "elev": 40}, +"06210": {"id": "EHVB", "name": "VALKENBURG", "state": "NL", "lat": 52.180, "lon": 4.420, "elev": 2}, +"06235": {"id": "EHKD", "name": "DE KOOY", "state": "NL", "lat": 52.920, "lon": 4.780, "elev": 2}, +"06242": {"id": "", "name": "VLIELAND", "state": "NL", "lat": 53.250, "lon": 4.920, "elev": 5}, +"06260": {"id": "EHDB", "name": "DE BILT", "state": "NL", "lat": 52.100, "lon": 5.180, "elev": 4}, +"06320": {"id": "", "name": "LE GOEREE", "state": "NL", "lat": 51.930, "lon": 3.670, "elev": 19}, +"06400": {"id": "EBFN", "name": "KOKSIJDE (BEL-AFB)", "state": "BX", "lat": 51.080, "lon": 2.650, "elev": 9}, +"06447": {"id": "", "name": "UCCLE/UKKLE", "state": "BX", "lat": 50.800, "lon": 4.350, "elev": 104}, +"06458": {"id": "EBBE", "name": "BEAUVECHAIN", "state": "BX", "lat": 50.750, "lon": 4.770, "elev": 127}, +"06463": {"id": "EBTN", "name": "GOETSENHOVEN (MIL)", "state": "BX", "lat": 50.780, "lon": 4.950, "elev": 81}, +"06476": {"id": "ESBU", "name": "SAINT HUBERT (MIL)", "state": "BX", "lat": 50.030, "lon": 5.400, "elev": 557}, +"06496": {"id": "EBLB", "name": "ELSENBORN (MIL)", "state": "BX", "lat": 50.470, "lon": 6.180, "elev": 570}, +"06610": {"id": "LSMP", "name": "PAYERNE (MIL/AUT)", "state": "SW", "lat": 46.820, "lon": 6.950, "elev": 491}, +"06680": {"id": "", "name": "SAENTIS (AUT)", "state": "SW", "lat": 47.250, "lon": 9.350, "elev": 2500}, +"07010": {"id": "", "name": "DUNKERQUE", "state": "FR", "lat": 51.050, "lon": 2.330, "elev": 23}, +"07100": {"id": "", "name": "OUESSANT ISLAND", "state": "FR", "lat": 48.470, "lon": -5.130, "elev": 32}, +"07107": {"id": "", "name": "BRIGNOGAN", "state": "FR", "lat": 48.680, "lon": -4.330, "elev": 22}, +"07110": {"id": "LFRB", "name": "BREST/GUIPAVAS", "state": "FR", "lat": 48.450, "lon": -4.420, "elev": 103}, +"07130": {"id": "LFRN", "name": "RENNES/ST. JACQUES", "state": "FR", "lat": 48.070, "lon": -1.730, "elev": 37}, +"07139": {"id": "", "name": "ALENCON", "state": "FR", "lat": 48.450, "lon": 0.100, "elev": 141}, +"07145": {"id": "", "name": "TRAPPES (AUT)", "state": "FR", "lat": 48.770, "lon": 2.020, "elev": 168}, +"07149": {"id": "LFPO", "name": "PARIS/ORLY", "state": "FR", "lat": 48.730, "lon": 2.400, "elev": 96}, +"07180": {"id": "LFSN", "name": "NANCY/ESSEY", "state": "FR", "lat": 48.630, "lon": 6.220, "elev": 217}, +"07210": {"id": "", "name": "VANNES (AUT)", "state": "FR", "lat": 47.630, "lon": -2.750, "elev": 11}, +"07240": {"id": "LFOT", "name": "TOURS/ST. SYMPHO", "state": "FR", "lat": 47.450, "lon": 0.720, "elev": 112}, +"07255": {"id": "", "name": "BOURGES", "state": "FR", "lat": 47.070, "lon": 2.370, "elev": 162}, +"07280": {"id": "LFSD", "name": "DIJON/LONGVIC", "state": "FR", "lat": 47.270, "lon": -5.080, "elev": 227}, +"07412": {"id": "", "name": "CONIAC", "state": "FR", "lat": 45.670, "lon": -0.320, "elev": 30}, +"07480": {"id": "LFLY", "name": "LYON/BRON", "state": "FR", "lat": 45.720, "lon": 4.950, "elev": 201}, +"07481": {"id": "LFLL", "name": "LYON/SATOLAS", "state": "FR", "lat": 45.730, "lon": 5.080, "elev": 240}, +"07500": {"id": "", "name": "CAPE FERRET", "state": "FR", "lat": 44.630, "lon": -1.250, "elev": 10}, +"07510": {"id": "LFBD", "name": "BORDEAUX/MERIGNAC", "state": "FR", "lat": 44.830, "lon": -0.700, "elev": 61}, +"07579": {"id": "LFMO", "name": "ORANGE/CARITAT", "state": "FR", "lat": 44.130, "lon": 4.830, "elev": 55}, +"07645": {"id": "LFME", "name": "NIMES/COURBESSAC", "state": "FR", "lat": 43.870, "lon": 4.400, "elev": 62}, +"07690": {"id": "LFMN", "name": "NICE/COTE D'AZUR", "state": "FR", "lat": 43.650, "lon": 7.200, "elev": 10}, +"07749": {"id": "", "name": "CAPE BEAR", "state": "FR", "lat": 42.520, "lon": 3.130, "elev": 86}, +"07761": {"id": "LFKJ", "name": "AJACCIO/CAMPO ORO", "state": "FR", "lat": 41.920, "lon": 8.800, "elev": 9}, +"08001": {"id": "", "name": "LA CORUNA CITY", "state": "SP", "lat": 43.370, "lon": -8.420, "elev": 67}, +"08002": {"id": "LECO", "name": "LA CORUNA/ALVED", "state": "SP", "lat": 43.300, "lon": -8.380, "elev": 103}, +"08023": {"id": "", "name": "SANTANDER CITY", "state": "SP", "lat": 43.470, "lon": -3.820, "elev": 65}, +"08029": {"id": "LESO", "name": "SAN SEBASTIAN", "state": "SP", "lat": 43.350, "lon": -1.800, "elev": 8}, +"08094": {"id": "", "name": "HUESCA/MONFLORITE", "state": "SP", "lat": 42.080, "lon": -0.330, "elev": 554}, +"08140": {"id": "LEVD", "name": "VALLADOLID (MIL/CIV)", "state": "SP", "lat": 41.720, "lon": -4.850, "elev": 854}, +"08160": {"id": "LEZG", "name": "ZARAGOZA (MIL/CIV)", "state": "SP", "lat": 41.670, "lon": -1.020, "elev": 258}, +"08190": {"id": "LBCN", "name": "BARCELONA", "state": "SP", "lat": 41.620, "lon": 2.200, "elev": 98}, +"08221": {"id": "LEMD", "name": "MADRID/BARAJAS", "state": "SP", "lat": 40.450, "lon": -3.550, "elev": 582}, +"08222": {"id": "", "name": "MADRID", "state": "SP", "lat": 40.420, "lon": -3.680, "elev": 655}, +"08301": {"id": "", "name": "PALMA DE MALLORCA", "state": "SP", "lat": 39.550, "lon": 2.620, "elev": 6}, +"08302": {"id": "", "name": "PALMA/SON BONET", "state": "SP", "lat": 39.600, "lon": 2.700, "elev": 41}, +"08383": {"id": "", "name": "HUELVA", "state": "SP", "lat": 37.280, "lon": -6.910, "elev": 20}, +"08430": {"id": "", "name": "MURCIA CITY", "state": "SP", "lat": 38.000, "lon": -1.170, "elev": 62}, +"08431": {"id": "", "name": "CASTILLO GALERA", "state": "SP", "lat": 37.580, "lon": -1.000, "elev": 217}, +"08449": {"id": "LERT", "name": "ROTA NAS", "state": "SP", "lat": 36.650, "lon": -6.350, "elev": 27}, +"08495": {"id": "LXGB", "name": "GIBRALTAR (CIV/MIL)", "state": "GI", "lat": 36.150, "lon": -5.350, "elev": 5}, +"08506": {"id": "", "name": "HORTA", "state": "PO", "lat": 38.520, "lon": -28.630, "elev": 62}, +"08507": {"id": "", "name": "GRACIOSA/AERODROME", "state": "PO", "lat": 39.100, "lon": -28.030, "elev": 28}, +"08508": {"id": "", "name": "LAJES/SANTA RITA", "state": "PO", "lat": 38.750, "lon": -27.080, "elev": 113}, +"08509": {"id": "LPLA", "name": "LAJES AB", "state": "PO", "lat": 38.770, "lon": -27.100, "elev": 55}, +"08521": {"id": "LPFU", "name": "FUNCHAL/MADEIRA", "state": "PO", "lat": 32.680, "lon": -16.770, "elev": 55}, +"08522": {"id": "", "name": "FUNCHAL", "state": "PO", "lat": 32.630, "lon": -16.900, "elev": 56}, +"08549": {"id": "", "name": "COIMBRA", "state": "PO", "lat": 40.200, "lon": -8.420, "elev": 140}, +"08554": {"id": "", "name": "FARO", "state": "PO", "lat": 37.020, "lon": -7.970, "elev": 7}, +"08579": {"id": "", "name": "LISBON/GAGO COUTINHO", "state": "PO", "lat": 38.770, "lon": -9.130, "elev": 105}, +"08589": {"id": "", "name": "PRAIA", "state": "CV", "lat": 14.900, "lon": -23.520, "elev": 35}, +"08594": {"id": "GVAC", "name": "SAL ISL/AMILCAR CAB", "state": "CV", "lat": 16.730, "lon": -22.950, "elev": 55}, +"10035": {"id": "", "name": "SCHLESWIG", "state": "DL", "lat": 54.530, "lon": 9.550, "elev": 48}, +"10044": {"id": "", "name": "KIEL", "state": "DL", "lat": 54.500, "lon": 10.280, "elev": 23}, +"10046": {"id": "", "name": "KIEL/HOLTENAU", "state": "DL", "lat": 54.380, "lon": 10.150, "elev": 32}, +"10113": {"id": "", "name": "NORDERNEY", "state": "DL", "lat": 53.710, "lon": 7.150, "elev": 11}, +"10168": {"id": "", "name": "GOLDBERT (AUT)", "state": "DL", "lat": 53.600, "lon": 12.100, "elev": 58}, +"10184": {"id": "", "name": "GREIFSWALD", "state": "DL", "lat": 54.100, "lon": 13.400, "elev": 6}, +"10200": {"id": "EDWE", "name": "EMDEN/KOENIGSPOLDER", "state": "DL", "lat": 53.380, "lon": 7.230, "elev": 5}, +"10238": {"id": "ETGB", "name": "BERGEN (MIL)", "state": "DL", "lat": 52.820, "lon": 9.930, "elev": 69}, +"10272": {"id": "ETGW", "name": "WITTSTOCK", "state": "DL", "lat": 53.200, "lon": 12.520, "elev": 74}, +"10304": {"id": "ETWM", "name": "MEPPEN (MIL)", "state": "DL", "lat": 52.620, "lon": 7.320, "elev": 26}, +"10307": {"id": "", "name": "RHEINE", "state": "DL", "lat": 52.270, "lon": 7.430, "elev": 78}, +"10337": {"id": "", "name": "HILDESHEIM", "state": "DL", "lat": 52.180, "lon": 9.950, "elev": 88}, +"10338": {"id": "EDDV", "name": "HANNOVER", "state": "DL", "lat": 52.470, "lon": 9.700, "elev": 54}, +"10384": {"id": "EDDI", "name": "BERLIN/TEMPELHOF", "state": "DL", "lat": 52.470, "lon": 13.400, "elev": 49}, +"10393": {"id": "", "name": "LINDENBERG", "state": "DL", "lat": 52.220, "lon": 14.120, "elev": 112}, +"10410": {"id": "EDZE", "name": "ESSEN/MULHEIM", "state": "DL", "lat": 51.400, "lon": 6.970, "elev": 161}, +"10418": {"id": "", "name": "LUEDENSCHEID", "state": "DL", "lat": 51.250, "lon": 7.650, "elev": 392}, +"10437": {"id": "", "name": "FRITZLAR/KASSEL", "state": "DL", "lat": 51.130, "lon": 9.300, "elev": 223}, +"10454": {"id": "", "name": "WERNIGERODE", "state": "DL", "lat": 51.850, "lon": 10.770, "elev": 240}, +"10468": {"id": "", "name": "OPPIN", "state": "DL", "lat": 51.330, "lon": 12.040, "elev": 106}, +"10486": {"id": "", "name": "WAHNSDORF", "state": "DL", "lat": 51.120, "lon": 13.680, "elev": 232}, +"10513": {"id": "", "name": "KOLN/BONN", "state": "DL", "lat": 50.870, "lon": 7.130, "elev": 91}, +"10548": {"id": "", "name": "MEININGEN", "state": "DL", "lat": 50.570, "lon": 10.370, "elev": 453}, +"10577": {"id": "", "name": "CHEMNITZ", "state": "DL", "lat": 50.800, "lon": 12.870, "elev": 419}, +"10582": {"id": "", "name": "ZINNWALD/GEORGE", "state": "DL", "lat": 50.730, "lon": 13.750, "elev": 882}, +"10618": {"id": "ETGI", "name": "IDAR-OBERSTEIN (MIL)", "state": "DL", "lat": 49.700, "lon": 7.330, "elev": 377}, +"10640": {"id": "", "name": "OFFENBACH/MAIN", "state": "DL", "lat": 50.120, "lon": 8.730, "elev": 112}, +"10642": {"id": "", "name": "HANAU", "state": "DL", "lat": 50.170, "lon": 8.950, "elev": 112}, +"10687": {"id": "", "name": "GRAFENWOHR", "state": "DL", "lat": 49.700, "lon": 11.950, "elev": 414}, +"10729": {"id": "", "name": "MANNHEIN/NEUOSTHEIM", "state": "DL", "lat": 49.520, "lon": 8.550, "elev": 100}, +"10735": {"id": "", "name": "SINSHEIM (AUT)", "state": "DL", "lat": 49.250, "lon": 8.880, "elev": 169}, +"10739": {"id": "", "name": "STUTTGART/SCHNARRENBERG", "state": "DL", "lat": 48.830, "lon": 9.200, "elev": 311}, +"10771": {"id": "ETGK", "name": "KUEMMERSBRUCK", "state": "DL", "lat": 49.430, "lon": 11.900, "elev": 418}, +"10775": {"id": "ETIH", "name": "HOHENFELS", "state": "DL", "lat": 49.220, "lon": 11.830, "elev": 442}, +"10777": {"id": "", "name": "GELBELSEE", "state": "DL", "lat": 48.950, "lon": 11.430, "elev": 539}, +"10828": {"id": "", "name": "SIGMARINGEN", "state": "DL", "lat": 48.100, "lon": 9.250, "elev": 646}, +"10860": {"id": "ETSI", "name": "INGOLSTADT", "state": "DL", "lat": 48.720, "lon": 11.530, "elev": 370}, +"10862": {"id": "", "name": "SIEGENBERG", "state": "DL", "lat": 48.750, "lon": 11.800, "elev": 404}, +"10868": {"id": "", "name": "MUNICH/OBERSCHLEISSHEIM", "state": "DL", "lat": 48.250, "lon": 11.580, "elev": 489}, +"10954": {"id": "", "name": "ALTENSTADT/SHONGUA", "state": "DL", "lat": 47.830, "lon": 10.870, "elev": 738}, +"10962": {"id": "", "name": "HOHENPEISSENBERG", "state": "DL", "lat": 47.800, "lon": 11.020, "elev": 986}, +"11010": {"id": "LOWL", "name": "LINZ (CIV/MIL)", "state": "OS", "lat": 48.230, "lon": 14.200, "elev": 313}, +"11011": {"id": "LOXL", "name": "HORSCHING", "state": "OS", "lat": 48.230, "lon": 14.180, "elev": 313}, +"11030": {"id": "LOXT", "name": "LANGENLEBARN", "state": "OS", "lat": 48.320, "lon": 16.230, "elev": 176}, +"11035": {"id": "", "name": "WIEN/HOHE WARTE", "state": "OS", "lat": 48.250, "lon": 16.370, "elev": 200}, +"11078": {"id": "", "name": "LILLIENFELD", "state": "OS", "lat": 48.030, "lon": 15.580, "elev": 681}, +"11120": {"id": "LOWI", "name": "INNSBRUCK AIRPORT", "state": "OS", "lat": 47.270, "lon": 11.350, "elev": 581}, +"11144": {"id": "", "name": "ZELL AM SEE", "state": "OS", "lat": 47.330, "lon": 12.800, "elev": 763}, +"11152": {"id": "", "name": "MATTSEE", "state": "OS", "lat": 47.970, "lon": 13.100, "elev": 502}, +"11240": {"id": "LOWG", "name": "GRAZ (MIL/CIV)", "state": "OS", "lat": 47.000, "lon": 15.430, "elev": 347}, +"11241": {"id": "", "name": "SCHOECKL MOUNTAIN", "state": "OS", "lat": 47.200, "lon": 15.470, "elev": 1438}, +"11270": {"id": "", "name": "DELLACH IM DRAUTAL", "state": "OS", "lat": 46.750, "lon": 13.080, "elev": 630}, +"11280": {"id": "", "name": "MURAU", "state": "OS", "lat": 47.120, "lon": 14.180, "elev": 818}, +"11320": {"id": "", "name": "INNSBRUCK/UNIVERSITY", "state": "OS", "lat": 47.270, "lon": 11.380, "elev": 579}, +"11351": {"id": "", "name": "RAMSAU/DACHSTEIN", "state": "OS", "lat": 13.630, "lon": 13.630, "elev": 1207}, +"11520": {"id": "", "name": "PRAGUE/LIBUS", "state": "CZ", "lat": 50.000, "lon": 14.450, "elev": 304}, +"11722": {"id": "", "name": "BRNO REBESOVICE", "state": "CZ", "lat": 49.080, "lon": 16.620, "elev": 195}, +"11747": {"id": "", "name": "PROSTEJOV", "state": "CZ", "lat": 49.450, "lon": 17.130, "elev": 216}, +"11952": {"id": "", "name": "POPRAD/GANOVCE", "state": "LO", "lat": 49.030, "lon": 20.320, "elev": 706}, +"12105": {"id": "", "name": "KOSZALIN", "state": "PL", "lat": 54.200, "lon": 16.200, "elev": 34}, +"12120": {"id": "", "name": "LEBA", "state": "PL", "lat": 54.750, "lon": 17.530, "elev": 2}, +"12185": {"id": "", "name": "KETRZYN", "state": "PL", "lat": 54.070, "lon": 21.370, "elev": 110}, +"12330": {"id": "EPPO", "name": "POZNAN/LAWICA", "state": "PL", "lat": 52.420, "lon": 16.830, "elev": 92}, +"12365": {"id": "", "name": "BRWINOW", "state": "PL", "lat": 52.130, "lon": 20.720, "elev": 98}, +"12374": {"id": "", "name": "LEGIONOWO", "state": "PL", "lat": 52.400, "lon": 20.970, "elev": 96}, +"12424": {"id": "EPWR", "name": "WROCLAW/STRACHOW", "state": "PL", "lat": 51.100, "lon": 16.870, "elev": 121}, +"12425": {"id": "", "name": "WROCLAW/MALY GADOW", "state": "PL", "lat": 51.130, "lon": 16.980, "elev": 116}, +"12520": {"id": "", "name": "KLODZKO", "state": "PL", "lat": 50.430, "lon": 16.620, "elev": 357}, +"12843": {"id": "", "name": "BUDAPEST/LORINC", "state": "HU", "lat": 47.430, "lon": 19.180, "elev": 139}, +"12982": {"id": "LHUD", "name": "SZEGED (AUT)", "state": "HU", "lat": 46.250, "lon": 20.100, "elev": 83}, +"13005": {"id": "", "name": "BOVEC", "state": "YG", "lat": 46.330, "lon": 13.570, "elev": 452}, +"13275": {"id": "", "name": "BEOGRAD KOSUTNJAK", "state": "YG", "lat": 44.770, "lon": 20.420, "elev": 203}, +"13279": {"id": "", "name": "SMEDEREVSKA", "state": "YG", "lat": 44.370, "lon": 20.950, "elev": 122}, +"13334": {"id": "", "name": "SPLIT/MARJAN", "state": "YG", "lat": 43.520, "lon": 16.430, "elev": 129}, +"13388": {"id": "LYBM", "name": "NIS", "state": "YG", "lat": 43.330, "lon": 21.900, "elev": 203}, +"13586": {"id": "LWSK", "name": "SKOPJE/PETROVAC", "state": "YG", "lat": 41.970, "lon": 21.650, "elev": 239}, +"14015": {"id": "", "name": "LUUBLUANA/BEZIGRAD", "state": "LJ", "lat": 46.070, "lon": 14.520, "elev": 316}, +"14240": {"id": "LDDD", "name": "ZAGREB/MAKSIMIR", "state": "RH", "lat": 45.820, "lon": 16.030, "elev": 123}, +"14241": {"id": "LDZA", "name": "ZAGREB/PLESO", "state": "RH", "lat": 45.730, "lon": 16.070, "elev": 107}, +"14320": {"id": "", "name": "CRIKVENICA", "state": "RH", "lat": 45.170, "lon": 14.700, "elev": 2}, +"14430": {"id": "LDZD", "name": "ZADAR ZEMUNIK", "state": "RH", "lat": 44.100, "lon": 15.350, "elev": 82}, +"15001": {"id": "", "name": "AVRAMENI", "state": "RO", "lat": 48.020, "lon": 26.950, "elev": 240}, +"15010": {"id": "", "name": "SATU MARE", "state": "RO", "lat": 47.780, "lon": 22.880, "elev": 122}, +"15080": {"id": "", "name": "ORADEA", "state": "RO", "lat": 47.050, "lon": 21.930, "elev": 136}, +"15090": {"id": "", "name": "IASI", "state": "RO", "lat": 47.170, "lon": 27.600, "elev": 102}, +"15120": {"id": "LRCL", "name": "CLUJ-NAPOCA/SOMESEN", "state": "RO", "lat": 46.780, "lon": 23.570, "elev": 413}, +"15150": {"id": "", "name": "BACAU", "state": "RO", "lat": 46.570, "lon": 26.980, "elev": 182}, +"15189": {"id": "", "name": "DUMBRAVENI", "state": "RO", "lat": 46.230, "lon": 24.570, "elev": 318}, +"15200": {"id": "", "name": "ARAD", "state": "RO", "lat": 46.170, "lon": 21.320, "elev": 108}, +"15209": {"id": "", "name": "BLAJ", "state": "RO", "lat": 46.180, "lon": 23.920, "elev": 334}, +"15310": {"id": "", "name": "GALATI", "state": "RO", "lat": 45.500, "lon": 28.020, "elev": 71}, +"15350": {"id": "", "name": "BUZAU", "state": "RO", "lat": 45.150, "lon": 26.820, "elev": 97}, +"15360": {"id": "", "name": "SULINA", "state": "RO", "lat": 45.150, "lon": 29.670, "elev": 9}, +"15410": {"id": "", "name": "TURNU SEVERAN", "state": "RO", "lat": 44.630, "lon": 22.630, "elev": 77}, +"15420": {"id": "LRBS", "name": "BUCHAREST/BANEASA", "state": "RO", "lat": 44.480, "lon": 26.180, "elev": 91}, +"15425": {"id": "", "name": "SOLACOLU", "state": "RO", "lat": 44.380, "lon": 26.570, "elev": 52}, +"15450": {"id": "", "name": "CRAIOVA", "state": "RO", "lat": 44.230, "lon": 23.870, "elev": 190}, +"15470": {"id": "", "name": "ROSIORI DE VEDE", "state": "RO", "lat": 44.120, "lon": 25.000, "elev": 103}, +"15480": {"id": "", "name": "CONSTANTA", "state": "RO", "lat": 44.220, "lon": 28.630, "elev": 14}, +"15481": {"id": "LRCK", "name": "KOGALNICEANU", "state": "RO", "lat": 44.330, "lon": 28.430, "elev": 102}, +"15490": {"id": "", "name": "TURNU MAGURELE", "state": "RO", "lat": 43.750, "lon": 24.880, "elev": 31}, +"15614": {"id": "LBSF", "name": "SOFIA (OBSERVATORY)", "state": "BU", "lat": 42.650, "lon": 23.380, "elev": 595}, +"15615": {"id": "", "name": "MUSSALA MTN", "state": "BU", "lat": 42.180, "lon": 23.580, "elev": 2927}, +"15730": {"id": "", "name": "KARDJALI", "state": "BU", "lat": 41.630, "lon": 25.400, "elev": 336}, +"16008": {"id": "", "name": "S. VALENTINO", "state": "IY", "lat": 45.750, "lon": 10.530, "elev": 1461}, +"16020": {"id": "LIPB", "name": "BOLZANO", "state": "IY", "lat": 46.470, "lon": 11.330, "elev": 241}, +"16044": {"id": "LIPD", "name": "UDINE/CAMPOFORMIDO", "state": "IY", "lat": 46.030, "lon": 13.180, "elev": 92}, +"16045": {"id": "LIPI", "name": "RIVOLTO AFB", "state": "IY", "lat": 45.980, "lon": 13.030, "elev": 53}, +"16080": {"id": "LIML", "name": "MILANO/LINATE", "state": "IY", "lat": 45.430, "lon": 9.280, "elev": 103}, +"16088": {"id": "LIPL", "name": "GHEDI (IT-AFB)", "state": "IY", "lat": 45.420, "lon": 10.280, "elev": 92}, +"16113": {"id": "", "name": "CUNEO-LEVALDIGI", "state": "IY", "lat": 44.530, "lon": 7.620, "elev": 386}, +"16129": {"id": "LIQP", "name": "PALMARIA ISLAND", "state": "IY", "lat": 44.030, "lon": 9.830, "elev": 192}, +"16142": {"id": "", "name": "RIFREDO MUGELLO", "state": "IY", "lat": 44.050, "lon": 11.380, "elev": 887}, +"16144": {"id": "", "name": "S. PIETRO CAPOFIUME", "state": "IY", "lat": 44.650, "lon": 11.620, "elev": 38}, +"16148": {"id": "LIPC", "name": "CERVIA (IT-AFB)", "state": "IY", "lat": 44.220, "lon": 12.300, "elev": 10}, +"16170": {"id": "LIRQ", "name": "FIRENZE", "state": "IY", "lat": 43.800, "lon": 11.200, "elev": 38}, +"16242": {"id": "", "name": "ROMA/FIUMICINO", "state": "IY", "lat": 41.800, "lon": 12.230, "elev": 5}, +"16245": {"id": "LIRE", "name": "PRATICA DI MARE(AB)", "state": "IY", "lat": 41.650, "lon": 12.430, "elev": 12}, +"16310": {"id": "LIQK", "name": "CAPE PALINURO", "state": "IY", "lat": 40.020, "lon": 15.280, "elev": 185}, +"16316": {"id": "LIBU", "name": "LATRONICO", "state": "IY", "lat": 40.080, "lon": 16.020, "elev": 896}, +"16320": {"id": "LIBR", "name": "BRINDISI/CASALE AFB", "state": "IY", "lat": 40.650, "lon": 17.950, "elev": 10}, +"16325": {"id": "LIBH", "name": "MARINA DI GINOS", "state": "IY", "lat": 40.430, "lon": 16.880, "elev": 12}, +"16350": {"id": "LIBC", "name": "CROTONE", "state": "IY", "lat": 39.000, "lon": 17.070, "elev": 161}, +"16420": {"id": "LICF", "name": "MESSINA", "state": "IY", "lat": 38.200, "lon": 15.550, "elev": 51}, +"16429": {"id": "LICT", "name": "TRAPANI/BIRGI (AFB)", "state": "IY", "lat": 37.920, "lon": 12.500, "elev": 7}, +"16459": {"id": "LICZ", "name": "SIGONELLA", "state": "IY", "lat": 37.400, "lon": 14.920, "elev": 22}, +"16522": {"id": "LIEH", "name": "CAPE CACCIA", "state": "IY", "lat": 40.570, "lon": 8.170, "elev": 205}, +"16546": {"id": "LIED", "name": "DECIMOMANNU", "state": "IY", "lat": 39.350, "lon": 8.970, "elev": 30}, +"16560": {"id": "LIEE", "name": "CAGLIARI/ELMAS(AFB)", "state": "IY", "lat": 39.250, "lon": 9.050, "elev": 1}, +"16597": {"id": "", "name": "LUQA", "state": "MALTA", "lat": 35.850, "lon": 14.480, "elev": 91}, +"16610": {"id": "", "name": "KOMOTINI", "state": "GR", "lat": 41.120, "lon": 25.400, "elev": 31}, +"16622": {"id": "LGTS", "name": "THESSALONIKI/MIKRA", "state": "GR", "lat": 40.520, "lon": 22.970, "elev": 4}, +"16706": {"id": "LGHI", "name": "CHIOS AIRPORT", "state": "GR", "lat": 38.220, "lon": 26.130, "elev": 4}, +"16716": {"id": "LGAT", "name": "ATHENS/HELLENKION", "state": "GR", "lat": 37.900, "lon": 23.730, "elev": 15}, +"16726": {"id": "LGKL", "name": "KALAMATA", "state": "GR", "lat": 37.070, "lon": 22.020, "elev": 8}, +"16744": {"id": "", "name": "THIRA ISLAND", "state": "GR", "lat": 36.420, "lon": 25.430, "elev": 36}, +"16754": {"id": "LGIR", "name": "IRAKLION (CIV/AFB)", "state": "GR", "lat": 35.330, "lon": 25.180, "elev": 39}, +"16759": {"id": "", "name": "TYMBAKION", "state": "GR", "lat": 35.000, "lon": 24.750, "elev": 7}, +"16760": {"id": "", "name": "KASTELI", "state": "GR", "lat": 35.180, "lon": 25.320, "elev": 360}, +"17030": {"id": "", "name": "SAMSUN CITY", "state": "TU", "lat": 41.280, "lon": 36.330, "elev": 4}, +"17038": {"id": "LTCG", "name": "TRABZON", "state": "TU", "lat": 41.000, "lon": 39.720, "elev": 30}, +"17042": {"id": "", "name": "HOPA", "state": "TU", "lat": 41.380, "lon": 41.420, "elev": 33}, +"17050": {"id": "", "name": "EDIRNE", "state": "TU", "lat": 41.670, "lon": 26.570, "elev": 48}, +"17062": {"id": "", "name": "ISTANBUL/GOZTEPE", "state": "TU", "lat": 40.970, "lon": 29.080, "elev": 33}, +"17064": {"id": "", "name": "ISTANBUL/KARTAL", "state": "TU", "lat": 40.900, "lon": 29.150, "elev": 18}, +"17067": {"id": "", "name": "GOLCUK/DUMLUPINAR", "state": "TU", "lat": 40.720, "lon": 29.820, "elev": 18}, +"17068": {"id": "", "name": "CENGIZTOPEL", "state": "TU", "lat": 40.730, "lon": 30.080, "elev": 70}, +"17095": {"id": "", "name": "ERZURUM BOLGE", "state": "TU", "lat": 39.900, "lon": 41.280, "elev": 1869}, +"17100": {"id": "", "name": "IGDIR", "state": "TU", "lat": 39.930, "lon": 44.030, "elev": 858}, +"17130": {"id": "", "name": "ANKARA/CENTRAL", "state": "TU", "lat": 39.950, "lon": 32.880, "elev": 894}, +"17160": {"id": "", "name": "KIRSEHIR", "state": "TU", "lat": 39.150, "lon": 34.170, "elev": 995}, +"17162": {"id": "", "name": "GEMEREK", "state": "TU", "lat": 39.180, "lon": 36.070, "elev": 1171}, +"17196": {"id": "LTAM", "name": "KAISERI CITY", "state": "TU", "lat": 38.720, "lon": 35.500, "elev": 1043}, +"17200": {"id": "LTAT", "name": "MALATYA/ERHAC", "state": "TU", "lat": 38.430, "lon": 38.080, "elev": 849}, +"17220": {"id": "", "name": "IZMIR", "state": "TU", "lat": 38.430, "lon": 27.170, "elev": 29}, +"17240": {"id": "LTBM", "name": "ISPARTA", "state": "TU", "lat": 37.750, "lon": 30.550, "elev": 997}, +"17250": {"id": "", "name": "NIGDE", "state": "TU", "lat": 37.970, "lon": 34.680, "elev": 1208}, +"17270": {"id": "LTCH", "name": "SANLIURFA", "state": "TU", "lat": 37.130, "lon": 38.770, "elev": 549}, +"17280": {"id": "LTCC", "name": "DIYARBAKIR(CIV/AFB)", "state": "TU", "lat": 37.880, "lon": 40.200, "elev": 674}, +"17281": {"id": "", "name": "DIYABAKIR-BOLGE", "state": "TU", "lat": 37.900, "lon": 40.200, "elev": 675}, +"17300": {"id": "", "name": "ANTALYA", "state": "TU", "lat": 36.700, "lon": 30.730, "elev": 43}, +"17350": {"id": "LTAG", "name": "ADANA/INCIRLIK", "state": "TU", "lat": 37.000, "lon": 35.420, "elev": 66}, +"17351": {"id": "", "name": "ADANA/BOLGE", "state": "TU", "lat": 36.980, "lon": 35.350, "elev": 28}, +"17352": {"id": "LTAF", "name": "ADANA/SAKIRPASA", "state": "TU", "lat": 36.980, "lon": 35.300, "elev": 20}, +"17600": {"id": "LCPH", "name": "PAPHOS", "state": "CY", "lat": 34.470, "lon": 32.250, "elev": 8}, +"17601": {"id": "LCRA", "name": "AKROTIRI (RAF)", "state": "CY", "lat": 34.580, "lon": 32.980, "elev": 23}, +"17606": {"id": "", "name": "NICOSIA", "state": "CY", "lat": 35.200, "lon": 33.300, "elev": 224}, +"17607": {"id": "LCNC", "name": "NICOSIA/ATHALASSA", "state": "CY", "lat": 35.200, "lon": 33.400, "elev": 161}, +"17609": {"id": "LCLK", "name": "LARNACA AIRPORT", "state": "CY", "lat": 34.880, "lon": 33.630, "elev": 2}, +"20046": {"id": "", "name": "OSTROV KHEISA", "state": "DK RA", "lat": 80.620, "lon": 58.050, "elev": 20}, +"20069": {"id": "", "name": "OSTROV LIEZ", "state": "DK RA", "lat": 79.500, "lon": 76.980, "elev": 12}, +"20107": {"id": "", "name": "BARENCBURG", "state": "AR RS", "lat": 78.070, "lon": 14.220, "elev": 49}, +"20274": {"id": "", "name": "UJEDINENIJA ISLAND", "state": "DK RA", "lat": 77.500, "lon": 82.230, "elev": 23}, +"20292": {"id": "", "name": "CAPE CELJUSKIN", "state": "DK RA", "lat": 77.720, "lon": 104.300, "elev": 13}, +"20353": {"id": "", "name": "MYS ZHELANIA", "state": "DK RA", "lat": 76.950, "lon": 68.580, "elev": 8}, +"20667": {"id": "", "name": "BELY ISLAND", "state": "DK RA", "lat": 73.330, "lon": 70.030, "elev": 6}, +"20674": {"id": "", "name": "DIKSON ISLAND", "state": "DK RA", "lat": 73.530, "lon": 80.400, "elev": 47}, +"20744": {"id": "", "name": "MALYE KARMAKULY", "state": "DK RA", "lat": 72.380, "lon": 52.730, "elev": 16}, +"20891": {"id": "", "name": "HATANGA", "state": "DK RA", "lat": 72.000, "lon": 102.570, "elev": 33}, +"21358": {"id": "", "name": "OSTROV ZOHOVA", "state": "TK RA", "lat": 76.150, "lon": 152.830, "elev": 14}, +"21432": {"id": "", "name": "KOTEL'NYJ ISLAND", "state": "TK RA", "lat": 76.000, "lon": 137.900, "elev": 10}, +"21504": {"id": "", "name": "PREOBRAZENIJA ISLAND", "state": "TK RA", "lat": 74.670, "lon": 112.930, "elev": 57}, +"21647": {"id": "", "name": "CAPE SALAUROVA", "state": "TK RA", "lat": 73.180, "lon": 143.930, "elev": 22}, +"21824": {"id": "", "name": "TIKSI", "state": "TK RA", "lat": 71.580, "lon": 128.920, "elev": 8}, +"21908": {"id": "", "name": "DZALINDA", "state": "TK RA", "lat": 70.130, "lon": 113.970, "elev": 62}, +"21946": {"id": "", "name": "COKURDAH", "state": "TK RA", "lat": 70.620, "lon": 147.900, "elev": 61}, +"21965": {"id": "", "name": "CHETYREKHSTOLBOVOY", "state": "TK RA", "lat": 70.630, "lon": 162.400, "elev": 38}, +"21982": {"id": "", "name": "VRANGELJA ISLAND", "state": "TK RA", "lat": 70.980, "lon": -178.480, "elev": 3}, +"22008": {"id": "ULMM", "name": "MURMANSK", "state": "AR RS", "lat": 68.980, "lon": 33.120, "elev": 121}, +"22113": {"id": "ULMM", "name": "MURMANSK", "state": "AR RS", "lat": 68.970, "lon": 33.050, "elev": 121}, +"22114": {"id": "", "name": "LOPARSKAYA", "state": "AR RS", "lat": 68.630, "lon": 33.200, "elev": 112}, +"22204": {"id": "", "name": "KOVDOR", "state": "AR RS", "lat": 67.570, "lon": 30.400, "elev": 247}, +"22213": {"id": "", "name": "APATITOVAYA", "state": "AR RS", "lat": 67.830, "lon": 33.750, "elev": 135}, +"22217": {"id": "", "name": "KANDALAKSA", "state": "AR RS", "lat": 67.150, "lon": 32.350, "elev": 26}, +"22271": {"id": "", "name": "SOJNA", "state": "AR RS", "lat": 67.880, "lon": 44.130, "elev": 16}, +"22522": {"id": "", "name": "KEM (PORT)", "state": "LE RS", "lat": 64.980, "lon": 34.800, "elev": 10}, +"22543": {"id": "", "name": "ARCHANGEL'SK", "state": "AR RS", "lat": 64.620, "lon": 40.510, "elev": 6}, +"22550": {"id": "ULAA", "name": "ARCHANGEL'SK", "state": "AR RS", "lat": 64.530, "lon": 40.470, "elev": 13}, +"22551": {"id": "", "name": "MUD'JUG ISLAND", "state": "AR RS", "lat": 64.850, "lon": 40.280, "elev": 3}, +"22802": {"id": "", "name": "SORTOVALA", "state": "LE RS", "lat": 61.720, "lon": 30.720, "elev": 18}, +"22820": {"id": "", "name": "PETROZAVODSK", "state": "LE RS", "lat": 61.820, "lon": 34.270, "elev": 112}, +"22845": {"id": "", "name": "KARGOPOL'", "state": "AR RS", "lat": 61.500, "lon": 38.930, "elev": 121}, +"23022": {"id": "", "name": "AMDERMA", "state": "DK RA", "lat": 69.770, "lon": 61.680, "elev": 53}, +"23058": {"id": "", "name": "ANTIPAJETA", "state": "DK RA", "lat": 69.150, "lon": 77.000, "elev": 4}, +"23078": {"id": "", "name": "NORILSK", "state": "DK RA", "lat": 69.330, "lon": 88.100, "elev": 62}, +"23146": {"id": "", "name": "MYS KAMENNY", "state": "DK RA", "lat": 68.470, "lon": 73.600, "elev": 2}, +"23205": {"id": "", "name": "NAR'JAN-MAR", "state": "AR RA", "lat": 67.650, "lon": 53.020, "elev": 7}, +"23215": {"id": "", "name": "HOREJ-VER", "state": "AR RA", "lat": 67.430, "lon": 58.030, "elev": 71}, +"23274": {"id": "", "name": "YGARKA", "state": "AR RA", "lat": 67.470, "lon": 86.570, "elev": 20}, +"23330": {"id": "", "name": "SALEHARD", "state": "NO RA", "lat": 66.530, "lon": 66.530, "elev": 16}, +"23415": {"id": "", "name": "PECHORA", "state": "AR RA", "lat": 65.120, "lon": 57.100, "elev": 56}, +"23418": {"id": "", "name": "PECORA", "state": "AR RA", "lat": 65.120, "lon": 57.100, "elev": 56}, +"23472": {"id": "", "name": "TURUHANSK", "state": "NO RA", "lat": 65.780, "lon": 87.950, "elev": 32}, +"23552": {"id": "", "name": "TARKO-SALE", "state": "NO RA", "lat": 64.920, "lon": 77.820, "elev": 27}, +"23802": {"id": "UUYY", "name": "SYKTYVKAR'", "state": "AR RA", "lat": 61.680, "lon": 50.780, "elev": 116}, +"23804": {"id": "UUYY", "name": "SYKTYVKAR", "state": "AR RA", "lat": 61.720, "lon": 50.830, "elev": 119}, +"23884": {"id": "", "name": "BOR", "state": "NO RA", "lat": 61.600, "lon": 90.000, "elev": 63}, +"23921": {"id": "", "name": "IVDEL'", "state": "SV RA", "lat": 60.680, "lon": 60.430, "elev": 101}, +"23927": {"id": "", "name": "SEVEROURALSK", "state": "SV RA", "lat": 60.150, "lon": 60.020, "elev": 186}, +"23933": {"id": "USHH", "name": "HANTY-MANSIJSK", "state": "NO RA", "lat": 60.970, "lon": 69.070, "elev": 40}, +"23955": {"id": "", "name": "ALEKSANDROVSKOE", "state": "NO RA", "lat": 60.430, "lon": 77.870, "elev": 47}, +"24122": {"id": "", "name": "OLENEK", "state": "HA RA", "lat": 68.500, "lon": 112.430, "elev": 220}, +"24125": {"id": "", "name": "OLENEK", "state": "HA RA", "lat": 68.500, "lon": 112.430, "elev": 220}, +"24266": {"id": "", "name": "VERHOJANSK", "state": "HA RA", "lat": 67.550, "lon": 133.380, "elev": 137}, +"24343": {"id": "", "name": "ZIGANSK", "state": "HA RA", "lat": 66.770, "lon": 123.400, "elev": 93}, +"24507": {"id": "", "name": "TURA", "state": "NO RA", "lat": 64.270, "lon": 100.230, "elev": 186}, +"24641": {"id": "", "name": "VILJUJSK", "state": "HA RA", "lat": 63.770, "lon": 121.620, "elev": 107}, +"24688": {"id": "", "name": "OJMJAKON", "state": "HA RA", "lat": 63.270, "lon": 143.150, "elev": 745}, +"24726": {"id": "UERR", "name": "MIRNY", "state": "HB RA", "lat": 62.550, "lon": 113.880, "elev": 347}, +"24790": {"id": "", "name": "SUSUMAN", "state": "HB RA", "lat": 62.780, "lon": 148.130, "elev": 649}, +"24817": {"id": "", "name": "ERBOGACEN", "state": "IR RA", "lat": 61.270, "lon": 108.020, "elev": 291}, +"24908": {"id": "", "name": "VANAVARA", "state": "NO RA", "lat": 60.330, "lon": 102.270, "elev": 260}, +"24944": {"id": "", "name": "OLEKMINSK", "state": "HA RA", "lat": 60.400, "lon": 120.420, "elev": 226}, +"24947": {"id": "", "name": "OLEKMINSK", "state": "HA RA", "lat": 60.370, "lon": 120.420, "elev": 226}, +"24959": {"id": "UEEE", "name": "YAKUTSK AIRPORT", "state": "HA RA", "lat": 62.080, "lon": 129.750, "elev": 103}, +"25042": {"id": "", "name": "AJON ISLAND", "state": "HB RA", "lat": 69.830, "lon": 168.670, "elev": 16}, +"25123": {"id": "", "name": "CERSKIJ", "state": "HA RA", "lat": 68.800, "lon": 161.280, "elev": 32}, +"25173": {"id": "", "name": "CAPE SMIDTA", "state": "HA RA", "lat": 68.920, "lon": 179.480, "elev": 7}, +"25399": {"id": "", "name": "CAPE UELEN", "state": "HA RA", "lat": 66.170, "lon": -169.830, "elev": 7}, +"25400": {"id": "", "name": "ZYRJANKA", "state": "HA RA", "lat": 65.730, "lon": 150.900, "elev": 43}, +"25403": {"id": "", "name": "ZYRYANKA", "state": "HA RA", "lat": 65.730, "lon": 150.900, "elev": 43}, +"25428": {"id": "", "name": "OMOLON", "state": "HA RA", "lat": 65.220, "lon": 160.500, "elev": 253}, +"25551": {"id": "", "name": "MARKOVO", "state": "HA RA", "lat": 64.680, "lon": 170.420, "elev": 33}, +"25563": {"id": "UHMA", "name": "ANADYR'", "state": "HA RA", "lat": 64.780, "lon": 177.570, "elev": 62}, +"25594": {"id": "", "name": "BUKHTA PROVIDENCIA", "state": "HA RA", "lat": 64.430, "lon": -173.230, "elev": 3}, +"25677": {"id": "", "name": "UGOLNAJA", "state": "HA RA", "lat": 63.050, "lon": 179.320, "elev": 85}, +"25703": {"id": "", "name": "SEJMCAN", "state": "HA RA", "lat": 62.920, "lon": 152.420, "elev": 207}, +"25822": {"id": "", "name": "KUSHKA", "state": "HA RA", "lat": 61.950, "lon": 160.370, "elev": 20}, +"25913": {"id": "UHMM", "name": "MAGADAN", "state": "HA RA", "lat": 59.580, "lon": 150.780, "elev": 118}, +"25954": {"id": "", "name": "KORF", "state": "HA RA", "lat": 60.350, "lon": 166.000, "elev": 3}, +"26038": {"id": "ULTT", "name": "TALLIN", "state": "HA RA", "lat": 59.450, "lon": 24.800, "elev": 44}, +"26063": {"id": "ULLI", "name": "ST. PETERSBURG", "state": "LE RS", "lat": 59.970, "lon": 30.300, "elev": 72}, +"26075": {"id": "", "name": "ST. PETERSBURG", "state": "LE RS", "lat": 59.950, "lon": 30.700, "elev": 72}, +"26080": {"id": "", "name": "MOTOKHOVO", "state": "LE RS", "lat": 59.530, "lon": 32.380, "elev": 34}, +"26258": {"id": "", "name": "PSKOV", "state": "LE RS", "lat": 57.800, "lon": 28.420, "elev": 42}, +"26291": {"id": "", "name": "BOROVICI", "state": "LE RS", "lat": 58.420, "lon": 33.900, "elev": 92}, +"26298": {"id": "", "name": "BOLOGOE", "state": "LE RS", "lat": 57.900, "lon": 34.050, "elev": 178}, +"26378": {"id": "", "name": "HOLM", "state": "LE RS", "lat": 57.150, "lon": 31.180, "elev": 71}, +"26381": {"id": "", "name": "DEMJANSK", "state": "LE RS", "lat": 57.650, "lon": 32.470, "elev": 48}, +"26406": {"id": "", "name": "LIEPAJA", "state": "LE BY", "lat": 56.480, "lon": 21.020, "elev": 7}, +"26422": {"id": "UMRR", "name": "RIGA", "state": "LE BY", "lat": 56.970, "lon": 24.070, "elev": 7}, +"26425": {"id": "", "name": "GELGABA", "state": "LE BY", "lat": 56.650, "lon": 23.730, "elev": 9}, +"26429": {"id": "", "name": "BAUSKA", "state": "RA", "lat": 56.400, "lon": 24.220, "elev": 33}, +"26435": {"id": "", "name": "LVA SKRIVERI", "state": "LE", "lat": 56.650, "lon": 25.130, "elev": 83}, +"26441": {"id": "", "name": "MADONA", "state": "BY", "lat": 56.880, "lon": 26.330, "elev": 251}, +"26477": {"id": "ULOL", "name": "VELIKIE LUKI", "state": "LE RS", "lat": 56.380, "lon": 30.600, "elev": 98}, +"26529": {"id": "", "name": "PANEVEZYS", "state": "MI BY", "lat": 55.750, "lon": 24.380, "elev": 59}, +"26629": {"id": "", "name": "KAUNAS", "state": "MI BY", "lat": 54.880, "lon": 23.880, "elev": 75}, +"26702": {"id": "", "name": "KALININGRAD", "state": "MI BY", "lat": 54.700, "lon": 20.620, "elev": 27}, +"26781": {"id": "", "name": "SMOLENSK", "state": "MI RS", "lat": 54.750, "lon": 32.070, "elev": 241}, +"26850": {"id": "UMMS", "name": "MINSK", "state": "MI BY", "lat": 53.870, "lon": 27.530, "elev": 234}, +"26863": {"id": "", "name": "MOGILEV", "state": "MI BY", "lat": 53.900, "lon": 30.320, "elev": 193}, +"27037": {"id": "ULWW", "name": "VOLOGDA", "state": "AR RS", "lat": 59.230, "lon": 39.870, "elev": 131}, +"27038": {"id": "", "name": "VOLOGDA", "state": "AR RS", "lat": 59.320, "lon": 39.930, "elev": 130}, +"27199": {"id": "", "name": "KIROV", "state": "MS RS", "lat": 58.600, "lon": 49.630, "elev": 158}, +"27225": {"id": "", "name": "RYBINSK", "state": "MS RS", "lat": 58.000, "lon": 38.830, "elev": 114}, +"27459": {"id": "", "name": "NIZNIJ NOVGOROD", "state": "MS RS", "lat": 56.270, "lon": 44.000, "elev": 157}, +"27594": {"id": "", "name": "KAZAN'", "state": "RS", "lat": 55.820, "lon": 48.520, "elev": 127}, +"27595": {"id": "UWKD", "name": "KAZAN'", "state": "MS RS", "lat": 55.780, "lon": 49.180, "elev": 116}, +"27611": {"id": "", "name": "NARO FOMINSK", "state": "MS RS", "lat": 55.380, "lon": 36.700, "elev": 193}, +"27612": {"id": "", "name": "MOSCOW", "state": "MS RS", "lat": 55.750, "lon": 37.950, "elev": 200}, +"27618": {"id": "", "name": "SERPUHOV", "state": "MS RS", "lat": 54.930, "lon": 37.470, "elev": 166}, +"27707": {"id": "", "name": "SUHINICI", "state": "MS RS", "lat": 54.120, "lon": 35.330, "elev": 239}, +"27713": {"id": "UUDD", "name": "MOSCOW POLGOPRUDNYJ", "state": "MS RS", "lat": 55.400, "lon": 37.900, "elev": 179}, +"27730": {"id": "", "name": "RJAZAN'", "state": "MS RS", "lat": 54.630, "lon": 39.700, "elev": 158}, +"27944": {"id": "", "name": "TAMBOV", "state": "MS RS", "lat": 52.720, "lon": 41.780, "elev": 161}, +"27947": {"id": "", "name": "TAMBOV", "state": "MS RS", "lat": 52.730, "lon": 41.470, "elev": 139}, +"27962": {"id": "UWPP", "name": "PENZA", "state": "MS RS", "lat": 53.130, "lon": 45.020, "elev": 174}, +"27995": {"id": "", "name": "BEZENCUKSKAJA", "state": "MS RS", "lat": 52.980, "lon": 49.430, "elev": 45}, +"28225": {"id": "USPP", "name": "PERM", "state": "SV RA", "lat": 57.950, "lon": 56.200, "elev": 170}, +"28275": {"id": "", "name": "TOBOLSK", "state": "NO RA", "lat": 58.150, "lon": 68.180, "elev": 44}, +"28440": {"id": "USSS", "name": "SVERDLOVSK", "state": "SV RA", "lat": 56.800, "lon": 60.630, "elev": 237}, +"28445": {"id": "", "name": "VERHNEE DUBROVO", "state": "SV RA", "lat": 56.730, "lon": 61.070, "elev": 290}, +"28598": {"id": "", "name": "SARGATSKOE", "state": "NO RA", "lat": 55.600, "lon": 73.480, "elev": 87}, +"28601": {"id": "", "name": "CISTOPOL", "state": "RA", "lat": 55.350, "lon": 50.620, "elev": 113}, +"28661": {"id": "", "name": "KURGAN", "state": "SV RA", "lat": 55.470, "lon": 65.400, "elev": 79}, +"28695": {"id": "", "name": "OMSK", "state": "NO RA", "lat": 54.930, "lon": 73.400, "elev": 122}, +"28698": {"id": "UNDD", "name": "OMSK", "state": "NO RA", "lat": 54.930, "lon": 73.400, "elev": 123}, +"28712": {"id": "", "name": "TUIMAZY", "state": "SV RA", "lat": 54.580, "lon": 53.730, "elev": 126}, +"28722": {"id": "UWUU", "name": "UFA", "state": "SV RA", "lat": 54.750, "lon": 56.000, "elev": 105}, +"28900": {"id": "UWWW", "name": "SAMARA", "state": "AL RA", "lat": 53.250, "lon": 50.450, "elev": 40}, +"28951": {"id": "", "name": "KOSTANAI", "state": "AL RA", "lat": 53.230, "lon": 63.620, "elev": 171}, +"28952": {"id": "", "name": "KUSTANAJ", "state": "AL RA", "lat": 53.220, "lon": 63.620, "elev": 156}, +"29231": {"id": "", "name": "KOLPASEV", "state": "NO RA", "lat": 58.300, "lon": 82.900, "elev": 76}, +"29263": {"id": "UNII", "name": "YENISEYSK", "state": "NO RA", "lat": 58.450, "lon": 92.150, "elev": 78}, +"29282": {"id": "", "name": "BOGUCANY", "state": "NO RA", "lat": 58.420, "lon": 97.400, "elev": 134}, +"29348": {"id": "", "name": "PERVOMAJSKOE", "state": "NO RA", "lat": 57.070, "lon": 86.220, "elev": 115}, +"29498": {"id": "", "name": "SITKINO", "state": "NO RA", "lat": 56.350, "lon": 98.370, "elev": 221}, +"29563": {"id": "", "name": "KACA", "state": "RA", "lat": 56.120, "lon": 92.200, "elev": 479}, +"29572": {"id": "", "name": "EMEL'JANOVO", "state": "NO RA", "lat": 56.180, "lon": 92.620, "elev": 296}, +"29574": {"id": "", "name": "KRASNOJARSK", "state": "NO RA", "lat": 56.000, "lon": 92.880, "elev": 206}, +"29612": {"id": "", "name": "BARABINSK", "state": "NO RA", "lat": 55.370, "lon": 78.370, "elev": 120}, +"29634": {"id": "UNNT", "name": "NOVOSIBIRSK", "state": "NO RA", "lat": 55.030, "lon": 82.900, "elev": 177}, +"29636": {"id": "", "name": "TOGUCIN", "state": "NO RA", "lat": 55.230, "lon": 84.400, "elev": 171}, +"29698": {"id": "UINN", "name": "NIZNE-UDINSK", "state": "IR RA", "lat": 54.880, "lon": 99.030, "elev": 410}, +"29736": {"id": "", "name": "MASLJANINO", "state": "NO RA", "lat": 54.330, "lon": 84.220, "elev": 198}, +"29838": {"id": "UNBB", "name": "BARNAUL", "state": "NO RA", "lat": 53.400, "lon": 83.700, "elev": 252}, +"29839": {"id": "", "name": "BARNAUL", "state": "NO RA", "lat": 53.210, "lon": 83.490, "elev": 159}, +"29862": {"id": "", "name": "HAKASSKATA", "state": "NO RA", "lat": 53.770, "lon": 91.320, "elev": 256}, +"29999": {"id": "", "name": "ORLIK (MTN STN)", "state": "IR RA", "lat": 52.500, "lon": 99.820, "elev": 1570}, +"30054": {"id": "", "name": "VITIM", "state": "HA RA", "lat": 59.450, "lon": 112.580, "elev": 193}, +"30230": {"id": "UIKK", "name": "KIRENSK", "state": "IR RA", "lat": 57.770, "lon": 108.120, "elev": 258}, +"30309": {"id": "UIBB", "name": "BRATSK", "state": "IR RA", "lat": 56.070, "lon": 101.830, "elev": 489}, +"30372": {"id": "", "name": "CARA", "state": "IR RA", "lat": 56.920, "lon": 118.370, "elev": 711}, +"30457": {"id": "", "name": "BAUNT", "state": "RA", "lat": 55.270, "lon": 113.130, "elev": 1078}, +"30507": {"id": "", "name": "IKEJ", "state": "IR RA", "lat": 54.180, "lon": 100.080, "elev": 527}, +"30521": {"id": "", "name": "ZIGALOVO", "state": "IR RA", "lat": 54.800, "lon": 105.220, "elev": 428}, +"30554": {"id": "", "name": "BOGDARIN", "state": "IR RA", "lat": 54.620, "lon": 113.130, "elev": 995}, +"30557": {"id": "", "name": "BAGDARIN", "state": "IR RA", "lat": 54.470, "lon": 113.580, "elev": 995}, +"30565": {"id": "", "name": "UST-KARENGA", "state": "IR RA", "lat": 54.450, "lon": 116.520, "elev": 686}, +"30635": {"id": "", "name": "UST-BARGUZIN", "state": "IR RA", "lat": 54.420, "lon": 109.020, "elev": 457}, +"30673": {"id": "", "name": "MOGOCA", "state": "IR RA", "lat": 53.730, "lon": 119.780, "elev": 619}, +"30692": {"id": "", "name": "SKOVORODINO", "state": "HA RA", "lat": 54.000, "lon": 123.970, "elev": 400}, +"30710": {"id": "", "name": "IRKUTSK", "state": "IR RA", "lat": 52.400, "lon": 104.200, "elev": 437}, +"30715": {"id": "", "name": "ANGARSK", "state": "IR RA", "lat": 52.480, "lon": 103.850, "elev": 450}, +"30758": {"id": "UIAA", "name": "CITA/KADALA", "state": "IR RA", "lat": 52.020, "lon": 113.330, "elev": 685}, +"30815": {"id": "", "name": "HAMAR-DABAN", "state": "IR RA", "lat": 51.530, "lon": 103.600, "elev": 1442}, +"30858": {"id": "", "name": "KUROT-DARASUA", "state": "IR RA", "lat": 51.200, "lon": 113.720, "elev": 805}, +"30935": {"id": "", "name": "KRASNYJ CIKOJ", "state": "IR RA", "lat": 50.370, "lon": 108.750, "elev": 770}, +"30965": {"id": "", "name": "BORZJA", "state": "IR RA", "lat": 50.380, "lon": 116.520, "elev": 684}, +"31004": {"id": "", "name": "ALDAN", "state": "HA RA", "lat": 58.620, "lon": 125.370, "elev": 682}, +"31088": {"id": "", "name": "OHOTSK", "state": "HA RA", "lat": 59.370, "lon": 143.200, "elev": 6}, +"31168": {"id": "", "name": "AJAN", "state": "HA RA", "lat": 56.450, "lon": 138.150, "elev": 9}, +"31300": {"id": "", "name": "ZEJA", "state": "HA RA", "lat": 53.750, "lon": 127.230, "elev": 232}, +"31329": {"id": "UHBP", "name": "EKIMCAN", "state": "HA RA", "lat": 53.070, "lon": 132.930, "elev": 543}, +"31369": {"id": "UHNN", "name": "NIKOLAEVSK", "state": "HA RA", "lat": 53.150, "lon": 140.700, "elev": 68}, +"31510": {"id": "", "name": "BLAGOVESCENSK", "state": "HA RA", "lat": 50.270, "lon": 127.500, "elev": 137}, +"31538": {"id": "", "name": "SUTUR", "state": "HA RA", "lat": 50.070, "lon": 132.130, "elev": 349}, +"31561": {"id": "", "name": "KOMSOMOLSK", "state": "HB RA", "lat": 50.600, "lon": 137.030, "elev": 19}, +"31707": {"id": "", "name": "EKATERINO-NIKOL", "state": "HA RA", "lat": 47.730, "lon": 130.970, "elev": 74}, +"31735": {"id": "UHHH", "name": "HABAROVSK/NOVY", "state": "HA RA", "lat": 48.520, "lon": 135.170, "elev": 72}, +"31736": {"id": "", "name": "HABAROVSK", "state": "HA RA", "lat": 48.530, "lon": 135.230, "elev": 72}, +"31770": {"id": "", "name": "SOVETSKAY GAVAN", "state": "HA RA", "lat": 49.000, "lon": 140.270, "elev": 22}, +"31873": {"id": "", "name": "DAL'NERECENSK", "state": "HA RA", "lat": 45.870, "lon": 133.730, "elev": 107}, +"31909": {"id": "", "name": "TERNEJ", "state": "HA RA", "lat": 45.030, "lon": 136.670, "elev": 68}, +"31960": {"id": "", "name": "VLADIVOSTOK", "state": "HA RA", "lat": 43.120, "lon": 131.900, "elev": 78}, +"31977": {"id": "", "name": "SAD-GOROD", "state": "HB RA", "lat": 43.270, "lon": 132.050, "elev": 82}, +"32061": {"id": "", "name": "ALEKSANDROVSK-SAHALINSKIJ", "state": "RA", "lat": 50.900, "lon": 142.170, "elev": 31}, +"32098": {"id": "", "name": "PORONAJSK", "state": "HA RA", "lat": 49.220, "lon": 143.100, "elev": 4}, +"32099": {"id": "", "name": "TERPENIYA", "state": "HA RA", "lat": 48.500, "lon": 144.750, "elev": 33}, +"32150": {"id": "UHSS", "name": "JUZNO-SAHALINSK", "state": "HA RA", "lat": 46.920, "lon": 142.730, "elev": 31}, +"32165": {"id": "", "name": "JUZNO-KURILSK", "state": "HA RA", "lat": 44.020, "lon": 145.870, "elev": 40}, +"32186": {"id": "", "name": "URUP ISLAND", "state": "HA RA", "lat": 46.200, "lon": 150.500, "elev": 70}, +"32215": {"id": "", "name": "SEVERO-KURILSK", "state": "HA RA", "lat": 50.680, "lon": 156.130, "elev": 23}, +"32217": {"id": "", "name": "MYS VASIL'EVA", "state": "HA RA", "lat": 50.020, "lon": 155.400, "elev": 23}, +"32389": {"id": "", "name": "KLJUCI", "state": "HA RA", "lat": 56.320, "lon": 160.830, "elev": 28}, +"32477": {"id": "", "name": "SOBOLEVO", "state": "HA RA", "lat": 54.300, "lon": 155.970, "elev": 25}, +"32540": {"id": "UHPP", "name": "PETROPAVLOVSK-KAMCHATSKIJ", "state": "RA", "lat": 52.970, "lon": 158.750, "elev": 24}, +"32618": {"id": "", "name": "NIKOLSKOE/BERINGA", "state": "HA RA", "lat": 55.200, "lon": 165.980, "elev": 6}, +"33008": {"id": "", "name": "BREST", "state": "MI BY", "lat": 52.120, "lon": 23.670, "elev": 142}, +"33041": {"id": "UMGG", "name": "GOMEL", "state": "MI BY", "lat": 52.400, "lon": 30.950, "elev": 126}, +"33317": {"id": "", "name": "SEPETOVKA", "state": "KI UR", "lat": 50.170, "lon": 27.050, "elev": 278}, +"33345": {"id": "UKKK", "name": "KIEV/ZHULYANY", "state": "KI UR", "lat": 50.400, "lon": 30.450, "elev": 168}, +"33347": {"id": "UKBB", "name": "BORYSPIL", "state": "KI UR", "lat": 50.330, "lon": 30.970, "elev": 122}, +"33393": {"id": "UKLL", "name": "L'VIV", "state": "KI UR", "lat": 49.820, "lon": 23.950, "elev": 325}, +"33631": {"id": "", "name": "UZGOROD", "state": "KI UR", "lat": 48.630, "lon": 22.270, "elev": 118}, +"33658": {"id": "", "name": "CHERNIVTSI", "state": "KI UR", "lat": 48.370, "lon": 25.900, "elev": 246}, +"33663": {"id": "", "name": "MOGILEV-PODOL'SKIJ", "state": "KI UR", "lat": 48.450, "lon": 27.780, "elev": 79}, +"33791": {"id": "", "name": "KRYVYI RIH", "state": "KI UR", "lat": 48.030, "lon": 33.220, "elev": 124}, +"33805": {"id": "", "name": "NIKOPOL'", "state": "KV UR", "lat": 47.580, "lon": 34.400, "elev": 56}, +"33815": {"id": "UKII", "name": "KISHINEV", "state": "KI UR", "lat": 47.020, "lon": 28.870, "elev": 180}, +"33837": {"id": "UKOO", "name": "ODESSA/TSENTRALNY", "state": "KI UR", "lat": 46.430, "lon": 30.770, "elev": 35}, +"33877": {"id": "", "name": "NIZNIE SEROGOZY", "state": "KI UR", "lat": 46.850, "lon": 34.400, "elev": 54}, +"33945": {"id": "", "name": "POCTOVOE", "state": "KI UR", "lat": 44.830, "lon": 33.950, "elev": 170}, +"33946": {"id": "UKFF", "name": "SIMFEROPOL'", "state": "KI UR", "lat": 44.680, "lon": 34.130, "elev": 181}, +"33966": {"id": "", "name": "KRYMSKAYA", "state": "KI UR", "lat": 45.050, "lon": 34.600, "elev": 183}, +"34009": {"id": "", "name": "KURSK", "state": "MS RS", "lat": 51.770, "lon": 36.170, "elev": 247}, +"34072": {"id": "", "name": "KARABULAK", "state": "MS RS", "lat": 52.330, "lon": 46.370, "elev": 292}, +"34122": {"id": "UUOO", "name": "VORONEZ", "state": "MS RS", "lat": 51.700, "lon": 39.170, "elev": 154}, +"34172": {"id": "UWSS", "name": "SARATOV", "state": "MS RS", "lat": 51.570, "lon": 46.030, "elev": 156}, +"34247": {"id": "", "name": "KALAC", "state": "MS RS", "lat": 50.420, "lon": 41.050, "elev": 93}, +"34300": {"id": "UKHH", "name": "KHARKOV", "state": "KI UR", "lat": 49.970, "lon": 36.130, "elev": 155}, +"34467": {"id": "", "name": "VOLOGRAD", "state": "TB RS", "lat": 48.780, "lon": 44.330, "elev": 141}, +"34560": {"id": "URWW", "name": "VOLGOGRAD", "state": "TB RS", "lat": 48.680, "lon": 44.350, "elev": 145}, +"34731": {"id": "URRR", "name": "ROSTOV-NA-DONU", "state": "TB RS", "lat": 47.250, "lon": 39.820, "elev": 77}, +"34737": {"id": "", "name": "KUSCEVSKAJA", "state": "TB RS", "lat": 46.570, "lon": 39.630, "elev": 14}, +"34772": {"id": "", "name": "YUSTA", "state": "TB RS", "lat": 47.120, "lon": 46.300, "elev": 2}, +"34838": {"id": "", "name": "TIHORECK", "state": "TB RS", "lat": 45.850, "lon": 40.080, "elev": 79}, +"34858": {"id": "", "name": "DIVNOE", "state": "TB RS", "lat": 46.920, "lon": 45.350, "elev": 77}, +"34880": {"id": "", "name": "ASTRAHAN", "state": "TB RS", "lat": 46.270, "lon": 48.030, "elev": -22}, +"34882": {"id": "", "name": "ASTRAHAN'", "state": "TB RS", "lat": 46.280, "lon": 47.980, "elev": -17}, +"35108": {"id": "UARR", "name": "URALSK", "state": "AL RA", "lat": 51.250, "lon": 51.400, "elev": 36}, +"35121": {"id": "UWOO", "name": "ORENBURG", "state": "AL RA", "lat": 51.780, "lon": 55.220, "elev": 109}, +"35225": {"id": "", "name": "MARTUK", "state": "AL RA", "lat": 50.750, "lon": 56.530, "elev": 178}, +"35229": {"id": "UATT", "name": "AKTJUBINSK", "state": "AL RA", "lat": 50.300, "lon": 57.230, "elev": 227}, +"35361": {"id": "", "name": "AMANGELDY", "state": "AL RA", "lat": 50.130, "lon": 65.230, "elev": 142}, +"35394": {"id": "", "name": "KARAGANDA", "state": "AL RA", "lat": 49.800, "lon": 73.130, "elev": 555}, +"35671": {"id": "", "name": "DZEZKAZGAN", "state": "AL RA", "lat": 47.800, "lon": 67.720, "elev": 345}, +"35700": {"id": "", "name": "GUR'EV", "state": "AL RA", "lat": 47.020, "lon": 51.850, "elev": -15}, +"35746": {"id": "UATA", "name": "ARALSK", "state": "AL RA", "lat": 46.780, "lon": 61.670, "elev": 56}, +"35796": {"id": "", "name": "BALHAS", "state": "AL RA", "lat": 46.900, "lon": 75.000, "elev": 352}, +"36003": {"id": "", "name": "PAVLODAR", "state": "AL RA", "lat": 52.280, "lon": 76.950, "elev": 123}, +"36096": {"id": "", "name": "KYZYL", "state": "NO RA", "lat": 51.700, "lon": 94.450, "elev": 629}, +"36177": {"id": "", "name": "SEMIPALATINSK", "state": "AL RA", "lat": 50.350, "lon": 80.250, "elev": 196}, +"36770": {"id": "", "name": "MULALY", "state": "AL RA", "lat": 45.450, "lon": 78.230, "elev": 564}, +"36859": {"id": "", "name": "PANFILOV", "state": "AL RA", "lat": 44.170, "lon": 80.070, "elev": 640}, +"36870": {"id": "UAAA", "name": "ALMA-ATA", "state": "AL RA", "lat": 43.230, "lon": 76.930, "elev": 847}, +"36872": {"id": "", "name": "ALMATY", "state": "AL RA", "lat": 43.350, "lon": 77.000, "elev": 662}, +"36974": {"id": "", "name": "NARYN (MTN STN)", "state": "TA RA", "lat": 41.430, "lon": 76.000, "elev": 2041}, +"37011": {"id": "", "name": "TUAPSE", "state": "TB RS", "lat": 44.100, "lon": 39.030, "elev": 52}, +"37018": {"id": "", "name": "TUAPSE (STREAM)", "state": "TB RS", "lat": 44.100, "lon": 39.070, "elev": 41}, +"37054": {"id": "URMM", "name": "MINERALYNE VODY", "state": "TB RS", "lat": 44.220, "lon": 43.100, "elev": 314}, +"37055": {"id": "URMM", "name": "MINERALYNE VODY'", "state": "TB RS", "lat": 44.220, "lon": 43.100, "elev": 321}, +"37099": {"id": "", "name": "RAZDOL'NOYE SOCI", "state": "TB RS", "lat": 43.580, "lon": 39.760, "elev": 57}, +"37259": {"id": "", "name": "MAHACHKALA", "state": "TB RS", "lat": 43.020, "lon": 47.480, "elev": 19}, +"37260": {"id": "", "name": "SUHUMI", "state": "TB RS", "lat": 42.870, "lon": 41.130, "elev": 13}, +"37395": {"id": "", "name": "KUTAISI", "state": "TB RS", "lat": 42.270, "lon": 42.630, "elev": 116}, +"37472": {"id": "", "name": "MAHACKALA", "state": "TB RS", "lat": 43.020, "lon": 47.430, "elev": 6}, +"37484": {"id": "", "name": "BATUMI (BAY)", "state": "TB RS", "lat": 41.650, "lon": 41.630, "elev": 6}, +"37507": {"id": "", "name": "ADIGENI", "state": "TB RS", "lat": 41.700, "lon": 42.700, "elev": 1185}, +"37545": {"id": "", "name": "TBILISI", "state": "TB RS", "lat": 41.700, "lon": 44.750, "elev": 403}, +"37549": {"id": "UGGG", "name": "TBILISI/NOVOALEXEYE", "state": "TB RS", "lat": 41.680, "lon": 44.950, "elev": 467}, +"37750": {"id": "", "name": "ISMAILLY", "state": "TB RS", "lat": 40.780, "lon": 48.130, "elev": 550}, +"37789": {"id": "UGEE", "name": "YEREVAN/ZVARTNOTS", "state": "TB RS", "lat": 40.130, "lon": 44.470, "elev": 890}, +"37860": {"id": "", "name": "MASHTAGA", "state": "TB RS", "lat": 40.530, "lon": 50.000, "elev": 28}, +"37936": {"id": "", "name": "NAKHICHEVAN", "state": "TB RS", "lat": 39.200, "lon": 45.420, "elev": 875}, +"37985": {"id": "", "name": "LENKORAN'", "state": "TB RS", "lat": 38.730, "lon": 48.830, "elev": 11}, +"37989": {"id": "", "name": "ASTRARA", "state": "TB RS", "lat": 38.450, "lon": 48.880, "elev": 21}, +"38062": {"id": "UAOO", "name": "KZYL-ORDA", "state": "AL RA", "lat": 44.770, "lon": 65.530, "elev": 129}, +"38064": {"id": "", "name": "KYSYLORDA", "state": "AL RA", "lat": 44.770, "lon": 65.520, "elev": 133}, +"38341": {"id": "", "name": "DZAMBUL", "state": "AL RA", "lat": 42.850, "lon": 71.380, "elev": 653}, +"38343": {"id": "", "name": "LUGOVAJA", "state": "AL RA", "lat": 45.950, "lon": 72.750, "elev": 683}, +"38352": {"id": "", "name": "BELOVODSKOE", "state": "TA RA", "lat": 42.850, "lon": 74.100, "elev": 726}, +"38353": {"id": "UAFF", "name": "FRUNZE", "state": "TA RA", "lat": 42.850, "lon": 74.530, "elev": 760}, +"38358": {"id": "", "name": "SUSAMYR", "state": "TA RA", "lat": 42.150, "lon": 73.980, "elev": 2092}, +"38388": {"id": "", "name": "EKEDGE", "state": "TA RA", "lat": 41.030, "lon": 57.770, "elev": 62}, +"38392": {"id": "", "name": "DASHKHOVUZ", "state": "TA RA", "lat": 41.830, "lon": 59.980, "elev": 88}, +"38413": {"id": "UTSM", "name": "TAMDY-BULAK", "state": "TA RA", "lat": 41.730, "lon": 64.620, "elev": 238}, +"38457": {"id": "UTTT", "name": "TASKENT/YUZNI", "state": "TA RA", "lat": 41.270, "lon": 69.270, "elev": 489}, +"38507": {"id": "", "name": "KRASNOVODSK", "state": "TA RA", "lat": 40.030, "lon": 52.980, "elev": 89}, +"38606": {"id": "", "name": "KOKAND", "state": "TA RA", "lat": 40.550, "lon": 70.950, "elev": 405}, +"38613": {"id": "", "name": "DZALAL-ABAD", "state": "TA RA", "lat": 40.920, "lon": 72.950, "elev": 765}, +"38687": {"id": "", "name": "CARDZOU", "state": "TA RA", "lat": 39.080, "lon": 63.600, "elev": 193}, +"38750": {"id": "", "name": "GASAN-KULI", "state": "TA RA", "lat": 37.470, "lon": 53.970, "elev": -23}, +"38755": {"id": "", "name": "BYGDAILI", "state": "TA RA", "lat": 38.530, "lon": 54.300, "elev": 1}, +"38836": {"id": "UTDD", "name": "DUSANBE", "state": "TA RA", "lat": 38.550, "lon": 68.780, "elev": 803}, +"38878": {"id": "", "name": "MURGAB", "state": "TA RA", "lat": 38.170, "lon": 73.980, "elev": 3576}, +"38880": {"id": "", "name": "ASHABAD", "state": "TA RA", "lat": 37.970, "lon": 58.330, "elev": 210}, +"38927": {"id": "UTST", "name": "TERMEZ", "state": "TA RA", "lat": 37.230, "lon": 67.270, "elev": 302}, +"38954": {"id": "", "name": "KHOROG", "state": "TA RA", "lat": 37.500, "lon": 71.500, "elev": 2080}, +"38989": {"id": "", "name": "TAHTA-BAZAR", "state": "TA RA", "lat": 35.920, "lon": 62.920, "elev": 349}, +"40007": {"id": "OSAP", "name": "ALEPPO (CIV/MIL)", "state": "SY", "lat": 36.180, "lon": 37.220, "elev": 393}, +"40039": {"id": "", "name": "RAQQA", "state": "SY", "lat": 35.930, "lon": 39.020, "elev": 245}, +"40041": {"id": "", "name": "AL-RASTAN", "state": "SY", "lat": 34.930, "lon": 36.730, "elev": 450}, +"40080": {"id": "OSDI", "name": "DAMASCUS (CIV/MIL)", "state": "SY", "lat": 33.420, "lon": 36.520, "elev": 609}, +"40100": {"id": "OLBA", "name": "BEIRUT (CIV/MIL)", "state": "LB", "lat": 33.820, "lon": 35.480, "elev": 19}, +"40155": {"id": "LLHA", "name": "HAIFA (ISR-AFB)", "state": "IS", "lat": 32.470, "lon": 35.010, "elev": 8}, +"40165": {"id": "", "name": "HAIFA/R.DAVID", "state": "IS", "lat": 32.670, "lon": 35.180, "elev": 51}, +"40179": {"id": "", "name": "BET DAGAN", "state": "IS", "lat": 32.000, "lon": 34.820, "elev": 30}, +"40180": {"id": "LLBG", "name": "BEN-GURION (CIV/MIL)", "state": "IS", "lat": 32.000, "lon": 34.530, "elev": 49}, +"40186": {"id": "", "name": "ASHDOD NORTH", "state": "IS", "lat": 31.870, "lon": 34.680, "elev": 20}, +"40191": {"id": "LLBS", "name": "BEER-SHEVA/TEYMA", "state": "IS", "lat": 31.170, "lon": 34.430, "elev": 200}, +"40198": {"id": "LLOV", "name": "OVDA (ISR-AFB/CIV)", "state": "IS", "lat": 29.560, "lon": 34.560, "elev": 432}, +"40199": {"id": "LLET", "name": "ELAT/J. HOZMAN", "state": "IS", "lat": 29.550, "lon": 34.950, "elev": 12}, +"40265": {"id": "OJMF", "name": "MAFRAQ (JOR-AFB)", "state": "JD", "lat": 32.370, "lon": 36.270, "elev": 687}, +"40290": {"id": "LLJR", "name": "JERUSALEM/ATAROT", "state": "IS", "lat": 31.520, "lon": 35.130, "elev": 759}, +"40373": {"id": "OEPA", "name": "HAFR AL-BATIN ARPT", "state": "SD", "lat": 28.330, "lon": 46.120, "elev": 355}, +"40375": {"id": "OETB", "name": "TABUK (SAUD-AFB)", "state": "SD", "lat": 28.370, "lon": 36.580, "elev": 770}, +"40376": {"id": "", "name": "TAYMA (AUT)", "state": "SD", "lat": 27.600, "lon": 38.600, "elev": 860}, +"40394": {"id": "OEHL", "name": "HAIL", "state": "SD", "lat": 27.520, "lon": 41.730, "elev": 1013}, +"40400": {"id": "", "name": "EL WEJH", "state": "SD", "lat": 26.230, "lon": 36.430, "elev": 22}, +"40416": {"id": "OEDR", "name": "DHAHRAN INTL ARPT", "state": "SD", "lat": 26.270, "lon": 50.150, "elev": 17}, +"40417": {"id": "OEDF", "name": "KING FAHAD INTERNATIONAL", "state": "SD", "lat": 26.450, "lon": 49.820, "elev": 12}, +"40420": {"id": "OEAH", "name": "AL AHSA", "state": "SD", "lat": 25.280, "lon": 49.480, "elev": 172}, +"40421": {"id": "", "name": "HUFUF (AUT)", "state": "SD", "lat": 25.300, "lon": 49.700, "elev": 151}, +"40430": {"id": "OEMA", "name": "MADINAH INTL ARPT", "state": "SD", "lat": 24.550, "lon": 39.720, "elev": 631}, +"40435": {"id": "OEDW", "name": "DAWADMI", "state": "SD", "lat": 24.500, "lon": 44.400, "elev": 990}, +"40437": {"id": "OERK", "name": "RIYADH/KING KHALID", "state": "SD", "lat": 24.930, "lon": 46.720, "elev": 612}, +"40438": {"id": "OERY", "name": "RIYADH (SAUD-AFB)", "state": "SD", "lat": 24.720, "lon": 46.720, "elev": 612}, +"40495": {"id": "", "name": "SULAYEL/ASSULAYYIL", "state": "SD", "lat": 20.470, "lon": 45.670, "elev": 617}, +"40582": {"id": "OKBK", "name": "KUWAIT INTERNATIONAL", "state": "KW", "lat": 29.220, "lon": 47.980, "elev": 55}, +"40585": {"id": "", "name": "AL SHUAIBA", "state": "KW", "lat": 29.030, "lon": 48.150, "elev": 6}, +"40594": {"id": "", "name": "AL-NWAISEEB", "state": "KW", "lat": 28.500, "lon": 48.470, "elev": 3}, +"40608": {"id": "", "name": "MOSUL", "state": "IQ", "lat": 36.320, "lon": 43.150, "elev": 222}, +"40642": {"id": "", "name": "RUTBA", "state": "IQ", "lat": 33.030, "lon": 40.280, "elev": 615}, +"40648": {"id": "ORBH", "name": "HABBANIYA (IRQ-AFB)", "state": "IQ", "lat": 33.370, "lon": 43.570, "elev": 45}, +"40650": {"id": "ORBB", "name": "BAGDAD/SIRSENK", "state": "IQ", "lat": 33.230, "lon": 44.230, "elev": 34}, +"40676": {"id": "", "name": "NASIRIYA", "state": "IQ", "lat": 31.080, "lon": 46.230, "elev": 3}, +"40688": {"id": "ORMS", "name": "SHAIBAH/BASRAH", "state": "IQ", "lat": 30.420, "lon": 47.650, "elev": 19}, +"40698": {"id": "", "name": "BASRA", "state": "IQ", "lat": 30.570, "lon": 47.780, "elev": 2}, +"40701": {"id": "", "name": "MAKKO", "state": "IR", "lat": 39.330, "lon": 44.430, "elev": 1470}, +"40706": {"id": "OITT", "name": "TABRIZ(IRAN-AB/CIV)", "state": "IR", "lat": 38.080, "lon": 46.280, "elev": 1361}, +"40710": {"id": "", "name": "SARAB", "state": "IR", "lat": 37.930, "lon": 47.530, "elev": 1500}, +"40745": {"id": "OIMM", "name": "MASHHAD (AFB/CIV)", "state": "IR", "lat": 36.270, "lon": 59.630, "elev": 980}, +"40747": {"id": "OICS", "name": "SANANDAJ", "state": "IR", "lat": 35.330, "lon": 47.000, "elev": 1373}, +"40754": {"id": "OIII", "name": "TEHRAN/MEHRABAD AFB", "state": "IR", "lat": 35.680, "lon": 51.350, "elev": 1191}, +"40765": {"id": "OICG", "name": "GHASRE-SHIRIN", "state": "IR", "lat": 34.520, "lon": 45.580, "elev": 378}, +"40766": {"id": "OICC", "name": "KERMANSHAH/BAKTARAN", "state": "IR", "lat": 34.270, "lon": 47.120, "elev": 1322}, +"40780": {"id": "", "name": "ILAM", "state": "IR", "lat": 33.630, "lon": 46.420, "elev": 1363}, +"40789": {"id": "", "name": "KHOR", "state": "IR", "lat": 33.780, "lon": 55.080, "elev": 921}, +"40791": {"id": "OIMT", "name": "TABAS", "state": "IR", "lat": 33.600, "lon": 56.920, "elev": 711}, +"40794": {"id": "", "name": "SAFI/ABAD DEZFU", "state": "IR", "lat": 32.270, "lon": 48.430, "elev": 82}, +"40795": {"id": "", "name": "VAHDATI AFB", "state": "IR", "lat": 32.430, "lon": 48.400, "elev": 152}, +"40800": {"id": "OIFM", "name": "ESFAHAN (CIV/AFB)", "state": "IR", "lat": 32.620, "lon": 51.670, "elev": 1590}, +"40809": {"id": "OIMB", "name": "BIRJAND", "state": "IR", "lat": 32.870, "lon": 59.200, "elev": 1491}, +"40811": {"id": "OIAW", "name": "AHWAZ", "state": "IR", "lat": 31.200, "lon": 48.400, "elev": 22}, +"40821": {"id": "OIYY", "name": "YAZD", "state": "IR", "lat": 31.900, "lon": 54.280, "elev": 1230}, +"40841": {"id": "OIKK", "name": "KERMAN", "state": "IR", "lat": 30.250, "lon": 56.970, "elev": 1754}, +"40848": {"id": "OISS", "name": "SHIRAZ (CIV/AFB)", "state": "IR", "lat": 29.530, "lon": 52.580, "elev": 1491}, +"40851": {"id": "", "name": "SIRJAN", "state": "IR", "lat": 29.470, "lon": 55.720, "elev": 1739}, +"40856": {"id": "OIZH", "name": "ZAHEDAN/INTL", "state": "IR", "lat": 29.470, "lon": 60.880, "elev": 1370}, +"40875": {"id": "OIKB", "name": "BANDAR ABBAS INTL", "state": "IR", "lat": 27.220, "lon": 56.370, "elev": 10}, +"40879": {"id": "OIZI", "name": "IRANSHAHR", "state": "IR", "lat": 27.200, "lon": 60.700, "elev": 591}, +"40911": {"id": "", "name": "MAZAR-I-SHARIF", "state": "AF", "lat": 42.700, "lon": 67.200, "elev": 378}, +"40938": {"id": "", "name": "HERAT", "state": "AF", "lat": 34.220, "lon": 62.220, "elev": 964}, +"40948": {"id": "KBLO", "name": "KABUL AIRPORT", "state": "AF", "lat": 34.330, "lon": 69.130, "elev": 1791}, +"40990": {"id": "", "name": "KANDAHAR AIRPORT", "state": "AF", "lat": 31.500, "lon": 65.850, "elev": 1010}, +"41014": {"id": "", "name": "OBAYLAH (AUT)", "state": "SD", "lat": 22.220, "lon": 50.880, "elev": 588}, +"41024": {"id": "OEJN", "name": "JEDDAH/ABDUL AZIZ", "state": "SD", "lat": 21.670, "lon": 39.150, "elev": 12}, +"41112": {"id": "OEAB", "name": "ABHA", "state": "SD", "lat": 18.230, "lon": 42.650, "elev": 2084}, +"41114": {"id": "OEKM", "name": "KHAMIS MUSHAIT AFB", "state": "SD", "lat": 18.300, "lon": 42.800, "elev": 2054}, +"41141": {"id": "", "name": "GIZAN (AUT)", "state": "SD", "lat": 16.900, "lon": 42.500, "elev": 6}, +"41170": {"id": "OTBD", "name": "DOHA INTL AIRPORT", "state": "QT", "lat": 25.250, "lon": 51.570, "elev": 10}, +"41181": {"id": "OMRK", "name": "RAS AL KHAIMAH", "state": "ER", "lat": 25.620, "lon": 55.930, "elev": 31}, +"41217": {"id": "OMAA", "name": "ABU DHABI INTL", "state": "ER", "lat": 24.430, "lon": 54.650, "elev": 27}, +"41218": {"id": "OMAL", "name": "AL AIN INTL AIRPORT", "state": "ER", "lat": 24.250, "lon": 55.600, "elev": 262}, +"41254": {"id": "OOSQ", "name": "SAIQ", "state": "OM", "lat": 23.070, "lon": 57.650, "elev": 1755}, +"41256": {"id": "OOMS", "name": "SEEB INTL/MUSCAT", "state": "OM", "lat": 23.580, "lon": 58.280, "elev": 8}, +"41288": {"id": "", "name": "MASIRAH", "state": "OM", "lat": 20.680, "lon": 58.900, "elev": 21}, +"41316": {"id": "OOSA", "name": "SALALAH", "state": "OM", "lat": 17.030, "lon": 54.080, "elev": 20}, +"41344": {"id": "", "name": "SANA'A (CIV/MIL)", "state": "YE", "lat": 15.520, "lon": 44.180, "elev": 2190}, +"41404": {"id": "OYSN", "name": "SANA'A", "state": "YE", "lat": 15.380, "lon": 44.200, "elev": 2190}, +"41414": {"id": "", "name": "THAMUD", "state": "YE", "lat": 17.370, "lon": 50.950, "elev": 610}, +"41416": {"id": "", "name": "KAMARAN", "state": "YE", "lat": 15.370, "lon": 4.420, "elev": 15}, +"41437": {"id": "OYAT", "name": "ATAQ", "state": "DY", "lat": 14.550, "lon": 46.830, "elev": 1067}, +"41480": {"id": "OYAA", "name": "ADEN INTL ARPT", "state": "DY", "lat": 12.670, "lon": 45.030, "elev": 3}, +"41506": {"id": "", "name": "CHITRAL", "state": "PK", "lat": 35.850, "lon": 71.830, "elev": 1494}, +"41508": {"id": "", "name": "DIR", "state": "PK", "lat": 35.200, "lon": 71.850, "elev": 1370}, +"41515": {"id": "", "name": "DROSH", "state": "PK", "lat": 35.400, "lon": 71.780, "elev": 1480}, +"41529": {"id": "", "name": "PESHAWAR", "state": "PK", "lat": 34.030, "lon": 71.930, "elev": 329}, +"41530": {"id": "OPPS", "name": "PESHAWAR (CIV/MIL)", "state": "PK", "lat": 34.020, "lon": 71.580, "elev": 360}, +"41531": {"id": "", "name": "BATTAL", "state": "PK", "lat": 34.580, "lon": 73.150, "elev": 1676}, +"41560": {"id": "", "name": "PARACHINAR", "state": "PK", "lat": 33.870, "lon": 70.080, "elev": 1728}, +"41571": {"id": "", "name": "CHAKLALA", "state": "PK", "lat": 33.620, "lon": 73.100, "elev": 509}, +"41593": {"id": "", "name": "SARGODHA (PAK-AFB)", "state": "PK", "lat": 32.050, "lon": 72.670, "elev": 187}, +"41594": {"id": "OPSR", "name": "SARGODHA (PAK-AFB)", "state": "PK", "lat": 32.050, "lon": 72.670, "elev": 188}, +"41598": {"id": "", "name": "JHELUM", "state": "PK", "lat": 32.930, "lon": 73.720, "elev": 233}, +"41620": {"id": "", "name": "FORT SANDEMAN", "state": "PK", "lat": 31.350, "lon": 69.470, "elev": 1405}, +"41624": {"id": "", "name": "DERA ISMAIL KHAN", "state": "PK", "lat": 31.820, "lon": 70.920, "elev": 174}, +"41640": {"id": "OPLH", "name": "LAHORE/WALTON", "state": "PK", "lat": 31.550, "lon": 74.330, "elev": 215}, +"41661": {"id": "", "name": "QUETTA BREWERY", "state": "PK", "lat": 30.270, "lon": 66.920, "elev": 1621}, +"41675": {"id": "OPMT", "name": "MULTAN", "state": "PK", "lat": 30.200, "lon": 71.430, "elev": 123}, +"41685": {"id": "", "name": "BARKHAN", "state": "PK", "lat": 29.880, "lon": 69.720, "elev": 1097}, +"41710": {"id": "", "name": "NOKKUNDI", "state": "PK", "lat": 28.820, "lon": 62.750, "elev": 683}, +"41712": {"id": "", "name": "DALBANDIN", "state": "PK", "lat": 28.880, "lon": 64.400, "elev": 849}, +"41715": {"id": "", "name": "JACOBABAD", "state": "PK", "lat": 28.300, "lon": 68.470, "elev": 56}, +"41718": {"id": "", "name": "KHABPUR", "state": "PK", "lat": 28.650, "lon": 70.680, "elev": 87}, +"41739": {"id": "", "name": "PANJGUR", "state": "PK", "lat": 26.970, "lon": 64.100, "elev": 981}, +"41744": {"id": "", "name": "KHUZDAR", "state": "PK", "lat": 27.830, "lon": 66.630, "elev": 1225}, +"41749": {"id": "", "name": "NAWABSHAH", "state": "PK", "lat": 26.250, "lon": 68.370, "elev": 37}, +"41756": {"id": "", "name": "JIWANI", "state": "PK", "lat": 25.040, "lon": 61.480, "elev": 57}, +"41768": {"id": "", "name": "CHHOR", "state": "PK", "lat": 25.520, "lon": 69.780, "elev": 6}, +"41780": {"id": "OPKC", "name": "KARACHI INTL ARPT", "state": "PK", "lat": 24.900, "lon": 67.130, "elev": 22}, +"41859": {"id": "", "name": "RANGPUR", "state": "BW", "lat": 25.730, "lon": 89.230, "elev": 34}, +"41883": {"id": "", "name": "BOGRA", "state": "BW", "lat": 24.850, "lon": 89.370, "elev": 20}, +"41891": {"id": "", "name": "SYLHET", "state": "BW", "lat": 24.900, "lon": 91.880, "elev": 35}, +"41895": {"id": "VGRJ", "name": "RAJSHAHI", "state": "BW", "lat": 24.370, "lon": 88.700, "elev": 20}, +"41907": {"id": "VGIS", "name": "ISHURDI", "state": "BW", "lat": 24.130, "lon": 89.050, "elev": 14}, +"41909": {"id": "", "name": "TANGAIL", "state": "BW", "lat": 24.250, "lon": 89.930, "elev": 56}, +"41923": {"id": "", "name": "COMILLA", "state": "BW", "lat": 23.430, "lon": 91.180, "elev": 9}, +"41929": {"id": "", "name": "FARODPUR", "state": "BW", "lat": 23.600, "lon": 89.850, "elev": 9}, +"41936": {"id": "", "name": "JESSORE", "state": "BW", "lat": 23.110, "lon": 89.100, "elev": 7}, +"41943": {"id": "", "name": "FENI", "state": "BW", "lat": 23.030, "lon": 91.420, "elev": 8}, +"41950": {"id": "", "name": "BARISAL", "state": "BW", "lat": 22.750, "lon": 90.370, "elev": 4}, +"41977": {"id": "", "name": "CHITTAGONG/AMBAGAN", "state": "BW", "lat": 22.350, "lon": 91.820, "elev": 34}, +"41992": {"id": "", "name": "COX'S BAZAR", "state": "BW", "lat": 21.430, "lon": 91.930, "elev": 4}, +"42027": {"id": "", "name": "SRINAGAR", "state": "IN", "lat": 34.080, "lon": 74.830, "elev": 1587}, +"42056": {"id": "", "name": "JAMMU", "state": "IN", "lat": 32.670, "lon": 74.830, "elev": 323}, +"42079": {"id": "", "name": "SUNDERNAGAR", "state": "IN", "lat": 31.530, "lon": 76.900, "elev": 875}, +"42097": {"id": "", "name": "BHATINDA", "state": "IN", "lat": 30.170, "lon": 74.580, "elev": 211}, +"42101": {"id": "", "name": "PATIALA", "state": "IN", "lat": 30.330, "lon": 76.470, "elev": 251}, +"42103": {"id": "", "name": "AMBALA", "state": "IN", "lat": 30.380, "lon": 76.770, "elev": 272}, +"42107": {"id": "", "name": "DHARMPUR", "state": "IN", "lat": 30.900, "lon": 77.020, "elev": 1986}, +"42111": {"id": "", "name": "DEHRA DUN", "state": "IN", "lat": 30.320, "lon": 78.030, "elev": 682}, +"42112": {"id": "", "name": "MUSSOORIE", "state": "IN", "lat": 30.450, "lon": 78.080, "elev": 2042}, +"42114": {"id": "", "name": "TEHRI", "state": "IN", "lat": 30.400, "lon": 78.480, "elev": 29}, +"42117": {"id": "", "name": "CHAMOLI", "state": "IN", "lat": 30.400, "lon": 79.330, "elev": 20}, +"42123": {"id": "", "name": "GANGANAGAR", "state": "IN", "lat": 29.920, "lon": 73.920, "elev": 177}, +"42131": {"id": "VIHR", "name": "HISSAR", "state": "IN", "lat": 29.170, "lon": 75.730, "elev": 221}, +"42147": {"id": "", "name": "MUKTESHWAR KUMA", "state": "IN", "lat": 29.470, "lon": 79.650, "elev": 2311}, +"42165": {"id": "", "name": "BIKANER", "state": "IN", "lat": 28.000, "lon": 73.300, "elev": 224}, +"42170": {"id": "", "name": "CHURU", "state": "IN", "lat": 28.250, "lon": 74.920, "elev": 286}, +"42181": {"id": "VIDP", "name": "INDIRA GANDHI", "state": "IN", "lat": 28.570, "lon": 77.120, "elev": 233}, +"42182": {"id": "VIDD", "name": "DELHI/SAFDARJUNG", "state": "IN", "lat": 28.580, "lon": 77.200, "elev": 216}, +"42187": {"id": "", "name": "MORADABAD", "state": "IN", "lat": 28.830, "lon": 78.750, "elev": 202}, +"42188": {"id": "", "name": "RAMPUR", "state": "IN", "lat": 28.750, "lon": 79.000, "elev": 191}, +"42189": {"id": "", "name": "BAREILLY", "state": "IN", "lat": 28.360, "lon": 79.400, "elev": 173}, +"42260": {"id": "", "name": "AGRA/VIAG", "state": "IN", "lat": 27.160, "lon": 77.960, "elev": 168}, +"42273": {"id": "", "name": "BAHRAICH", "state": "IN", "lat": 27.570, "lon": 81.600, "elev": 124}, +"42282": {"id": "", "name": "NAUTANWA", "state": "IN", "lat": 27.430, "lon": 83.420, "elev": 98}, +"42299": {"id": "", "name": "GANGTOK", "state": "IN", "lat": 27.330, "lon": 88.620, "elev": 1756}, +"42314": {"id": "VEMN", "name": "DIBRUGARH/MOHANBARI", "state": "IN", "lat": 27.480, "lon": 95.020, "elev": 111}, +"42328": {"id": "", "name": "JAISALMER", "state": "IN", "lat": 26.900, "lon": 70.920, "elev": 257}, +"42339": {"id": "VIJO", "name": "JODHPUR (IN-AFB)", "state": "IN", "lat": 26.300, "lon": 73.020, "elev": 224}, +"42348": {"id": "", "name": "JAIPUR", "state": "IN", "lat": 26.820, "lon": 75.800, "elev": 390}, +"42361": {"id": "VIGR", "name": "GWALIOR (IN-AFB)", "state": "IN", "lat": 26.230, "lon": 78.250, "elev": 207}, +"42364": {"id": "", "name": "ETAWAH", "state": "IN", "lat": 26.450, "lon": 79.020, "elev": 151}, +"42367": {"id": "VICX", "name": "KANPUR/CHAKERI", "state": "IN", "lat": 26.400, "lon": 80.400, "elev": 126}, +"42368": {"id": "", "name": "LUCKNOW", "state": "IN", "lat": 26.870, "lon": 80.930, "elev": 110}, +"42369": {"id": "VILK", "name": "LUCKNOW/AMAUSI", "state": "IN", "lat": 26.750, "lon": 80.880, "elev": 128}, +"42375": {"id": "", "name": "SULTANPUR", "state": "IN", "lat": 26.250, "lon": 82.000, "elev": 97}, +"42379": {"id": "VEGK", "name": "GORAKHPUR (IN-AFB)", "state": "IN", "lat": 26.750, "lon": 83.370, "elev": 77}, +"42393": {"id": "", "name": "SUPAUL", "state": "IN", "lat": 26.130, "lon": 86.580, "elev": 50}, +"42397": {"id": "", "name": "SILIGURI", "state": "IN", "lat": 26.670, "lon": 88.370, "elev": 123}, +"42398": {"id": "VEBD", "name": "BAGHDOGRA (IN-AFB)", "state": "IN", "lat": 26.630, "lon": 88.320, "elev": 131}, +"42399": {"id": "", "name": "JALPAIGURI", "state": "IN", "lat": 26.530, "lon": 88.720, "elev": 81}, +"42409": {"id": "", "name": "TANGLA", "state": "IN", "lat": 26.650, "lon": 91.920, "elev": 65}, +"42410": {"id": "VEGT", "name": "GAUHATI (IN-AFB)", "state": "IN", "lat": 26.100, "lon": 91.580, "elev": 54}, +"42420": {"id": "", "name": "GOLAGHAT", "state": "IN", "lat": 26.520, "lon": 93.980, "elev": 95}, +"42435": {"id": "", "name": "BARMER", "state": "IN", "lat": 25.750, "lon": 71.380, "elev": 193}, +"42447": {"id": "", "name": "BHILWARA", "state": "IN", "lat": 25.350, "lon": 74.630, "elev": 425}, +"42452": {"id": "VIKO", "name": "KOTA AERODROME", "state": "IN", "lat": 25.150, "lon": 75.850, "elev": 274}, +"42474": {"id": "", "name": "ALLAHABAD", "state": "IN", "lat": 25.450, "lon": 81.730, "elev": 98}, +"42475": {"id": "", "name": "ALLAHABAD/BAMRAULI", "state": "IN", "lat": 25.450, "lon": 81.730, "elev": 98}, +"42492": {"id": "VEPT", "name": "PATNA", "state": "IN", "lat": 25.600, "lon": 85.100, "elev": 60}, +"42498": {"id": "", "name": "BHAGALPUR", "state": "IN", "lat": 25.230, "lon": 86.950, "elev": 49}, +"42539": {"id": "", "name": "DEESA", "state": "IN", "lat": 24.200, "lon": 72.200, "elev": 136}, +"42542": {"id": "", "name": "UDAIPUR", "state": "IN", "lat": 24.610, "lon": 73.880, "elev": 514}, +"42543": {"id": "", "name": "UDAIPUR", "state": "IN", "lat": 24.580, "lon": 73.700, "elev": 582}, +"42571": {"id": "VIST", "name": "SATNA", "state": "IN", "lat": 24.570, "lon": 80.830, "elev": 317}, +"42591": {"id": "", "name": "GAVA", "state": "IN", "lat": 24.750, "lon": 84.950, "elev": 116}, +"42623": {"id": "", "name": "IMPHAL", "state": "IN", "lat": 24.670, "lon": 93.900, "elev": 774}, +"42634": {"id": "", "name": "BHUJ", "state": "IN", "lat": 23.250, "lon": 69.670, "elev": 80}, +"42647": {"id": "VAAH", "name": "AHMADABAD", "state": "IN", "lat": 23.070, "lon": 72.630, "elev": 55}, +"42667": {"id": "VABP", "name": "BHOPAL/BAIRAGARH", "state": "IN", "lat": 23.280, "lon": 77.350, "elev": 523}, +"42675": {"id": "", "name": "JABALPUR", "state": "IN", "lat": 23.160, "lon": 79.950, "elev": 393}, +"42701": {"id": "VERC", "name": "RANCHI", "state": "IN", "lat": 23.320, "lon": 85.320, "elev": 652}, +"42706": {"id": "", "name": "BANKURA", "state": "IN", "lat": 23.380, "lon": 87.080, "elev": 99}, +"42709": {"id": "", "name": "BURDWAN", "state": "IN", "lat": 23.230, "lon": 87.850, "elev": 30}, +"42724": {"id": "VEAT", "name": "AGARTALA", "state": "IN", "lat": 23.880, "lon": 91.250, "elev": 16}, +"42767": {"id": "", "name": "PACHMARHI", "state": "IN", "lat": 22.470, "lon": 78.430, "elev": 1061}, +"42798": {"id": "VEJS", "name": "JAMSHEDPUR", "state": "IN", "lat": 22.810, "lon": 86.180, "elev": 142}, +"42805": {"id": "", "name": "ULUBERIA", "state": "IN", "lat": 22.500, "lon": 87.950, "elev": 57}, +"42807": {"id": "", "name": "CALCUTTA AIRPORT", "state": "IN", "lat": 22.530, "lon": 88.330, "elev": 5}, +"42809": {"id": "VECC", "name": "CALCUTTA/DUM DUM", "state": "IN", "lat": 22.650, "lon": 88.450, "elev": 6}, +"42830": {"id": "", "name": "PORBANDAR", "state": "IN", "lat": 21.650, "lon": 69.670, "elev": 5}, +"42867": {"id": "VANP", "name": "NAGPUR SONEGAON AFB", "state": "IN", "lat": 21.100, "lon": 79.050, "elev": 310}, +"42874": {"id": "", "name": "PBO RAIPUR", "state": "IN", "lat": 21.220, "lon": 81.670, "elev": 298}, +"42886": {"id": "", "name": "JHARSUGUDA", "state": "IN", "lat": 21.910, "lon": 84.080, "elev": 230}, +"42895": {"id": "", "name": "BALASORE", "state": "IN", "lat": 21.500, "lon": 86.930, "elev": 20}, +"42909": {"id": "", "name": "VERAVAL", "state": "IN", "lat": 20.900, "lon": 70.370, "elev": 8}, +"42931": {"id": "", "name": "BULDANA", "state": "IN", "lat": 20.530, "lon": 76.230, "elev": 650}, +"42970": {"id": "", "name": "CUTTACK", "state": "IN", "lat": 20.470, "lon": 85.930, "elev": 27}, +"42971": {"id": "VEBS", "name": "BHUBANESWAR", "state": "IN", "lat": 20.250, "lon": 85.830, "elev": 46}, +"42977": {"id": "", "name": "SANDHEADS", "state": "IN", "lat": 20.850, "lon": 88.250, "elev": 10}, +"43003": {"id": "VABB", "name": "BOMBAY/SANTACRUZ", "state": "IN", "lat": 19.120, "lon": 72.850, "elev": 14}, +"43013": {"id": "", "name": "AURANGABAD", "state": "IN", "lat": 19.880, "lon": 75.330, "elev": 581}, +"43014": {"id": "VAAU", "name": "AURANGABAD AIRPORT", "state": "IN", "lat": 19.850, "lon": 75.400, "elev": 579}, +"43017": {"id": "", "name": "PARBHANI", "state": "IN", "lat": 19.270, "lon": 76.770, "elev": 423}, +"43041": {"id": "", "name": "JAGDALPUR", "state": "IN", "lat": 19.080, "lon": 82.030, "elev": 553}, +"43049": {"id": "", "name": "GOPALPUR", "state": "IN", "lat": 19.260, "lon": 84.880, "elev": 17}, +"43063": {"id": "", "name": "POONA", "state": "IN", "lat": 18.530, "lon": 73.850, "elev": 559}, +"43083": {"id": "", "name": "MEDAK", "state": "IN", "lat": 18.050, "lon": 78.270, "elev": 472}, +"43110": {"id": "", "name": "RATNAGIRI", "state": "IN", "lat": 16.980, "lon": 73.330, "elev": 34}, +"43121": {"id": "", "name": "GULBARGA", "state": "IN", "lat": 17.350, "lon": 76.850, "elev": 458}, +"43128": {"id": "VOHY", "name": "HYDERABAD (CIV/MIL)", "state": "IN", "lat": 17.450, "lon": 78.470, "elev": 545}, +"43150": {"id": "", "name": "VISHAKHAPATNAM/WALT", "state": "IN", "lat": 17.700, "lon": 83.300, "elev": 66}, +"43158": {"id": "", "name": "SANGLI", "state": "IN", "lat": 16.850, "lon": 74.600, "elev": 549}, +"43182": {"id": "", "name": "NANDIGAMA", "state": "IN", "lat": 16.780, "lon": 80.280, "elev": 51}, +"43185": {"id": "", "name": "MACHILIPATNAM", "state": "IN", "lat": 16.200, "lon": 81.150, "elev": 3}, +"43189": {"id": "", "name": "KAKINADA", "state": "IN", "lat": 16.950, "lon": 82.230, "elev": 8}, +"43192": {"id": "", "name": "GOA/PANJIM", "state": "IN", "lat": 15.480, "lon": 73.820, "elev": 60}, +"43194": {"id": "", "name": "GOA/DABOLIM AIRPORT", "state": "IN", "lat": 15.380, "lon": 73.820, "elev": 52}, +"43198": {"id": "VABM", "name": "BELGAUM/SAMBRA", "state": "IN", "lat": 15.850, "lon": 74.620, "elev": 747}, +"43201": {"id": "", "name": "GADAG", "state": "IN", "lat": 15.420, "lon": 75.630, "elev": 650}, +"43205": {"id": "VOBI", "name": "BELLARY", "state": "IN", "lat": 15.150, "lon": 76.850, "elev": 449}, +"43237": {"id": "", "name": "ANANTAPUR", "state": "IN", "lat": 14.680, "lon": 77.620, "elev": 350}, +"43257": {"id": "", "name": "AGUMBE", "state": "IN", "lat": 13.500, "lon": 75.080, "elev": 659}, +"43275": {"id": "", "name": "TIRUPATHI", "state": "IN", "lat": 13.670, "lon": 79.580, "elev": 105}, +"43279": {"id": "VOMM", "name": "MADRAS/MINAMBAKKAM", "state": "IN", "lat": 13.000, "lon": 80.180, "elev": 16}, +"43285": {"id": "", "name": "MANGALORE/PANAMBUR", "state": "IN", "lat": 12.950, "lon": 74.830, "elev": 31}, +"43289": {"id": "", "name": "MANDYA", "state": "IN", "lat": 12.500, "lon": 76.830, "elev": 695}, +"43295": {"id": "", "name": "BANGALORE", "state": "IN", "lat": 12.970, "lon": 77.580, "elev": 921}, +"43296": {"id": "", "name": "BANGALORE AERO", "state": "IN", "lat": 12.950, "lon": 77.630, "elev": 897}, +"43299": {"id": "", "name": "KOLAR GOLD FIELD", "state": "IN", "lat": 12.950, "lon": 78.300, "elev": 882}, +"43310": {"id": "", "name": "LONG ISLAND", "state": "IN", "lat": 12.420, "lon": 92.930, "elev": 25}, +"43311": {"id": "", "name": "AMINI DIVI", "state": "IN", "lat": 11.120, "lon": 72.730, "elev": 4}, +"43312": {"id": "", "name": "AGATTI", "state": "IN", "lat": 10.850, "lon": 72.470, "elev": 4}, +"43316": {"id": "", "name": "NILAMBUR", "state": "IN", "lat": 11.280, "lon": 76.230, "elev": 31}, +"43319": {"id": "", "name": "COIMBATURE", "state": "IN", "lat": 11.000, "lon": 76.970, "elev": 409}, +"43321": {"id": "VOCB", "name": "COIMBATORE/PEEL", "state": "IN", "lat": 11.030, "lon": 77.050, "elev": 399}, +"43330": {"id": "", "name": "THANJAVUR", "state": "IN", "lat": 10.780, "lon": 79.130, "elev": 68}, +"43333": {"id": "VEPB", "name": "PORT BLAIR", "state": "IN", "lat": 11.670, "lon": 92.720, "elev": 79}, +"43344": {"id": "", "name": "TIRUCHIRAPALLI", "state": "IN", "lat": 10.760, "lon": 78.710, "elev": 88}, +"43346": {"id": "", "name": "KARAIKAL", "state": "IN", "lat": 10.920, "lon": 79.830, "elev": 7}, +"43349": {"id": "", "name": "VEDARANNIYAM", "state": "IN", "lat": 10.370, "lon": 79.850, "elev": 2}, +"43353": {"id": "VOCC", "name": "COCHIN (IN-NAVY)", "state": "IN", "lat": 9.950, "lon": 76.270, "elev": 3}, +"43359": {"id": "", "name": "MADURAI", "state": "IN", "lat": 9.920, "lon": 78.120, "elev": 132}, +"43368": {"id": "", "name": "CAR NICOBAR", "state": "IN", "lat": 9.150, "lon": 92.820, "elev": 14}, +"43369": {"id": "", "name": "MINICOY ISLAND", "state": "IN", "lat": 8.300, "lon": 73.150, "elev": 2}, +"43371": {"id": "", "name": "THIRUVANANTHAPURAM", "state": "IN", "lat": 8.480, "lon": 76.950, "elev": 64}, +"43377": {"id": "", "name": "KANNIYAKUMARI", "state": "IN", "lat": 8.080, "lon": 77.500, "elev": 37}, +"43413": {"id": "", "name": "MANNAR", "state": "SB", "lat": 8.980, "lon": 79.920, "elev": 3}, +"43418": {"id": "", "name": "TRINCOMALEE", "state": "SB", "lat": 8.580, "lon": 81.250, "elev": 7}, +"43424": {"id": "", "name": "PUTTALAM", "state": "SB", "lat": 8.030, "lon": 79.830, "elev": 2}, +"43466": {"id": "", "name": "COLOMBO", "state": "SB", "lat": 6.900, "lon": 79.870, "elev": 7}, +"43497": {"id": "", "name": "HAMBANTOTA", "state": "SB", "lat": 6.120, "lon": 81.130, "elev": 20}, +"43555": {"id": "", "name": "MALE", "state": "MD", "lat": 4.200, "lon": 73.530, "elev": 2}, +"43599": {"id": "", "name": "GAN", "state": "MD", "lat": -0.680, "lon": 73.150, "elev": 2}, +"44212": {"id": "", "name": "ULAN-GOM", "state": "MO", "lat": 49.970, "lon": 92.080, "elev": 936}, +"44218": {"id": "", "name": "KOHO HOVDO", "state": "MO", "lat": 48.020, "lon": 91.650, "elev": 1406}, +"44231": {"id": "", "name": "MUREN", "state": "MO", "lat": 49.630, "lon": 100.170, "elev": 1288}, +"44232": {"id": "", "name": "HUTAG", "state": "MO", "lat": 49.380, "lon": 102.700, "elev": 949}, +"44237": {"id": "", "name": "ERDENEMANDAL", "state": "MO", "lat": 48.530, "lon": 101.380, "elev": 1510}, +"44259": {"id": "", "name": "CHOIBALSAN", "state": "MO", "lat": 48.070, "lon": 114.500, "elev": 756}, +"44272": {"id": "", "name": "ULAN KOM", "state": "MO", "lat": 47.750, "lon": 96.850, "elev": 1753}, +"44277": {"id": "", "name": "ALTAI", "state": "MO", "lat": 46.400, "lon": 96.250, "elev": 2147}, +"44282": {"id": "", "name": "TSETSERLEG", "state": "MO", "lat": 47.450, "lon": 101.470, "elev": 1697}, +"44287": {"id": "", "name": "BAIANHONGOR", "state": "MO", "lat": 46.130, "lon": 100.680, "elev": 1879}, +"44288": {"id": "", "name": "ARBAIHER", "state": "MO", "lat": 46.270, "lon": 102.780, "elev": 1832}, +"44292": {"id": "", "name": "ULAN-BATOR", "state": "MO", "lat": 47.930, "lon": 106.980, "elev": 1313}, +"44298": {"id": "", "name": "CHOIR", "state": "MO", "lat": 46.450, "lon": 108.220, "elev": 1285}, +"44341": {"id": "", "name": "MANDALGOVL", "state": "MO", "lat": 45.770, "lon": 106.280, "elev": 1397}, +"44354": {"id": "", "name": "SAINSHAND", "state": "MO", "lat": 44.900, "lon": 110.120, "elev": 936}, +"44373": {"id": "", "name": "DALANZADGAD", "state": "MO", "lat": 43.580, "lon": 104.420, "elev": 1470}, +"44454": {"id": "VNKT", "name": "KATHMANDU INTL", "state": "NP", "lat": 27.700, "lon": 85.370, "elev": 1337}, +"45004": {"id": "", "name": "KING'S PARK", "state": "HK", "lat": 22.320, "lon": 114.170, "elev": 66}, +"46689": {"id": "", "name": "MATSU", "state": "TW", "lat": 26.170, "lon": 119.930, "elev": 91}, +"46692": {"id": "", "name": "TAIBEI", "state": "TW", "lat": 25.030, "lon": 121.520, "elev": 9}, +"46695": {"id": "", "name": "PENGJIA YU", "state": "TW", "lat": 25.630, "lon": 122.070, "elev": 102}, +"46699": {"id": "", "name": "HUA-LIEN CITY", "state": "TW", "lat": 23.980, "lon": 121.600, "elev": 19}, +"46734": {"id": "RCQC", "name": "MAKUNG AB", "state": "TW", "lat": 23.580, "lon": 119.620, "elev": 31}, +"46736": {"id": "", "name": "KINMEN", "state": "TW", "lat": 24.430, "lon": 118.430, "elev": 12}, +"46741": {"id": "", "name": "TAINAN", "state": "TW", "lat": 23.000, "lon": 120.220, "elev": 14}, +"46746": {"id": "", "name": "CHIA-I", "state": "TW", "lat": 23.470, "lon": 120.380, "elev": 25}, +"46747": {"id": "RCMJ", "name": "DONGGANG", "state": "TW", "lat": 22.470, "lon": 120.430, "elev": 8}, +"46750": {"id": "", "name": "PINGTUNG SOUTH", "state": "TW", "lat": 22.670, "lon": 120.450, "elev": 24}, +"46757": {"id": "", "name": "HSINCHU CITY", "state": "TW", "lat": 24.830, "lon": 120.000, "elev": 27}, +"46763": {"id": "RCYU", "name": "JULIEN AB", "state": "TW", "lat": 24.030, "lon": 121.620, "elev": 16}, +"46765": {"id": "", "name": "JOYUTANG", "state": "TW", "lat": 23.880, "lon": 120.850, "elev": 1015}, +"46780": {"id": "", "name": "LU-TAO", "state": "TW", "lat": 22.680, "lon": 121.500, "elev": 280}, +"46810": {"id": "RCLM", "name": "DONGSHA", "state": "TW", "lat": 20.670, "lon": 116.720, "elev": 6}, +"46902": {"id": "", "name": "NENSHA ISLAND", "state": "TW", "lat": 10.380, "lon": 114.370, "elev": 5}, +"47022": {"id": "", "name": "PUNGSAN", "state": "KR", "lat": 40.820, "lon": 128.150, "elev": 1206}, +"47041": {"id": "", "name": "HAMHEUNG", "state": "KR", "lat": 39.930, "lon": 127.550, "elev": 38}, +"47058": {"id": "ZKPY", "name": "PYONGYANG", "state": "KR", "lat": 39.030, "lon": 125.780, "elev": 38}, +"47090": {"id": "", "name": "SOKCHO", "state": "KR", "lat": 38.150, "lon": 128.340, "elev": 19}, +"47102": {"id": "", "name": "BAENGNYEONGDO", "state": "KO", "lat": 37.580, "lon": 124.380, "elev": 158}, +"47103": {"id": "", "name": "PAENGNYONG-DO ISLAND", "state": "KO", "lat": 37.970, "lon": 124.670, "elev": 177}, +"47104": {"id": "", "name": "BUKGANGNEUNG", "state": "KO", "lat": 37.830, "lon": 128.860, "elev": 79}, +"47107": {"id": "", "name": "KANGNUNG AFB", "state": "KO", "lat": 37.750, "lon": 128.950, "elev": 6}, +"47110": {"id": "", "name": "SEOUL/KIMPO", "state": "KO", "lat": 37.550, "lon": 126.800, "elev": 20}, +"47112": {"id": "", "name": "INCHON", "state": "KO", "lat": 37.480, "lon": 126.630, "elev": 70}, +"47114": {"id": "RKNW", "name": "WONJU", "state": "KO", "lat": 37.330, "lon": 127.950, "elev": 150}, +"47118": {"id": "", "name": "HOENGSONG AFB", "state": "KO", "lat": 37.430, "lon": 127.950, "elev": 101}, +"47122": {"id": "RKSO", "name": "OSAN (US/KOR-AFB)", "state": "KO", "lat": 37.100, "lon": 127.030, "elev": 12}, +"47129": {"id": "", "name": "SOSAN", "state": "KO", "lat": 36.770, "lon": 126.470, "elev": 21}, +"47130": {"id": "", "name": "ULCHIN", "state": "KO", "lat": 36.980, "lon": 129.420, "elev": 51}, +"47132": {"id": "", "name": "TAEJON AFB", "state": "KO", "lat": 36.330, "lon": 127.380, "elev": 64}, +"47134": {"id": "", "name": "YECHON", "state": "KO", "lat": 36.620, "lon": 128.350, "elev": 120}, +"47135": {"id": "", "name": "CHUPUNGNYONG", "state": "KO", "lat": 36.220, "lon": 128.000, "elev": 249}, +"47138": {"id": "", "name": "POHANG", "state": "KO", "lat": 36.030, "lon": 129.380, "elev": 6}, +"47142": {"id": "", "name": "TAEGU", "state": "KO", "lat": 35.900, "lon": 128.650, "elev": 35}, +"47144": {"id": "", "name": "KUNSAN RADAR", "state": "KO", "lat": 36.020, "lon": 126.780, "elev": 215}, +"47153": {"id": "", "name": "PUSAN WEST", "state": "KO", "lat": 35.180, "lon": 128.930, "elev": 4}, +"47158": {"id": "RKJJ", "name": "KWANGJU (KOR-AFB)", "state": "KO", "lat": 35.120, "lon": 126.820, "elev": 13}, +"47161": {"id": "", "name": "SACHON", "state": "KO", "lat": 35.080, "lon": 128.080, "elev": 8}, +"47169": {"id": "", "name": "HEUKSANDO", "state": "KO", "lat": 34.700, "lon": 125.400, "elev": 15}, +"47170": {"id": "", "name": "WANDO ISLAND", "state": "KO", "lat": 34.300, "lon": 126.750, "elev": 12}, +"47185": {"id": "", "name": "CHEJU", "state": "KO", "lat": 33.280, "lon": 126.170, "elev": 73}, +"47186": {"id": "", "name": "NAT TYPHOON CENTER", "state": "KO", "lat": 33.330, "lon": 126.680, "elev": 246}, +"47401": {"id": "", "name": "WAKKANAI", "state": "JP", "lat": 45.420, "lon": 141.680, "elev": 11}, +"47402": {"id": "", "name": "KITAMIESASHI", "state": "JP", "lat": 44.930, "lon": 142.580, "elev": 8}, +"47407": {"id": "", "name": "ASAHIKAWA", "state": "JP", "lat": 43.770, "lon": 142.370, "elev": 116}, +"47412": {"id": "", "name": "SAPPORO", "state": "JP", "lat": 43.050, "lon": 141.330, "elev": 19}, +"47413": {"id": "", "name": "IWAMIZAWA", "state": "JP", "lat": 43.220, "lon": 141.780, "elev": 51}, +"47418": {"id": "RJCS", "name": "KUSHIRO/KENEBET", "state": "JP", "lat": 42.980, "lon": 144.400, "elev": 37}, +"47420": {"id": "", "name": "NEMURO", "state": "JP", "lat": 43.330, "lon": 145.580, "elev": 26}, +"47421": {"id": "", "name": "SUTTSU", "state": "JP", "lat": 42.800, "lon": 140.230, "elev": 38}, +"47428": {"id": "", "name": "ESASHI", "state": "JP", "lat": 41.870, "lon": 140.130, "elev": 12}, +"47472": {"id": "", "name": "RISHIRI AIRPORT", "state": "JP", "lat": 45.230, "lon": 141.200, "elev": 33}, +"47481": {"id": "RJCM", "name": "MEMAMBETSU AIRPORT", "state": "JP", "lat": 43.900, "lon": 144.170, "elev": 36}, +"47545": {"id": "RJSK", "name": "AKITA AIRPORT", "state": "JP", "lat": 39.620, "lon": 140.220, "elev": 96}, +"47549": {"id": "RJSI", "name": "HANAMAKI AIRPORT", "state": "JP", "lat": 39.430, "lon": 141.130, "elev": 93}, +"47570": {"id": "", "name": "WAKAMATSU", "state": "JP", "lat": 37.480, "lon": 139.920, "elev": 213}, +"47572": {"id": "", "name": "NIIGATA/YAHIKOY", "state": "JP", "lat": 37.720, "lon": 138.820, "elev": 634}, +"47573": {"id": "RJSN", "name": "NIIGATA", "state": "JP", "lat": 37.950, "lon": 139.120, "elev": 4}, +"47580": {"id": "RJSM", "name": "MISAWA", "state": "JP", "lat": 40.680, "lon": 141.380, "elev": 36}, +"47581": {"id": "", "name": "HACHINOHE", "state": "JP", "lat": 40.530, "lon": 141.530, "elev": 28}, +"47582": {"id": "", "name": "AKITA", "state": "JP", "lat": 39.720, "lon": 140.100, "elev": 21}, +"47590": {"id": "", "name": "SENDAI", "state": "JP", "lat": 38.270, "lon": 140.900, "elev": 43}, +"47591": {"id": "RJST", "name": "MATSUSHIMA", "state": "JP", "lat": 38.400, "lon": 141.220, "elev": 5}, +"47592": {"id": "", "name": "ISHINOMAKI", "state": "JP", "lat": 38.430, "lon": 141.300, "elev": 45}, +"47597": {"id": "", "name": "SHIRAKAWA", "state": "JP", "lat": 37.120, "lon": 140.220, "elev": 356}, +"47600": {"id": "", "name": "WAJIMA", "state": "JP", "lat": 37.380, "lon": 136.900, "elev": 14}, +"47601": {"id": "", "name": "HEGURASHIMA", "state": "JP", "lat": 37.850, "lon": 136.920, "elev": 14}, +"47604": {"id": "", "name": "NIIGATA", "state": "JP", "lat": 37.920, "lon": 139.050, "elev": 7}, +"47615": {"id": "", "name": "UTSUNOMIYA", "state": "JP", "lat": 36.550, "lon": 139.870, "elev": 140}, +"47617": {"id": "", "name": "TAKAJAMA", "state": "JP", "lat": 36.150, "lon": 137.250, "elev": 561}, +"47618": {"id": "", "name": "MATSUMOTO", "state": "JP", "lat": 36.250, "lon": 137.970, "elev": 611}, +"47620": {"id": "", "name": "SUWA", "state": "JP", "lat": 36.050, "lon": 138.120, "elev": 762}, +"47622": {"id": "", "name": "KARUIZAWA MOUNTAIN", "state": "JP", "lat": 36.330, "lon": 138.550, "elev": 1004}, +"47638": {"id": "", "name": "KOFU", "state": "JP", "lat": 35.670, "lon": 138.550, "elev": 274}, +"47640": {"id": "", "name": "KAWAGUCHIKO", "state": "JP", "lat": 35.500, "lon": 138.770, "elev": 861}, +"47641": {"id": "", "name": "CHICHIBU", "state": "JP", "lat": 35.980, "lon": 139.080, "elev": 219}, +"47646": {"id": "", "name": "TATENO", "state": "JP", "lat": 36.050, "lon": 140.130, "elev": 31}, +"47651": {"id": "", "name": "TSU", "state": "JP", "lat": 34.730, "lon": 136.520, "elev": 18}, +"47657": {"id": "", "name": "MISHIMA ISLAND", "state": "JP", "lat": 35.120, "lon": 138.930, "elev": 22}, +"47670": {"id": "", "name": "YOKOHAMA", "state": "JP", "lat": 35.430, "lon": 139.650, "elev": 42}, +"47671": {"id": "RJTT", "name": "TOKYO INTL AIRPORT", "state": "JP", "lat": 35.550, "lon": 139.780, "elev": 8}, +"47677": {"id": "", "name": "MIYAKEJIMA ISLAND", "state": "JP", "lat": 34.120, "lon": 139.520, "elev": 37}, +"47678": {"id": "", "name": "HACHIJOJIMA/OMURE", "state": "JP", "lat": 33.120, "lon": 139.780, "elev": 80}, +"47680": {"id": "RJTR", "name": "KASTNER", "state": "JP", "lat": 35.520, "lon": 139.400, "elev": 109}, +"47681": {"id": "RJNH", "name": "HAMAMATSU AB", "state": "JP", "lat": 34.730, "lon": 137.670, "elev": 48}, +"47686": {"id": "RJAA", "name": "NEW TOKYO INTL AIRPORT", "state": "JP", "lat": 35.770, "lon": 140.380, "elev": 44}, +"47688": {"id": "", "name": "TATEYAMA AFB", "state": "JP", "lat": 34.980, "lon": 139.830, "elev": 5}, +"47707": {"id": "RJNT", "name": "TOYAMA AIRPORT", "state": "JP", "lat": 36.650, "lon": 137.180, "elev": 27}, +"47727": {"id": "", "name": "SHIMOFUSA AFB", "state": "JP", "lat": 35.780, "lon": 140.020, "elev": 33}, +"47738": {"id": "RJTH", "name": "HACHIJOJIMA ISLAND", "state": "JP", "lat": 33.120, "lon": 139.780, "elev": 95}, +"47740": {"id": "", "name": "SAIGO", "state": "JP", "lat": 36.200, "lon": 133.330, "elev": 31}, +"47741": {"id": "", "name": "MATSUE", "state": "JP", "lat": 35.450, "lon": 133.060, "elev": 22}, +"47744": {"id": "", "name": "YONAGO", "state": "JP", "lat": 35.430, "lon": 133.350, "elev": 8}, +"47764": {"id": "", "name": "IWAKUNI", "state": "JP", "lat": 34.150, "lon": 132.230, "elev": 3}, +"47770": {"id": "", "name": "KOBE", "state": "JP", "lat": 34.680, "lon": 135.180, "elev": 59}, +"47771": {"id": "RJOO", "name": "OSAKA INTL AIRPORT", "state": "JP", "lat": 34.780, "lon": 135.450, "elev": 15}, +"47772": {"id": "", "name": "OSAKA", "state": "JP", "lat": 34.680, "lon": 135.530, "elev": 50}, +"47778": {"id": "", "name": "SHIONOMISAKI", "state": "JP", "lat": 33.450, "lon": 135.770, "elev": 75}, +"47788": {"id": "RJOF", "name": "HOFU (JASDF)", "state": "JP", "lat": 34.030, "lon": 131.550, "elev": 5}, +"47807": {"id": "", "name": "FUKUOKA", "state": "JP", "lat": 33.580, "lon": 130.380, "elev": 14}, +"47808": {"id": "RJFF", "name": "FUKUOKA/ITAZUKE", "state": "JP", "lat": 33.580, "lon": 130.450, "elev": 12}, +"47817": {"id": "", "name": "NAGASAKI", "state": "JP", "lat": 32.730, "lon": 129.870, "elev": 35}, +"47821": {"id": "", "name": "ASOSAN MOUNTAIN", "state": "JP", "lat": 32.880, "lon": 131.080, "elev": 1144}, +"47822": {"id": "", "name": "NOBEOKA", "state": "JP", "lat": 32.580, "lon": 131.670, "elev": 21}, +"47827": {"id": "", "name": "KAGOSHIMA/YOSHINO", "state": "JP", "lat": 31.630, "lon": 130.580, "elev": 31}, +"47855": {"id": "RJFU", "name": "NAGASAKI (CIV/JMSDF)", "state": "JP", "lat": 32.920, "lon": 129.920, "elev": 5}, +"47858": {"id": "RJDB", "name": "IKI AIRPORT", "state": "JP", "lat": 33.750, "lon": 129.780, "elev": 16}, +"47881": {"id": "RJOS", "name": "TOKUSHIMA(JMSDF/CV)", "state": "JP", "lat": 34.130, "lon": 134.620, "elev": 11}, +"47887": {"id": "", "name": "MATSUYAMA", "state": "JP", "lat": 33.830, "lon": 132.780, "elev": 34}, +"47909": {"id": "", "name": "NAZE/FUNCHATOGE", "state": "JP", "lat": 28.380, "lon": 129.550, "elev": 7}, +"47910": {"id": "RJKN", "name": "TOKUNOSHIMA ISLAND", "state": "JP", "lat": 27.830, "lon": 128.880, "elev": 5}, +"47911": {"id": "ROYN", "name": "TONAGUNI AIRPORT", "state": "JP", "lat": 24.470, "lon": 122.980, "elev": 17}, +"47917": {"id": "", "name": "IRIOMOTEJIMA ISLAND", "state": "JP", "lat": 24.380, "lon": 123.750, "elev": 9}, +"47918": {"id": "ROIG", "name": "ISHIGAKIJIMA ISLAND", "state": "JP", "lat": 24.330, "lon": 124.170, "elev": 7}, +"47919": {"id": "", "name": "ISHIGIKI AIRPORT", "state": "JP", "lat": 24.330, "lon": 125.180, "elev": 26}, +"47935": {"id": "ROHF", "name": "HAMBY", "state": "JP", "lat": 26.300, "lon": 127.770, "elev": 6}, +"47936": {"id": "", "name": "NAHA AIRPORT", "state": "JP", "lat": 26.200, "lon": 127.680, "elev": 53}, +"47945": {"id": "ROMD", "name": "MINAMIDAITOJIMA ISLAND", "state": "JP", "lat": 25.830, "lon": 131.230, "elev": 15}, +"47971": {"id": "RJAO", "name": "CHICHIJIMA ISLAND", "state": "JP", "lat": 27.080, "lon": 142.180, "elev": 8}, +"47981": {"id": "RJAW", "name": "IWOJIMA (JMSDF)", "state": "JP", "lat": 24.780, "lon": 141.330, "elev": 116}, +"47991": {"id": "RJAM", "name": "MINAMITORISHIMA", "state": "JP", "lat": 24.300, "lon": 153.970, "elev": 9}, +"48008": {"id": "", "name": "MYITKYINA", "state": "BM", "lat": 25.370, "lon": 97.400, "elev": 147}, +"48017": {"id": "", "name": "PINLEBU", "state": "BM", "lat": 24.080, "lon": 95.370, "elev": 259}, +"48024": {"id": "", "name": "KALEMYO", "state": "BM", "lat": 23.200, "lon": 94.070, "elev": 152}, +"48042": {"id": "VBRM", "name": "MANDALAY", "state": "BM", "lat": 21.980, "lon": 96.100, "elev": 76}, +"48053": {"id": "", "name": "MEIKTILA", "state": "BM", "lat": 20.880, "lon": 95.900, "elev": 220}, +"48058": {"id": "", "name": "LOILEM", "state": "BM", "lat": 20.920, "lon": 97.550, "elev": 1355}, +"48062": {"id": "", "name": "AKYAB", "state": "BM", "lat": 20.130, "lon": 92.880, "elev": 5}, +"48080": {"id": "", "name": "SANDOWAY", "state": "BM", "lat": 18.470, "lon": 94.350, "elev": 11}, +"48087": {"id": "", "name": "HENZADA", "state": "BM", "lat": 17.670, "lon": 95.420, "elev": 26}, +"48097": {"id": "", "name": "RANGOON", "state": "BM", "lat": 16.770, "lon": 96.170, "elev": 15}, +"48108": {"id": "", "name": "TAVOY", "state": "BM", "lat": 14.100, "lon": 98.220, "elev": 17}, +"48110": {"id": "", "name": "MERGUI", "state": "BM", "lat": 12.430, "lon": 98.600, "elev": 37}, +"48112": {"id": "", "name": "VICTORIA POINT", "state": "BM", "lat": 9.970, "lon": 98.580, "elev": 47}, +"48327": {"id": "VTCC", "name": "CHIANG MAI (CIV/AFB)", "state": "TH", "lat": 18.780, "lon": 98.980, "elev": 314}, +"48328": {"id": "VTCL", "name": "LAMPANG", "state": "TH", "lat": 18.280, "lon": 99.520, "elev": 242}, +"48353": {"id": "VTUL", "name": "LOEI", "state": "TH", "lat": 17.450, "lon": 101.730, "elev": 254}, +"48354": {"id": "VTUD", "name": "UDON THANI", "state": "TH", "lat": 17.380, "lon": 102.800, "elev": 182}, +"48357": {"id": "", "name": "NAKHON PHANOM", "state": "TH", "lat": 17.380, "lon": 104.650, "elev": 176}, +"48375": {"id": "VTPM", "name": "MAE SOT/TAK", "state": "TH", "lat": 16.670, "lon": 98.550, "elev": 197}, +"48378": {"id": "VTPS", "name": "PHITSANULOK", "state": "TH", "lat": 16.820, "lon": 100.270, "elev": 45}, +"48379": {"id": "", "name": "PHETCHABUN", "state": "TH", "lat": 16.420, "lon": 101.130, "elev": 114}, +"48407": {"id": "VTUU", "name": "UBON/RATCHATHANI", "state": "TH", "lat": 15.250, "lon": 104.870, "elev": 127}, +"48416": {"id": "", "name": "THA TUM", "state": "TH", "lat": 15.320, "lon": 103.680, "elev": 129}, +"48429": {"id": "", "name": "SUVARNABHUMI INTL AIRPORT", "state": "TH", "lat": 13.630, "lon": 100.770, "elev": 6}, +"48430": {"id": "VTBI", "name": "PRACHIN BURI", "state": "TH", "lat": 14.050, "lon": 101.370, "elev": 6}, +"48431": {"id": "VTUN", "name": "NAKHON RATCHASIMA", "state": "TH", "lat": 14.970, "lon": 102.830, "elev": 181}, +"48453": {"id": "", "name": "BANG NA AGROMET", "state": "TH", "lat": 13.670, "lon": 100.600, "elev": 3}, +"48455": {"id": "", "name": "BANGKOK", "state": "TH", "lat": 13.730, "lon": 100.500, "elev": 20}, +"48456": {"id": "VTBD", "name": "BANGKOK/DON MUA", "state": "TH", "lat": 13.920, "lon": 100.600, "elev": 12}, +"48459": {"id": "VTBS", "name": "CHON BURI/SATTA", "state": "TH", "lat": 13.370, "lon": 100.980, "elev": 2}, +"48477": {"id": "STTA", "name": "SATTAHIP", "state": "TH", "lat": 12.680, "lon": 100.980, "elev": 18}, +"48478": {"id": "VTBU", "name": "RAYONG", "state": "TH", "lat": 12.630, "lon": 101.350, "elev": 5}, +"48480": {"id": "VTBC", "name": "CHANTHABURI", "state": "TH", "lat": 12.600, "lon": 102.120, "elev": 4}, +"48500": {"id": "VTBP", "name": "PRACHUAP KHIRIKHAN", "state": "TH", "lat": 11.830, "lon": 99.830, "elev": 5}, +"48551": {"id": "VTSB", "name": "SURAT THANI", "state": "TH", "lat": 9.120, "lon": 99.350, "elev": 11}, +"48565": {"id": "VTSP", "name": "PHUKET INTL AIRPORT", "state": "TH", "lat": 8.120, "lon": 98.320, "elev": 10}, +"48568": {"id": "VTSH", "name": "SONGKHLA (THAI-NAVY)", "state": "TH", "lat": 7.200, "lon": 100.600, "elev": 5}, +"48569": {"id": "VTSS", "name": "HAT YAI INTL (AFB)", "state": "TH", "lat": 6.920, "lon": 100.430, "elev": 35}, +"48580": {"id": "VTSK", "name": "PATTANI", "state": "TH", "lat": 6.780, "lon": 101.150, "elev": 9}, +"48601": {"id": "WMKP", "name": "PENANG/BAYAN LEPAS", "state": "MS", "lat": 5.300, "lon": 100.270, "elev": 4}, +"48602": {"id": "", "name": "BUTTERWORTH", "state": "MS", "lat": 5.470, "lon": 100.380, "elev": 1}, +"48615": {"id": "WMKC", "name": "KOTA BHARU/SULTAN P", "state": "MS", "lat": 6.170, "lon": 102.280, "elev": 5}, +"48619": {"id": "WMKN", "name": "KUALA TRENGGANU", "state": "MS", "lat": 5.380, "lon": 103.100, "elev": 6}, +"48620": {"id": "WMBA", "name": "SITIAWAN", "state": "MS", "lat": 4.220, "lon": 100.700, "elev": 8}, +"48647": {"id": "", "name": "KUALA LUMPUR", "state": "MS", "lat": 3.120, "lon": 101.550, "elev": 22}, +"48648": {"id": "", "name": "PETALING JAYA", "state": "MS", "lat": 3.100, "lon": 101.650, "elev": 57}, +"48650": {"id": "", "name": "SEPANG (KUALA LUMPUR)", "state": "KL", "lat": 2.730, "lon": 101.700, "elev": 16}, +"48657": {"id": "WMKD", "name": "KUANTAN (AFB)", "state": "MS", "lat": 3.780, "lon": 103.220, "elev": 16}, +"48698": {"id": "WSSS", "name": "SINGAPORE/CHANGI", "state": "SR", "lat": 1.370, "lon": 103.980, "elev": 16}, +"48811": {"id": "VVDB", "name": "DIEN BIEN PHU ARPT", "state": "VS", "lat": 21.400, "lon": 103.020, "elev": 472}, +"48820": {"id": "VVNB", "name": "HANOI/NOIBAI INTL", "state": "VS", "lat": 21.020, "lon": 105.800, "elev": 6}, +"48839": {"id": "", "name": "BACH LONGVI", "state": "VS", "lat": 20.130, "lon": 107.720, "elev": 60}, +"48845": {"id": "VVVH", "name": "VINH", "state": "VS", "lat": 18.700, "lon": 105.670, "elev": 6}, +"48848": {"id": "", "name": "DONG HOI", "state": "VS", "lat": 17.520, "lon": 106.580, "elev": 8}, +"48855": {"id": "VVDN", "name": "DA NANG INTL ARPT", "state": "VS", "lat": 16.030, "lon": 108.180, "elev": 7}, +"48870": {"id": "", "name": "QUI NHON", "state": "VS", "lat": 13.770, "lon": 109.220, "elev": 5}, +"48873": {"id": "", "name": "TUY-HOA", "state": "VS", "lat": 13.080, "lon": 109.280, "elev": 11}, +"48877": {"id": "VVNT", "name": "NHA TRANG", "state": "VS", "lat": 12.150, "lon": 109.120, "elev": 10}, +"48887": {"id": "", "name": "PHAN THIET", "state": "VS", "lat": 10.930, "lon": 108.100, "elev": 8}, +"48900": {"id": "VVTS", "name": "HO CHI MINH/TANSONN", "state": "VS", "lat": 10.820, "lon": 106.670, "elev": 19}, +"48910": {"id": "", "name": "VINH LONG", "state": "VS", "lat": 10.250, "lon": 105.950, "elev": 3}, +"48914": {"id": "", "name": "CA MAU (POINT)", "state": "VS", "lat": 9.170, "lon": 105.170, "elev": 3}, +"48918": {"id": "", "name": "CON SON", "state": "VS", "lat": 8.700, "lon": 106.580, "elev": 5}, +"48936": {"id": "", "name": "PAKLAY", "state": "LA", "lat": 18.200, "lon": 101.200, "elev": 220}, +"48968": {"id": "", "name": "KRAKOR", "state": "KP", "lat": 12.520, "lon": 104.180, "elev": 5}, +"48995": {"id": "VDKC", "name": "KOMPONG-CHAM", "state": "KP", "lat": 12.000, "lon": 105.450, "elev": 16}, +"48998": {"id": "", "name": "SVAY RIENG", "state": "KP", "lat": 11.080, "lon": 105.800, "elev": 6}, +"50527": {"id": "", "name": "HAILAR", "state": "SY CI", "lat": 49.220, "lon": 119.750, "elev": 611}, +"50557": {"id": "", "name": "NENJIANG", "state": "SY CI", "lat": 49.170, "lon": 125.230, "elev": 243}, +"50603": {"id": "", "name": "CHIN-BARAG", "state": "SY CI", "lat": 48.670, "lon": 116.820, "elev": 555}, +"50745": {"id": "ZYQQ", "name": "QIQIHAR", "state": "SY CI", "lat": 47.380, "lon": 123.920, "elev": 148}, +"50774": {"id": "", "name": "YICHUN", "state": "SY CI", "lat": 47.720, "lon": 128.900, "elev": 232}, +"50834": {"id": "", "name": "TE-PO-SU-K", "state": "SY CI", "lat": 46.500, "lon": 121.370, "elev": 427}, +"50953": {"id": "", "name": "HARBIN", "state": "SY CI", "lat": 45.750, "lon": 126.770, "elev": 143}, +"50963": {"id": "", "name": "TONGHE", "state": "SY CI", "lat": 45.970, "lon": 128.730, "elev": 110}, +"50968": {"id": "", "name": "SHANGZHI", "state": "SY CI", "lat": 45.220, "lon": 127.970, "elev": 191}, +"51076": {"id": "", "name": "ALTAY", "state": "UQ CI", "lat": 47.730, "lon": 88.080, "elev": 737}, +"51133": {"id": "", "name": "TA CHENG", "state": "UQ CI", "lat": 46.730, "lon": 83.000, "elev": 549}, +"51156": {"id": "", "name": "HOBOG SAIR", "state": "UQ CI", "lat": 46.780, "lon": 85.720, "elev": 1294}, +"51232": {"id": "", "name": "BORDER STATION", "state": "UQ CI", "lat": 45.970, "lon": 82.530, "elev": 823}, +"51243": {"id": "", "name": "KARAMAY", "state": "UQ CI", "lat": 45.600, "lon": 84.850, "elev": 428}, +"51288": {"id": "", "name": "BAYTIK SHAN", "state": "UQ CI", "lat": 45.370, "lon": 90.530, "elev": 1651}, +"51334": {"id": "", "name": "JINGHE", "state": "UQ CI", "lat": 44.620, "lon": 82.900, "elev": 321}, +"51379": {"id": "", "name": "CHITAI", "state": "UQ CI", "lat": 44.020, "lon": 89.570, "elev": 797}, +"51431": {"id": "ZWYN", "name": "YINING", "state": "UQ CI", "lat": 43.950, "lon": 81.330, "elev": 663}, +"51437": {"id": "", "name": "ZHAOSU/MONGGOLK", "state": "UQ CI", "lat": 43.170, "lon": 81.120, "elev": 1830}, +"51463": {"id": "", "name": "URUMQI", "state": "UQ CI", "lat": 43.780, "lon": 87.620, "elev": 919}, +"51467": {"id": "", "name": "BALGUNTAY", "state": "UQ CI", "lat": 42.670, "lon": 86.330, "elev": 1753}, +"51495": {"id": "", "name": "CHI CHIA CHIENG", "state": "UQ CI", "lat": 43.480, "lon": 91.630, "elev": 874}, +"51567": {"id": "", "name": "YANQI", "state": "UQ CI", "lat": 42.050, "lon": 86.570, "elev": 1097}, +"51573": {"id": "", "name": "TULUFAN", "state": "UQ CI", "lat": 42.930, "lon": 89.200, "elev": 35}, +"51628": {"id": "", "name": "WEN-SU", "state": "UQ CI", "lat": 41.270, "lon": 80.230, "elev": 1286}, +"51633": {"id": "", "name": "BAICHENG/BAY", "state": "UQ CI", "lat": 41.770, "lon": 81.870, "elev": 1280}, +"51642": {"id": "", "name": "LUNTAI/BUGUR", "state": "UQ CI", "lat": 41.780, "lon": 84.170, "elev": 883}, +"51644": {"id": "", "name": "KUQA", "state": "UQ CI", "lat": 41.720, "lon": 82.950, "elev": 1100}, +"51656": {"id": "", "name": "KORLA", "state": "UQ CI", "lat": 41.750, "lon": 86.130, "elev": 933}, +"51701": {"id": "", "name": "SAI KO LO TEMA", "state": "UQ CI", "lat": 40.450, "lon": 75.380, "elev": 3651}, +"51705": {"id": "", "name": "AR-TUX", "state": "UQ CI", "lat": 39.670, "lon": 76.170, "elev": 1494}, +"51709": {"id": "ZWSH", "name": "KASHI", "state": "UQ CI", "lat": 39.470, "lon": 75.980, "elev": 1291}, +"51711": {"id": "", "name": "AIT BAI", "state": "UQ CI", "lat": 40.850, "lon": 77.930, "elev": 1986}, +"51716": {"id": "", "name": "BACHU", "state": "UQ CI", "lat": 39.800, "lon": 78.570, "elev": 1117}, +"51765": {"id": "", "name": "TIKANLIK", "state": "UQ CI", "lat": 40.630, "lon": 87.700, "elev": 847}, +"51777": {"id": "", "name": "RUOQIANG", "state": "UQ CI", "lat": 39.030, "lon": 88.170, "elev": 889}, +"51811": {"id": "", "name": "SOCHE", "state": "UQ CI", "lat": 38.430, "lon": 77.270, "elev": 1232}, +"51818": {"id": "", "name": "PISHAN", "state": "UQ CI", "lat": 37.620, "lon": 78.280, "elev": 1376}, +"51828": {"id": "ZWTN", "name": "HOTAN", "state": "UQ CI", "lat": 37.130, "lon": 79.930, "elev": 1375}, +"51839": {"id": "", "name": "MINFENG/NIYA", "state": "UQ CI", "lat": 37.070, "lon": 82.770, "elev": 1410}, +"51848": {"id": "", "name": "ANDIR", "state": "UQ CI", "lat": 37.930, "lon": 83.650, "elev": 1264}, +"51855": {"id": "", "name": "CHIEMO", "state": "UQ CI", "lat": 38.150, "lon": 85.550, "elev": 1248}, +"51886": {"id": "", "name": "MANGNAI", "state": "LZ CI", "lat": 38.250, "lon": 90.850, "elev": 2945}, +"52203": {"id": "ZWHM", "name": "HAMI", "state": "UQ CI", "lat": 42.820, "lon": 93.520, "elev": 739}, +"52267": {"id": "", "name": "EJIN QI", "state": "LZ CI", "lat": 41.950, "lon": 101.070, "elev": 941}, +"52323": {"id": "", "name": "MAZONG SHAN (MOUNT)", "state": "LZ CI", "lat": 41.800, "lon": 97.030, "elev": 1770}, +"52378": {"id": "", "name": "GUAIZIHU", "state": "LZ CI", "lat": 41.370, "lon": 102.370, "elev": 960}, +"52418": {"id": "", "name": "DUNHUANG", "state": "LZ CI", "lat": 40.150, "lon": 94.680, "elev": 1140}, +"52495": {"id": "", "name": "BAYAN MOD", "state": "LZ CI", "lat": 40.750, "lon": 104.500, "elev": 1329}, +"52533": {"id": "ZLJQ", "name": "JIUQUAN/SUZHOU", "state": "LZ CI", "lat": 39.770, "lon": 98.480, "elev": 1478}, +"52602": {"id": "", "name": "LING HU CHEN", "state": "LZ CI", "lat": 38.830, "lon": 93.380, "elev": 2734}, +"52652": {"id": "", "name": "ZHANGYE", "state": "LZ CI", "lat": 38.930, "lon": 100.430, "elev": 1483}, +"52681": {"id": "", "name": "MINQIN", "state": "LZ CI", "lat": 38.630, "lon": 103.080, "elev": 1367}, +"52818": {"id": "", "name": "GOLMUD", "state": "LZ CI", "lat": 36.420, "lon": 94.900, "elev": 2809}, +"52833": {"id": "", "name": "SERH", "state": "LZ CI", "lat": 36.950, "lon": 98.350, "elev": 2987}, +"52836": {"id": "", "name": "DULAN/QAGAN US", "state": "LZ CI", "lat": 36.300, "lon": 98.100, "elev": 3192}, +"52856": {"id": "", "name": "GONGHE", "state": "LZ CI", "lat": 36.350, "lon": 100.780, "elev": 2743}, +"52866": {"id": "ZLXN", "name": "XINING", "state": "LZ CI", "lat": 36.620, "lon": 101.770, "elev": 2262}, +"52889": {"id": "", "name": "LANZHOU", "state": "LZ CI", "lat": 36.050, "lon": 103.880, "elev": 1518}, +"52983": {"id": "", "name": "YU ZHONG", "state": "LZ CI", "lat": 35.520, "lon": 104.090, "elev": 1875}, +"53068": {"id": "", "name": "ERENHOT", "state": "BJ CI", "lat": 43.650, "lon": 112.000, "elev": 966}, +"53336": {"id": "", "name": "HALIUT", "state": "BJ CI", "lat": 41.570, "lon": 108.520, "elev": 1290}, +"53463": {"id": "ZBHH", "name": "HOHHOT", "state": "BJ CI", "lat": 40.820, "lon": 111.680, "elev": 1065}, +"53513": {"id": "", "name": "LINHE", "state": "BJ CI", "lat": 40.770, "lon": 107.400, "elev": 1041}, +"53543": {"id": "", "name": "DONGSHENG", "state": "BJ CI", "lat": 39.830, "lon": 109.980, "elev": 1459}, +"53614": {"id": "ZLIC", "name": "YINCHUAN", "state": "LZ CI", "lat": 38.480, "lon": 106.220, "elev": 1112}, +"53772": {"id": "ZBYN", "name": "TAIYUAN/WUSU", "state": "BJ CI", "lat": 37.780, "lon": 112.550, "elev": 779}, +"53782": {"id": "", "name": "YANGQUAN", "state": "BJ CI", "lat": 37.820, "lon": 113.570, "elev": 908}, +"53798": {"id": "", "name": "ZINGTAI", "state": "BJ CI", "lat": 37.070, "lon": 114.500, "elev": 78}, +"53845": {"id": "ZLYA", "name": "YAN AN", "state": "LZ CI", "lat": 36.600, "lon": 109.500, "elev": 959}, +"53898": {"id": "", "name": "ANYANG/ZHANGDE", "state": "BJ CI", "lat": 36.120, "lon": 114.370, "elev": 76}, +"53915": {"id": "", "name": "PINGLIANG", "state": "LZ CI", "lat": 35.550, "lon": 106.670, "elev": 1348}, +"54102": {"id": "", "name": "XILIN HOT/ABAGNAR", "state": "BJ CI", "lat": 43.950, "lon": 116.070, "elev": 991}, +"54135": {"id": "", "name": "TONGLIAO", "state": "SY CI", "lat": 43.600, "lon": 122.270, "elev": 180}, +"54161": {"id": "ZYCC", "name": "CHANGCHUN", "state": "SY CI", "lat": 43.900, "lon": 125.220, "elev": 238}, +"54208": {"id": "", "name": "DUOLUN/DOLONNUR", "state": "BJ CI", "lat": 42.180, "lon": 116.470, "elev": 1247}, +"54218": {"id": "", "name": "CHIFENG/ULANHAD", "state": "SY CI", "lat": 42.270, "lon": 118.970, "elev": 572}, +"54237": {"id": "", "name": "FUXIN", "state": "SY CI", "lat": 42.000, "lon": 121.630, "elev": 182}, +"54285": {"id": "", "name": "LUSHUIHE", "state": "SY CI", "lat": 42.520, "lon": 127.800, "elev": 853}, +"54292": {"id": "", "name": "YANJI", "state": "SY CI", "lat": 42.880, "lon": 129.470, "elev": 178}, +"54337": {"id": "", "name": "JINZHOU", "state": "SY CI", "lat": 41.120, "lon": 121.070, "elev": 70}, +"54342": {"id": "ZYYY", "name": "SHENYANG/DONGTA", "state": "SY CI", "lat": 41.770, "lon": 123.430, "elev": 43}, +"54374": {"id": "", "name": "LINJIANG", "state": "SY CI", "lat": 41.720, "lon": 126.920, "elev": 333}, +"54401": {"id": "", "name": "ZHANGJIAKOU", "state": "BJ CI", "lat": 40.780, "lon": 114.880, "elev": 726}, +"54405": {"id": "", "name": "HUAILAI/SHACHEN", "state": "BJ CI", "lat": 40.400, "lon": 115.500, "elev": 538}, +"54497": {"id": "", "name": "DANDONG", "state": "SY CI", "lat": 40.080, "lon": 124.330, "elev": 14}, +"54511": {"id": "ZBAA", "name": "BEIJING/PEKING", "state": "BJ CI", "lat": 39.930, "lon": 116.280, "elev": 55}, +"54539": {"id": "", "name": "LETING", "state": "BJ CI", "lat": 39.420, "lon": 118.900, "elev": 11}, +"54662": {"id": "ZYTL", "name": "DALIAN/DAIREN/LUDA", "state": "BJ CI", "lat": 38.900, "lon": 121.630, "elev": 97}, +"54727": {"id": "", "name": "ZHANGQIU", "state": "BJ CI", "lat": 36.420, "lon": 117.330, "elev": 123}, +"54776": {"id": "", "name": "CHENGSHANTOU", "state": "BJ CI", "lat": 37.400, "lon": 122.680, "elev": 47}, +"54823": {"id": "ZSTN", "name": "JINAN/TSINAN", "state": "BJ CI", "lat": 36.680, "lon": 116.980, "elev": 58}, +"54826": {"id": "", "name": "TAI SHAN (MTNS)", "state": "BJ CI", "lat": 36.250, "lon": 117.100, "elev": 1536}, +"54857": {"id": "ZSQD", "name": "QINGDAO/TSINGTAO", "state": "BJ CI", "lat": 36.070, "lon": 120.330, "elev": 77}, +"55228": {"id": "", "name": "KA-ERH", "state": "CD CI", "lat": 32.120, "lon": 80.070, "elev": 4279}, +"55248": {"id": "", "name": "LA KAO TSU", "state": "CD CI", "lat": 32.070, "lon": 84.050, "elev": 4420}, +"55279": {"id": "", "name": "BANGON", "state": "CD CI", "lat": 31.370, "lon": 90.020, "elev": 4701}, +"55299": {"id": "", "name": "NAGQU", "state": "CD CI", "lat": 31.480, "lon": 92.070, "elev": 4508}, +"55472": {"id": "", "name": "XANZA", "state": "CD CI", "lat": 30.950, "lon": 88.630, "elev": 4671}, +"55578": {"id": "", "name": "ZHIKATSE", "state": "CD CI", "lat": 29.220, "lon": 88.920, "elev": 3837}, +"55591": {"id": "ZULS", "name": "LHASA", "state": "CD CI", "lat": 29.670, "lon": 91.130, "elev": 3650}, +"55598": {"id": "", "name": "NEDONG", "state": "CD CI", "lat": 29.230, "lon": 91.770, "elev": 3657}, +"55664": {"id": "", "name": "TINGRI/XEGAR", "state": "CD CI", "lat": 28.630, "lon": 87.080, "elev": 4302}, +"55696": {"id": "", "name": "LHUNZE", "state": "CD CI", "lat": 28.420, "lon": 92.470, "elev": 3900}, +"55773": {"id": "", "name": "PALI MONASTERY", "state": "CD CI", "lat": 27.750, "lon": 89.170, "elev": 4301}, +"56004": {"id": "", "name": "TUOTUOHE/TANGGU", "state": "LZ CI", "lat": 34.220, "lon": 92.430, "elev": 4535}, +"56029": {"id": "", "name": "YUSHU", "state": "LZ CI", "lat": 33.020, "lon": 97.020, "elev": 3682}, +"56046": {"id": "", "name": "DARLAG", "state": "LZ CI", "lat": 33.750, "lon": 99.650, "elev": 3968}, +"56067": {"id": "", "name": "JIGZHI", "state": "LZ CI", "lat": 33.470, "lon": 101.480, "elev": 3350}, +"56080": {"id": "", "name": "HEZUO", "state": "LZ CI", "lat": 34.970, "lon": 102.900, "elev": 2910}, +"56096": {"id": "", "name": "WUDU", "state": "LZ CI", "lat": 33.400, "lon": 104.920, "elev": 1079}, +"56106": {"id": "", "name": "SO TSIAN", "state": "CD CI", "lat": 31.870, "lon": 93.770, "elev": 3951}, +"56116": {"id": "", "name": "DENG CHEN", "state": "CD CI", "lat": 31.420, "lon": 95.600, "elev": 3874}, +"56137": {"id": "", "name": "QAMDO", "state": "CD CI", "lat": 31.150, "lon": 97.170, "elev": 3307}, +"56144": {"id": "", "name": "DEGE", "state": "CD CI", "lat": 31.800, "lon": 98.580, "elev": 3185}, +"56146": {"id": "", "name": "GARZE", "state": "CD CI", "lat": 31.620, "lon": 100.000, "elev": 3394}, +"56173": {"id": "", "name": "MA-TANG", "state": "CD CI", "lat": 31.850, "lon": 102.700, "elev": 3422}, +"56178": {"id": "", "name": "ZIAO-JIN", "state": "CD CI", "lat": 31.020, "lon": 102.370, "elev": 2426}, +"56187": {"id": "", "name": "WENJIANG", "state": "CD CI", "lat": 30.700, "lon": 103.830, "elev": 540}, +"56188": {"id": "", "name": "WENCHUAN", "state": "CD CI", "lat": 31.480, "lon": 103.580, "elev": 2042}, +"56247": {"id": "", "name": "BATANG", "state": "CD CI", "lat": 30.000, "lon": 99.100, "elev": 2589}, +"56294": {"id": "ZUUU", "name": "CHENGDU", "state": "CD CI", "lat": 30.670, "lon": 104.020, "elev": 508}, +"56312": {"id": "", "name": "NYINGCHI", "state": "CD CI", "lat": 29.570, "lon": 94.470, "elev": 3001}, +"56374": {"id": "", "name": "KANGDING/DARDO", "state": "CD CI", "lat": 30.050, "lon": 101.970, "elev": 2617}, +"56462": {"id": "", "name": "JIULONG/GYAISI", "state": "CD CI", "lat": 29.000, "lon": 101.500, "elev": 2994}, +"56492": {"id": "", "name": "YIBIN", "state": "CD CI", "lat": 28.800, "lon": 104.600, "elev": 342}, +"56571": {"id": "", "name": "XICHANG", "state": "CD CI", "lat": 27.900, "lon": 102.270, "elev": 1599}, +"56651": {"id": "", "name": "LIJING", "state": "CD CI", "lat": 26.830, "lon": 100.470, "elev": 2394}, +"56671": {"id": "", "name": "HUILI", "state": "CD CI", "lat": 26.650, "lon": 102.250, "elev": 1788}, +"56691": {"id": "", "name": "WEINING", "state": "CD CI", "lat": 26.870, "lon": 104.280, "elev": 2236}, +"56739": {"id": "", "name": "TENGCHONG", "state": "CD CI", "lat": 25.030, "lon": 98.480, "elev": 1649}, +"56778": {"id": "ZPPP", "name": "KUNMING/WUJIABA", "state": "CD CI", "lat": 25.020, "lon": 102.680, "elev": 1892}, +"56946": {"id": "", "name": "GENGMA", "state": "CD CI", "lat": 23.550, "lon": 99.400, "elev": 1104}, +"56964": {"id": "", "name": "SIMAO", "state": "CD CI", "lat": 22.770, "lon": 100.980, "elev": 1303}, +"56985": {"id": "", "name": "MENGZI", "state": "CD CI", "lat": 23.380, "lon": 103.380, "elev": 1302}, +"56989": {"id": "", "name": "HE KOU", "state": "CD CI", "lat": 22.500, "lon": 104.000, "elev": 142}, +"57034": {"id": "", "name": "QIAN-ZIAN", "state": "LZ CI", "lat": 34.530, "lon": 108.230, "elev": 335}, +"57036": {"id": "ZLXY", "name": "XI'AN", "state": "LZ CI", "lat": 34.300, "lon": 108.930, "elev": 398}, +"57067": {"id": "", "name": "LU SHIH", "state": "LZ CI", "lat": 34.000, "lon": 111.020, "elev": 569}, +"57083": {"id": "ZHCC", "name": "ZHENGZHOU", "state": "BJ CI", "lat": 34.720, "lon": 113.650, "elev": 111}, +"57127": {"id": "", "name": "HANZHONG", "state": "LZ CI", "lat": 33.070, "lon": 107.030, "elev": 509}, +"57131": {"id": "", "name": "JINGHE", "state": "CI", "lat": 34.430, "lon": 108.970, "elev": 411}, +"57178": {"id": "", "name": "NANYANG", "state": "BJ CI", "lat": 33.020, "lon": 112.530, "elev": 131}, +"57245": {"id": "", "name": "ANKANG", "state": "LZ CI", "lat": 32.720, "lon": 109.030, "elev": 291}, +"57290": {"id": "", "name": "ZHUMADIAN", "state": "BJ CI", "lat": 33.000, "lon": 114.020, "elev": 83}, +"57328": {"id": "", "name": "DA ZIAN", "state": "CD CI", "lat": 31.200, "lon": 107.500, "elev": 311}, +"57411": {"id": "", "name": "NANCHONG", "state": "CD CI", "lat": 30.800, "lon": 106.080, "elev": 310}, +"57447": {"id": "", "name": "ENSHI", "state": "HK CI", "lat": 30.280, "lon": 109.470, "elev": 458}, +"57461": {"id": "", "name": "YICHANG", "state": "HK CI", "lat": 30.700, "lon": 111.280, "elev": 134}, +"57494": {"id": "ZHHH", "name": "WUHAN/NANHU", "state": "HK CI", "lat": 30.620, "lon": 114.130, "elev": 23}, +"57516": {"id": "ZUCK", "name": "CHONGQING/CHUNGKING", "state": "CD CI", "lat": 29.520, "lon": 106.480, "elev": 351}, +"57606": {"id": "", "name": "CH'IH-SHUI HO", "state": "CD CI", "lat": 28.480, "lon": 105.930, "elev": 977}, +"57679": {"id": "ZGCS", "name": "CHANGSHA/DATUOPU", "state": "HK CI", "lat": 28.200, "lon": 113.080, "elev": 46}, +"57687": {"id": "", "name": "CHANGSHA", "state": "CI", "lat": 28.120, "lon": 112.780, "elev": 120}, +"57745": {"id": "", "name": "ZHIJIANG", "state": "HK CI", "lat": 27.450, "lon": 109.680, "elev": 273}, +"57749": {"id": "", "name": "HUAIHUA", "state": "HK CI", "lat": 27.570, "lon": 110.000, "elev": 261}, +"57816": {"id": "ZUGY", "name": "GUIYANG", "state": "CD CI", "lat": 26.580, "lon": 106.720, "elev": 1074}, +"57866": {"id": "", "name": "LINGLING", "state": "HK CI", "lat": 26.230, "lon": 111.620, "elev": 174}, +"57916": {"id": "", "name": "LUODIAN", "state": "CD CI", "lat": 25.430, "lon": 106.770, "elev": 441}, +"57957": {"id": "ZGKL", "name": "GUILIN", "state": "GZ CI", "lat": 25.330, "lon": 110.300, "elev": 166}, +"57972": {"id": "", "name": "CHENZHOU", "state": "HK CI", "lat": 25.820, "lon": 113.020, "elev": 185}, +"57993": {"id": "ZSGZ", "name": "GANZHOU", "state": "HK CI", "lat": 25.850, "lon": 114.950, "elev": 125}, +"58027": {"id": "", "name": "XUZHOU", "state": "SH CI", "lat": 34.280, "lon": 117.150, "elev": 42}, +"58150": {"id": "", "name": "SHEYANG/HEDE", "state": "SH CI", "lat": 33.770, "lon": 120.250, "elev": 7}, +"58203": {"id": "", "name": "FUYANG", "state": "HK CI", "lat": 32.930, "lon": 115.830, "elev": 39}, +"58238": {"id": "ZSNJ", "name": "NANJING/NANKING", "state": "SH CI", "lat": 32.000, "lon": 118.800, "elev": 12}, +"58321": {"id": "", "name": "HO FEI", "state": "SH CI", "lat": 31.870, "lon": 117.230, "elev": 36}, +"58362": {"id": "", "name": "SHANGHAI", "state": "SH CI", "lat": 31.400, "lon": 121.470, "elev": 4}, +"58367": {"id": "", "name": "SHANGHAI", "state": "SH CI", "lat": 31.170, "lon": 121.430, "elev": 5}, +"58424": {"id": "", "name": "ANQING", "state": "HK CI", "lat": 30.520, "lon": 117.030, "elev": 20}, +"58457": {"id": "ZSHC", "name": "HANGZHOU/JIANQIAO", "state": "SH CI", "lat": 30.230, "lon": 120.170, "elev": 43}, +"58506": {"id": "", "name": "LU SHAN MOUNTAIN", "state": "HK CI", "lat": 29.580, "lon": 115.980, "elev": 1165}, +"58527": {"id": "", "name": "JINGDEZHEN", "state": "HK CI", "lat": 29.300, "lon": 117.200, "elev": 60}, +"58606": {"id": "ZSCN", "name": "NANCHANG", "state": "HK CI", "lat": 28.600, "lon": 115.920, "elev": 50}, +"58608": {"id": "", "name": "QINGJIANG/ZHANG", "state": "HK CI", "lat": 28.050, "lon": 115.530, "elev": 40}, +"58633": {"id": "", "name": "QU XIAN", "state": "SH CI", "lat": 28.970, "lon": 118.870, "elev": 71}, +"58646": {"id": "", "name": "LISHUI", "state": "SH CI", "lat": 28.450, "lon": 119.920, "elev": 62}, +"58665": {"id": "", "name": "LUQIAO", "state": "SH CI", "lat": 28.650, "lon": 120.080, "elev": 9}, +"58666": {"id": "", "name": "DACHEN DAO", "state": "SH CI", "lat": 28.450, "lon": 121.880, "elev": 84}, +"58725": {"id": "", "name": "SHAOWU", "state": "SH CI", "lat": 27.330, "lon": 117.430, "elev": 192}, +"58765": {"id": "", "name": "NANJI SHAN", "state": "SH CI", "lat": 27.470, "lon": 121.050, "elev": 2}, +"58847": {"id": "ZSFZ", "name": "FUZHOU", "state": "SH CI", "lat": 26.080, "lon": 119.280, "elev": 85}, +"58968": {"id": "", "name": "TAIBEI", "state": "SH CI", "lat": 25.030, "lon": 121.530, "elev": 9}, +"59023": {"id": "", "name": "HECHI/JNCHENGJI", "state": "GZ CI", "lat": 24.700, "lon": 108.050, "elev": 214}, +"59096": {"id": "", "name": "LIANGPING", "state": "GZ CI", "lat": 24.370, "lon": 114.480, "elev": 214}, +"59134": {"id": "ZSAM", "name": "XIAMEN", "state": "SH CI", "lat": 24.480, "lon": 118.080, "elev": 139}, +"59158": {"id": "", "name": "TAIZHONG", "state": "CI", "lat": 24.150, "lon": 120.700, "elev": 78}, +"59211": {"id": "", "name": "BOSE", "state": "GZ CI", "lat": 23.920, "lon": 106.620, "elev": 242}, +"59265": {"id": "", "name": "WUZHOU", "state": "GZ CI", "lat": 23.480, "lon": 111.300, "elev": 120}, +"59280": {"id": "", "name": "PING YUAN", "state": "CI", "lat": 23.670, "lon": 113.050, "elev": 19}, +"59287": {"id": "ZGGG", "name": "GUANGZHOU/BAIYU", "state": "GZ CI", "lat": 23.130, "lon": 113.320, "elev": 8}, +"59293": {"id": "", "name": "HEYUAN", "state": "CI", "lat": 23.800, "lon": 114.730, "elev": 61}, +"59316": {"id": "ZGOW", "name": "SHANTOU", "state": "GZ CI", "lat": 23.400, "lon": 116.680, "elev": 3}, +"59358": {"id": "", "name": "TAINAN", "state": "CI", "lat": 23.000, "lon": 120.230, "elev": 14}, +"59362": {"id": "", "name": "HUALIAN", "state": "CI", "lat": 24.020, "lon": 121.620, "elev": 14}, +"59431": {"id": "ZGNN", "name": "NANNING/WUXU", "state": "GZ CI", "lat": 22.820, "lon": 108.350, "elev": 73}, +"59644": {"id": "", "name": "BEIHAI", "state": "GZ CI", "lat": 21.480, "lon": 109.100, "elev": 16}, +"59663": {"id": "", "name": "YANGJIANG", "state": "GZ CI", "lat": 21.870, "lon": 111.970, "elev": 22}, +"59758": {"id": "ZGHK", "name": "HAIKOU", "state": "GZ CI", "lat": 20.030, "lon": 110.350, "elev": 15}, +"59792": {"id": "", "name": "DONGSHA", "state": "GZ CI", "lat": 20.670, "lon": 116.730, "elev": 6}, +"59948": {"id": "", "name": "YAXIAN/SANYA", "state": "GZ CI", "lat": 18.230, "lon": 109.520, "elev": 7}, +"59981": {"id": "", "name": "XISHA ISLAND", "state": "GZ CI", "lat": 16.830, "lon": 112.330, "elev": 5}, +"60010": {"id": "", "name": "IZANA MOUNTAIN", "state": "CR", "lat": 28.300, "lon": -16.500, "elev": 2368}, +"60018": {"id": "", "name": "TENERIFE-GUIMAR", "state": "CR", "lat": 28.320, "lon": -16.380, "elev": 111}, +"60020": {"id": "", "name": "SANTA CRUZ TENERIFE", "state": "CR", "lat": 28.470, "lon": -16.250, "elev": 46}, +"60096": {"id": "GSVO", "name": "VILLA CISNEROS MIL/DAKHLA", "state": "MC", "lat": 23.710, "lon": -15.930, "elev": 9}, +"60115": {"id": "", "name": "OUJDA", "state": "MC", "lat": 34.780, "lon": -1.950, "elev": 468}, +"60120": {"id": "GMMY", "name": "KENITRA", "state": "MC", "lat": 34.300, "lon": -6.600, "elev": 14}, +"60155": {"id": "GMMC", "name": "CASABLANCA/ANFA", "state": "MC", "lat": 33.570, "lon": -7.670, "elev": 62}, +"60191": {"id": "", "name": "BENI-MELLAL", "state": "MC", "lat": 32.370, "lon": -6.400, "elev": 468}, +"60252": {"id": "GMAD", "name": "AL MASSIRA MC", "state": "MC", "lat": 30.330, "lon": -9.420, "elev": 74}, +"60265": {"id": "", "name": "OUARZAZATE", "state": "MC", "lat": 30.930, "lon": -6.900, "elev": 1131}, +"60360": {"id": "DABB", "name": "ANNABA", "state": "AL", "lat": 36.500, "lon": 7.490, "elev": 4}, +"60390": {"id": "DAAG", "name": "DAR-EL-BEIDA/HOUARI", "state": "AL", "lat": 36.680, "lon": 3.220, "elev": 29}, +"60419": {"id": "DABC", "name": "CONSTANTINE/EL", "state": "AL", "lat": 36.280, "lon": 6.620, "elev": 694}, +"60430": {"id": "", "name": "MILIANA", "state": "AL", "lat": 36.300, "lon": 2.230, "elev": 715}, +"60461": {"id": "", "name": "ORAN PORT", "state": "AL", "lat": 35.700, "lon": -0.650, "elev": 22}, +"60490": {"id": "DAOO", "name": "ORAN/ES SENIA", "state": "AL", "lat": 35.630, "lon": -0.600, "elev": 90}, +"60520": {"id": "", "name": "SIDI BEL ABBES", "state": "AL", "lat": 35.180, "lon": -2.620, "elev": 450}, +"60525": {"id": "DAUB", "name": "BISKRA", "state": "AL", "lat": 34.800, "lon": 5.730, "elev": 87}, +"60549": {"id": "DAAY", "name": "MECHERIA", "state": "AL", "lat": 34.580, "lon": -0.280, "elev": 1149}, +"60550": {"id": "", "name": "EL BAYADH", "state": "AL", "lat": 33.670, "lon": 1.000, "elev": 1341}, +"60559": {"id": "DAUO", "name": "EL OUED/GUEMER", "state": "AL", "lat": 33.500, "lon": 6.120, "elev": 63}, +"60566": {"id": "DAUG", "name": "GHARDAIA/NOUMER", "state": "AL", "lat": 32.380, "lon": 3.820, "elev": 450}, +"60571": {"id": "DAOR", "name": "BECHAR/OUAKDA", "state": "AL", "lat": 31.620, "lon": -2.230, "elev": 773}, +"60580": {"id": "DAUU", "name": "OUARGLA", "state": "AL", "lat": 31.920, "lon": 5.400, "elev": 141}, +"60581": {"id": "DAUH", "name": "HASSI-MESSAOUD", "state": "AL", "lat": 31.670, "lon": 6.150, "elev": 142}, +"60590": {"id": "DAUE", "name": "EL GOLEA", "state": "AL", "lat": 30.570, "lon": 2.870, "elev": 397}, +"60611": {"id": "", "name": "IN AMENAS", "state": "AL", "lat": 28.050, "lon": 9.630, "elev": 562}, +"60620": {"id": "DAUA", "name": "ADRAR/TOUAT", "state": "AL", "lat": 27.880, "lon": -0.280, "elev": 263}, +"60630": {"id": "", "name": "IN SALAH", "state": "AL", "lat": 27.200, "lon": 2.470, "elev": 293}, +"60640": {"id": "DAAP", "name": "ILLIZI/ILLIRANE", "state": "AL", "lat": 26.500, "lon": 8.420, "elev": 558}, +"60656": {"id": "DAOF", "name": "TINDOUF", "state": "AL", "lat": 27.670, "lon": -8.130, "elev": 431}, +"60670": {"id": "", "name": "DJANET", "state": "AL", "lat": 24.550, "lon": 9.470, "elev": 1054}, +"60680": {"id": "", "name": "TAMANRASSET", "state": "AL", "lat": 22.780, "lon": 5.520, "elev": 1378}, +"60710": {"id": "DTKA", "name": "TABARKA", "state": "TS", "lat": 36.950, "lon": 8.750, "elev": 21}, +"60714": {"id": "", "name": "BIZERTE", "state": "TS", "lat": 37.230, "lon": 9.820, "elev": 2}, +"60715": {"id": "DTTA", "name": "TUNIS/CARTHAGE", "state": "TS", "lat": 36.830, "lon": 10.230, "elev": 4}, +"60720": {"id": "", "name": "KELIBIA", "state": "TS", "lat": 37.070, "lon": 11.030, "elev": 4}, +"60725": {"id": "", "name": "SOUK EL ARBA/JENDOUBA", "state": "TS", "lat": 36.480, "lon": 8.800, "elev": 144}, +"60735": {"id": "", "name": "KAIROUAN", "state": "TS", "lat": 35.670, "lon": 10.100, "elev": 68}, +"60740": {"id": "DTTM", "name": "MONASTIR-SKANES", "state": "TS", "lat": 35.670, "lon": 10.750, "elev": 3}, +"60745": {"id": "", "name": "GAFSA", "state": "TS", "lat": 34.420, "lon": 8.820, "elev": 314}, +"60750": {"id": "DTTX", "name": "SFAX/EL-MAOU", "state": "TS", "lat": 34.720, "lon": 10.680, "elev": 23}, +"60760": {"id": "DTTZ", "name": "TOZEUR/NEFTA", "state": "TS", "lat": 33.920, "lon": 8.100, "elev": 93}, +"60765": {"id": "", "name": "GABES", "state": "TS", "lat": 33.880, "lon": 10.100, "elev": 2}, +"60769": {"id": "DTTJ", "name": "DJERBA MELLITA", "state": "TS", "lat": 33.870, "lon": 10.770, "elev": 4}, +"60775": {"id": "", "name": "REMADA", "state": "TS", "lat": 32.320, "lon": 10.400, "elev": 301}, +"61017": {"id": "", "name": "BILMA", "state": "NR", "lat": 18.680, "lon": 12.920, "elev": 357}, +"61024": {"id": "DRZA", "name": "AGADEZ SOUTH (MIL)", "state": "NR", "lat": 16.970, "lon": 7.980, "elev": 502}, +"61052": {"id": "DRRR", "name": "NIAMEY-AERO", "state": "NR", "lat": 13.480, "lon": 2.170, "elev": 234}, +"61075": {"id": "", "name": "BIRNI-N'KONIA", "state": "NR", "lat": 13.800, "lon": 5.250, "elev": 274}, +"61080": {"id": "", "name": "MARADI", "state": "NR", "lat": 13.470, "lon": 7.080, "elev": 369}, +"61090": {"id": "", "name": "ZINDER", "state": "NR", "lat": 13.780, "lon": 8.980, "elev": 460}, +"61202": {"id": "GATS", "name": "TESSALIT", "state": "MI", "lat": 20.200, "lon": 0.980, "elev": 491}, +"61214": {"id": "GAKL", "name": "KIDAL", "state": "MI", "lat": 18.430, "lon": 1.350, "elev": 459}, +"61223": {"id": "GATB", "name": "TOMBOUCTOU/TIMBUKTU", "state": "MI", "lat": 16.720, "lon": -3.000, "elev": 264}, +"61226": {"id": "GAGO", "name": "GAO", "state": "MI", "lat": 16.270, "lon": -0.050, "elev": 260}, +"61233": {"id": "GANK", "name": "NARA/KEIBANE", "state": "MI", "lat": 15.170, "lon": -7.280, "elev": 265}, +"61257": {"id": "GAKY", "name": "KAYES", "state": "MI", "lat": 14.430, "lon": -11.430, "elev": 47}, +"61265": {"id": "GAMB", "name": "MOPTI/BARBE", "state": "MI", "lat": 14.520, "lon": -4.100, "elev": 272}, +"61272": {"id": "GASG", "name": "SEGOU", "state": "MI", "lat": 13.400, "lon": -6.150, "elev": 289}, +"61291": {"id": "GABS", "name": "BAMAKO/SENOU (MIL)", "state": "MI", "lat": 12.530, "lon": -7.950, "elev": 381}, +"61297": {"id": "GASK", "name": "SIKASSO", "state": "MI", "lat": 11.350, "lon": -5.680, "elev": 375}, +"61401": {"id": "", "name": "BIR MOGHREIN", "state": "MT", "lat": 25.230, "lon": -11.620, "elev": 360}, +"61404": {"id": "", "name": "ZOUERATE", "state": "MT", "lat": 22.750, "lon": -12.480, "elev": 343}, +"61415": {"id": "GQPP", "name": "NOUADHIBOU", "state": "MT", "lat": 20.930, "lon": -17.030, "elev": 3}, +"61421": {"id": "", "name": "ATAR", "state": "MT", "lat": 20.520, "lon": -13.070, "elev": 226}, +"61442": {"id": "GQNN", "name": "NOUAKCHOTT", "state": "MT", "lat": 18.100, "lon": -15.950, "elev": 3}, +"61497": {"id": "", "name": "NEMA", "state": "MT", "lat": 16.600, "lon": -7.270, "elev": 246}, +"61498": {"id": "", "name": "KIFFA", "state": "MT", "lat": 16.630, "lon": -11.400, "elev": 115}, +"61520": {"id": "", "name": "BASSIKOUNOU", "state": "MT", "lat": 15.870, "lon": -5.930, "elev": 261}, +"61600": {"id": "GOSS", "name": "SAINT LOUIS", "state": "SG", "lat": 16.050, "lon": -16.450, "elev": 4}, +"61641": {"id": "GOOY", "name": "DAKAR/YOFF", "state": "SG", "lat": 14.730, "lon": -17.500, "elev": 24}, +"61660": {"id": "", "name": "DAKAR-DIASS-AIBD", "state": "SG", "lat": 14.670, "lon": -17.070, "elev": 86}, +"61687": {"id": "GOTT", "name": "TAMBACOUNDA", "state": "SG", "lat": 13.770, "lon": -13.680, "elev": 50}, +"61695": {"id": "GOGG", "name": "ZIGUINCHOR", "state": "SG", "lat": 12.550, "lon": -16.270, "elev": 23}, +"61831": {"id": "", "name": "CONAKRY", "state": "GUINEA", "lat": 9.570, "lon": -13.620, "elev": 49}, +"61901": {"id": "", "name": "ST. HELENA ISLAND", "state": "HE", "lat": -15.930, "lon": -5.670, "elev": 436}, +"61902": {"id": "FHAW", "name": "WIDE AWAKE FIELD", "state": "HE", "lat": -7.970, "lon": -14.400, "elev": 79}, +"61931": {"id": "", "name": "SAO TOME", "state": "PRINCIPE", "lat": 0.380, "lon": 6.720, "elev": 10}, +"61967": {"id": "FJDG", "name": "DIEGO GARCIA", "state": "BT", "lat": -7.300, "lon": 72.400, "elev": 2}, +"61968": {"id": "", "name": "ILES GLORIEUSES", "state": "RE", "lat": -11.550, "lon": 47.280, "elev": 3}, +"61970": {"id": "", "name": "ILE JUAN DE NOVA", "state": "RE", "lat": -17.050, "lon": 42.720, "elev": 10}, +"61972": {"id": "", "name": "ILE EUROPA", "state": "RE", "lat": -22.350, "lon": 40.350, "elev": 12}, +"61974": {"id": "", "name": "AGALEGA", "state": "RE", "lat": -10.430, "lon": 56.750, "elev": 3}, +"61976": {"id": "", "name": "SERGE-FROLOW/TROMELIN", "state": "RE", "lat": -15.880, "lon": 54.520, "elev": 13}, +"61980": {"id": "FMEE", "name": "ST. DENIS/GILLO", "state": "RE", "lat": -20.880, "lon": 55.520, "elev": 25}, +"61986": {"id": "", "name": "ST. BRANDON ISLAND", "state": "RE", "lat": -16.450, "lon": 59.620, "elev": 4}, +"61988": {"id": "FIMR", "name": "RODRIGUES", "state": "RE", "lat": -19.410, "lon": 63.250, "elev": 59}, +"61995": {"id": "", "name": "VACOAS", "state": "MA", "lat": -20.300, "lon": 57.500, "elev": 425}, +"61996": {"id": "", "name": "MARTIN DE VIVIES", "state": "FR", "lat": -37.800, "lon": 77.530, "elev": 29}, +"61997": {"id": "", "name": "ALFRED FAURE/CROZET", "state": "FR", "lat": -46.430, "lon": 51.870, "elev": 146}, +"61998": {"id": "", "name": "PORT-AUX-FRANCAIS", "state": "FR", "lat": -49.350, "lon": 70.250, "elev": 30}, +"62010": {"id": "HLLT", "name": "TRIPOLI INTL ARPT", "state": "LY", "lat": 32.670, "lon": 13.150, "elev": 81}, +"62019": {"id": "", "name": "SIRTE", "state": "LY", "lat": 31.200, "lon": 16.580, "elev": 13}, +"62053": {"id": "HLLB", "name": "BENINA/BENGHAZI", "state": "LY", "lat": 32.080, "lon": 20.270, "elev": 132}, +"62055": {"id": "", "name": "AGEDABIA", "state": "LY", "lat": 30.720, "lon": 20.170, "elev": 6}, +"62059": {"id": "", "name": "DERNA", "state": "LY", "lat": 32.730, "lon": 22.630, "elev": 9}, +"62062": {"id": "", "name": "TOBRUK", "state": "LY", "lat": 32.100, "lon": 23.920, "elev": 50}, +"62103": {"id": "HLTD", "name": "GHADAMES", "state": "LY", "lat": 30.130, "lon": 9.500, "elev": 357}, +"62120": {"id": "", "name": "GARIAT EL SHARGHIA", "state": "LY", "lat": 30.380, "lon": 13.580, "elev": 501}, +"62124": {"id": "HLLS", "name": "SEBHA (AUT)", "state": "LY", "lat": 27.020, "lon": 14.430, "elev": 432}, +"62271": {"id": "", "name": "KUFRA", "state": "LY", "lat": 24.220, "lon": 23.300, "elev": 407}, +"62301": {"id": "", "name": "SIDI BARRANI", "state": "EG", "lat": 31.620, "lon": 25.900, "elev": 24}, +"62305": {"id": "", "name": "SALLOUM PLATEAU", "state": "EG", "lat": 31.570, "lon": 25.300, "elev": 179}, +"62306": {"id": "HEMM", "name": "MERSA MATRUH (MIL)", "state": "EG", "lat": 31.330, "lon": 27.220, "elev": 30}, +"62309": {"id": "", "name": "DABAA", "state": "EG", "lat": 30.930, "lon": 28.470, "elev": 18}, +"62318": {"id": "HEAX", "name": "ALEXANDRIA", "state": "EG", "lat": 31.200, "lon": 29.950, "elev": 7}, +"62333": {"id": "HEPS", "name": "PORT SAID", "state": "EG", "lat": 31.270, "lon": 32.300, "elev": 6}, +"62337": {"id": "HEAR", "name": "EL ARISH", "state": "EG", "lat": 31.080, "lon": 33.830, "elev": 32}, +"62338": {"id": "", "name": "GHAZZA", "state": "EG", "lat": 31.500, "lon": 34.450, "elev": 16}, +"62371": {"id": "", "name": "CAIRO HQ", "state": "EG", "lat": 30.080, "lon": 31.280, "elev": 26}, +"62378": {"id": "", "name": "HELWAN", "state": "EG", "lat": 29.870, "lon": 31.330, "elev": 141}, +"62387": {"id": "", "name": "MINYA", "state": "EG", "lat": 28.080, "lon": 30.730, "elev": 40}, +"62397": {"id": "", "name": "SOHAG", "state": "EG", "lat": 26.570, "lon": 37.700, "elev": 61}, +"62403": {"id": "", "name": "SOUTH OF VALLEY UNIV.", "state": "EG", "lat": 26.200, "lon": 32.750, "elev": 96}, +"62405": {"id": "", "name": "LUXOR", "state": "EG", "lat": 25.670, "lon": 32.700, "elev": 95}, +"62414": {"id": "HESN", "name": "ASWAN (CIV/MIL)", "state": "EG", "lat": 23.970, "lon": 32.780, "elev": 194}, +"62416": {"id": "", "name": "SIWA", "state": "EG", "lat": 29.250, "lon": 25.520, "elev": 3}, +"62417": {"id": "", "name": "SIWA", "state": "EG", "lat": 29.200, "lon": 25.480, "elev": -15}, +"62420": {"id": "", "name": "BAHARIA", "state": "EG", "lat": 28.330, "lon": 28.900, "elev": 128}, +"62423": {"id": "", "name": "FARAFRA", "state": "EG", "lat": 27.050, "lon": 27.970, "elev": 92}, +"62425": {"id": "", "name": "SHARK EL QUINAT", "state": "EG", "lat": 22.470, "lon": 28.700, "elev": 275}, +"62432": {"id": "", "name": "DAKHLA", "state": "EG", "lat": 25.480, "lon": 29.000, "elev": 111}, +"62435": {"id": "", "name": "KHARGA", "state": "EG", "lat": 25.450, "lon": 30.530, "elev": 78}, +"62450": {"id": "", "name": "EL-SUEZ", "state": "EG", "lat": 29.870, "lon": 32.470, "elev": 4}, +"62600": {"id": "", "name": "WADI HALFA", "state": "SU", "lat": 21.920, "lon": 31.350, "elev": 150}, +"62640": {"id": "", "name": "ABU HAMED", "state": "SU", "lat": 19.530, "lon": 33.320, "elev": 312}, +"62641": {"id": "", "name": "PORT SUDAN", "state": "SU", "lat": 19.580, "lon": 37.220, "elev": 3}, +"62650": {"id": "", "name": "DONGOLA", "state": "SU", "lat": 19.170, "lon": 30.480, "elev": 226}, +"62721": {"id": "HSSS", "name": "KHARTOUM (CIV/MIL)", "state": "SU", "lat": 15.600, "lon": 32.550, "elev": 380}, +"62730": {"id": "", "name": "KASSALA", "state": "SU", "lat": 15.470, "lon": 36.400, "elev": 500}, +"62760": {"id": "", "name": "EL FASHER", "state": "SU", "lat": 13.620, "lon": 25.330, "elev": 733}, +"62771": {"id": "", "name": "EL OBEID", "state": "SU", "lat": 13.170, "lon": 30.230, "elev": 574}, +"62772": {"id": "", "name": "KOSTI", "state": "SU", "lat": 13.170, "lon": 32.670, "elev": 381}, +"62805": {"id": "", "name": "DAMAZINE", "state": "SU", "lat": 11.780, "lon": 34.380, "elev": 474}, +"62840": {"id": "", "name": "MALAKAL", "state": "SU", "lat": 9.550, "lon": 31.650, "elev": 387}, +"62880": {"id": "", "name": "WAU", "state": "SU", "lat": 7.700, "lon": 28.020, "elev": 438}, +"63021": {"id": "", "name": "ASMARA", "state": "ERITREA", "lat": 15.280, "lon": 38.920, "elev": 2325}, +"63125": {"id": "", "name": "DJIBOUTI CITY", "state": "DJ", "lat": 11.550, "lon": 43.150, "elev": 19}, +"63450": {"id": "HAAB", "name": "ADDIS ABABA/BOLE", "state": "ET", "lat": 8.980, "lon": 38.800, "elev": 2355}, +"63478": {"id": "", "name": "GODE", "state": "ET", "lat": 5.100, "lon": 44.580, "elev": 320}, +"63533": {"id": "HANG", "name": "NEGHELLI (MIL)", "state": "ET", "lat": 5.280, "lon": 39.750, "elev": 1455}, +"63602": {"id": "HUAR", "name": "ARUA", "state": "UG", "lat": 3.050, "lon": 30.920, "elev": 1211}, +"63612": {"id": "HKLO", "name": "LODWAR", "state": "KN", "lat": 3.120, "lon": 35.620, "elev": 515}, +"63624": {"id": "", "name": "MANDERA", "state": "KN", "lat": 3.930, "lon": 41.870, "elev": 230}, +"63674": {"id": "HUKS", "name": "KASESE", "state": "UG", "lat": 0.180, "lon": 30.100, "elev": 961}, +"63705": {"id": "HUEN", "name": "ENTEBBE INTL ARPT", "state": "UG", "lat": 0.050, "lon": 32.450, "elev": 1155}, +"63710": {"id": "HKKR", "name": "KERICHO", "state": "KN", "lat": -0.370, "lon": 35.350, "elev": 2184}, +"63714": {"id": "", "name": "NAKURU", "state": "KN", "lat": -0.270, "lon": 36.070, "elev": 1871}, +"63723": {"id": "HKGA", "name": "GARISSA", "state": "KN", "lat": -0.470, "lon": 39.630, "elev": 147}, +"63741": {"id": "HKNC", "name": "NAIROBI/DAGORETTI", "state": "KN", "lat": -1.300, "lon": 36.750, "elev": 1798}, +"63756": {"id": "", "name": "MWANZA", "state": "KN", "lat": -2.470, "lon": 32.920, "elev": 1139}, +"63766": {"id": "", "name": "MAKINDU", "state": "KN", "lat": -2.280, "lon": 37.830, "elev": 998}, +"63793": {"id": "", "name": "VOI", "state": "KN", "lat": -3.400, "lon": 38.570, "elev": 560}, +"63799": {"id": "", "name": "MALINDI", "state": "KN", "lat": -3.230, "lon": 40.100, "elev": 24}, +"63801": {"id": "", "name": "KIGOMA", "state": "KN", "lat": -4.880, "lon": 29.630, "elev": 882}, +"63832": {"id": "", "name": "TABORA", "state": "KN", "lat": -5.070, "lon": 32.830, "elev": 1181}, +"63862": {"id": "", "name": "DODOMA", "state": "KN", "lat": -6.170, "lon": 35.770, "elev": 1120}, +"63894": {"id": "", "name": "DAR ES SALAAM", "state": "KN", "lat": -6.880, "lon": 39.200, "elev": 55}, +"63932": {"id": "", "name": "MBEYA", "state": "KN", "lat": -8.930, "lon": 33.470, "elev": 1704}, +"63985": {"id": "FSSS", "name": "SEYCHELLES INTL", "state": "SC", "lat": -4.680, "lon": 55.530, "elev": 4}, +"64210": {"id": "", "name": "KINSHASA/N'DJILI", "state": "ZAIRE", "lat": -4.380, "lon": 15.430, "elev": 309}, +"64387": {"id": "", "name": "KIGALI", "state": "RWANDA", "lat": -1.970, "lon": 30.120, "elev": 1483}, +"64400": {"id": "FCPP", "name": "POINTE-NOIRE", "state": "CG", "lat": -4.820, "lon": 11.900, "elev": 17}, +"64401": {"id": "", "name": "DOLISIE", "state": "CG", "lat": -4.220, "lon": 12.700, "elev": 329}, +"64450": {"id": "FCBB", "name": "BRAZZAVILLE/MAYA-MAYA", "state": "CG", "lat": -4.250, "lon": 15.250, "elev": 316}, +"64453": {"id": "", "name": "DJAMBALA", "state": "CG", "lat": -2.530, "lon": 14.770, "elev": 791}, +"64458": {"id": "FCOU", "name": "OUESSO", "state": "CG", "lat": 1.620, "lon": 16.050, "elev": 352}, +"64459": {"id": "", "name": "IMPFONDO", "state": "CG", "lat": 1.620, "lon": 18.070, "elev": 335}, +"64500": {"id": "FOOL", "name": "LIBREVILLE/LEON MBA", "state": "GO", "lat": 0.450, "lon": 9.420, "elev": 15}, +"64501": {"id": "", "name": "PORT GENTIL", "state": "GO", "lat": -0.700, "lon": 8.750, "elev": 4}, +"64510": {"id": "", "name": "BITAM", "state": "GO", "lat": 2.080, "lon": 11.480, "elev": 600}, +"64565": {"id": "", "name": "MOANDA", "state": "GO", "lat": -1.530, "lon": 13.270, "elev": 571}, +"64600": {"id": "", "name": "BERBERATI", "state": "CE", "lat": 4.220, "lon": 15.780, "elev": 584}, +"64601": {"id": "", "name": "BOUAR", "state": "CE", "lat": 5.970, "lon": 15.630, "elev": 1024}, +"64610": {"id": "", "name": "BOSSANGOA", "state": "CE", "lat": 6.480, "lon": 17.430, "elev": 499}, +"64650": {"id": "FEFF", "name": "BANGUI/M'POKO (MIL)", "state": "CE", "lat": 4.400, "lon": 18.520, "elev": 366}, +"64654": {"id": "", "name": "N'DELE", "state": "CE", "lat": 8.400, "lon": 20.650, "elev": 510}, +"64656": {"id": "", "name": "BANGASSOU", "state": "CE", "lat": 4.730, "lon": 22.830, "elev": 500}, +"64700": {"id": "FTTJ", "name": "NDJAMENA", "state": "CD", "lat": 12.130, "lon": 15.030, "elev": 295}, +"64705": {"id": "FTTS", "name": "BOUSSO", "state": "CD", "lat": 10.480, "lon": 16.720, "elev": 336}, +"64750": {"id": "FTTA", "name": "SARH", "state": "CD", "lat": 9.150, "lon": 18.380, "elev": 365}, +"64756": {"id": "", "name": "ABECHER", "state": "CD", "lat": 13.850, "lon": 20.850, "elev": 545}, +"64860": {"id": "", "name": "GARDOUA", "state": "CM", "lat": 9.330, "lon": 13.380, "elev": 241}, +"64870": {"id": "", "name": "NGAOUNDERE", "state": "CM", "lat": 7.350, "lon": 13.570, "elev": 1114}, +"64893": {"id": "", "name": "KOUNDJA", "state": "CM", "lat": 5.650, "lon": 10.750, "elev": 1208}, +"64900": {"id": "", "name": "YOKO", "state": "CM", "lat": 5.550, "lon": 12.320, "elev": 1027}, +"64910": {"id": "FKKD", "name": "DOUALA (CIV/MIL)", "state": "CM", "lat": 4.000, "lon": 9.730, "elev": 9}, +"64931": {"id": "", "name": "BATOURI", "state": "CM", "lat": 4.470, "lon": 14.370, "elev": 655}, +"64950": {"id": "", "name": "YAOUNDE", "state": "CM", "lat": 3.830, "lon": 11.520, "elev": 751}, +"65046": {"id": "", "name": "KANO", "state": "NIGERIA", "lat": 12.050, "lon": 8.530, "elev": 476}, +"65082": {"id": "", "name": "MAIDUGURI", "state": "NIGERIA", "lat": 11.850, "lon": 13.080, "elev": 354}, +"65123": {"id": "", "name": "MINNA", "state": "NIGERIA", "lat": 9.620, "lon": 6.530, "elev": 260}, +"65202": {"id": "", "name": "LAGOS/OSHODI", "state": "NIGERIA", "lat": 6.550, "lon": 3.350, "elev": 14}, +"65306": {"id": "", "name": "KANDI", "state": "BENIN", "lat": 11.130, "lon": 2.930, "elev": 292}, +"65330": {"id": "", "name": "PARAKOU", "state": "BENIN", "lat": 9.350, "lon": 2.620, "elev": 393}, +"65344": {"id": "", "name": "COTONOU", "state": "BENIN", "lat": 6.350, "lon": 2.380, "elev": 6}, +"65361": {"id": "", "name": "SOKODE", "state": "TOGO", "lat": 8.980, "lon": 1.130, "elev": 403}, +"65387": {"id": "", "name": "LOME", "state": "TOGO", "lat": 6.170, "lon": 1.250, "elev": 21}, +"65418": {"id": "", "name": "TAMALE", "state": "GHANA", "lat": 9.500, "lon": -0.850, "elev": 173}, +"65467": {"id": "", "name": "TAKORADI", "state": "GHANA", "lat": 4.880, "lon": -1.770, "elev": 5}, +"65472": {"id": "", "name": "ACCRA", "state": "GHANA", "lat": 5.600, "lon": -0.170, "elev": 65}, +"65503": {"id": "DHHH", "name": "OUAGADOUGOU", "state": "HV", "lat": 12.350, "lon": -1.520, "elev": 306}, +"65510": {"id": "", "name": "BOBO-DIOULASSO", "state": "HV", "lat": 11.170, "lon": -4.300, "elev": 460}, +"65528": {"id": "", "name": "ODIENNE", "state": "IV", "lat": 9.500, "lon": -7.570, "elev": 433}, +"65536": {"id": "DIKO", "name": "KORHOGO", "state": "IV", "lat": 9.420, "lon": -5.620, "elev": 381}, +"65548": {"id": "DIMN", "name": "MAN", "state": "IV", "lat": 7.380, "lon": -7.520, "elev": 340}, +"65555": {"id": "DIBK", "name": "BOUAKE", "state": "IV", "lat": 7.730, "lon": -5.070, "elev": 376}, +"65578": {"id": "DIAP", "name": "ABIDJAN/PORT BOUET", "state": "IV", "lat": 5.250, "lon": -3.930, "elev": 8}, +"65592": {"id": "DITB", "name": "TABOU", "state": "IV", "lat": 4.420, "lon": -7.370, "elev": 21}, +"65660": {"id": "", "name": "ROBERTS FIELD", "state": "LIBERIA", "lat": 6.250, "lon": -10.350, "elev": 9}, +"66152": {"id": "", "name": "DUNDO", "state": "ANGOLA", "lat": -7.370, "lon": 20.800, "elev": 762}, +"66160": {"id": "", "name": "LUANDA", "state": "ANGOLA", "lat": -8.850, "lon": 13.230, "elev": 84}, +"66422": {"id": "", "name": "MOCAMEDES", "state": "ANGOLA", "lat": -15.200, "lon": 12.150, "elev": 52}, +"67001": {"id": "", "name": "MORONI", "state": "COMOROS", "lat": -11.700, "lon": 43.230, "elev": 6}, +"67002": {"id": "", "name": "HAHAYA AIRPORT", "state": "COMOROS", "lat": -11.530, "lon": 43.270, "elev": 29}, +"67005": {"id": "", "name": "DZAOUDZI", "state": "MAYOTTE", "lat": -12.800, "lon": 45.280, "elev": 7}, +"67009": {"id": "FMNA", "name": "DIEGO-SUAREZ", "state": "MG", "lat": -12.350, "lon": 49.300, "elev": 105}, +"67012": {"id": "", "name": "FASCENE", "state": "MG", "lat": -13.320, "lon": 48.320, "elev": 10}, +"67017": {"id": "", "name": "VOHMAR", "state": "MG", "lat": -13.370, "lon": 50.000, "elev": 6}, +"67023": {"id": "", "name": "SAMBAVA", "state": "MG", "lat": -14.280, "lon": 50.170, "elev": 7}, +"67025": {"id": "FMNH", "name": "ANTALAHA", "state": "MG", "lat": -14.880, "lon": 50.250, "elev": 88}, +"67027": {"id": "FMNM", "name": "MAJUNGA/MAHAJAN", "state": "MG", "lat": -15.670, "lon": 46.350, "elev": 18}, +"67073": {"id": "FMMO", "name": "MAINTIRANO", "state": "MG", "lat": -18.050, "lon": 44.030, "elev": 23}, +"67083": {"id": "FMMI", "name": "ANTANANARIVO/IVATO", "state": "MG", "lat": -18.800, "lon": 47.480, "elev": 1276}, +"67095": {"id": "FMMT", "name": "TAMATAVE/TOAMAS", "state": "MG", "lat": -18.120, "lon": 49.400, "elev": 6}, +"67113": {"id": "FMMH", "name": "MAHANORO", "state": "MG", "lat": -19.830, "lon": 48.800, "elev": 5}, +"67117": {"id": "", "name": "MORONDAVA", "state": "MG", "lat": -20.280, "lon": 44.320, "elev": 8}, +"67131": {"id": "", "name": "MOROMBE", "state": "MG", "lat": -21.750, "lon": 43.370, "elev": 5}, +"67157": {"id": "FMSG", "name": "FARAFANGANA", "state": "MG", "lat": -22.800, "lon": 47.830, "elev": 6}, +"67161": {"id": "", "name": "TULEAR", "state": "MG", "lat": -23.380, "lon": 43.730, "elev": 8}, +"67197": {"id": "FMSD", "name": "FT. DAUPHIN/TOLAGNA", "state": "MG", "lat": -25.030, "lon": 46.950, "elev": 9}, +"67215": {"id": "", "name": "PORTO AMELIA", "state": "MZ", "lat": -12.970, "lon": 40.500, "elev": 50}, +"67231": {"id": "FQCB", "name": "CUAMBA", "state": "MZ", "lat": -14.820, "lon": 36.530, "elev": 607}, +"67237": {"id": "FQNP", "name": "NAMPULA", "state": "MZ", "lat": -15.100, "lon": 39.280, "elev": 441}, +"67283": {"id": "", "name": "QUELIMANE", "state": "MZ", "lat": -17.880, "lon": 36.880, "elev": 11}, +"67297": {"id": "", "name": "BEIRA", "state": "MZ", "lat": -19.800, "lon": 34.900, "elev": 8}, +"67323": {"id": "", "name": "INHAMBANE", "state": "MZ", "lat": -23.870, "lon": 35.380, "elev": 15}, +"67341": {"id": "FQMA", "name": "MAPUTO/MAVALANE", "state": "MZ", "lat": -25.920, "lon": 32.570, "elev": 44}, +"67424": {"id": "", "name": "BOLERO", "state": "MW", "lat": -10.970, "lon": 33.730, "elev": 1100}, +"67441": {"id": "", "name": "MWINILUNGA", "state": "ZB", "lat": -11.750, "lon": 24.430, "elev": 1362}, +"67475": {"id": "", "name": "KASAMA", "state": "ZB", "lat": -10.220, "lon": 31.130, "elev": 1384}, +"67489": {"id": "", "name": "MZUZU", "state": "MW", "lat": -11.450, "lon": 34.020, "elev": 1253}, +"67586": {"id": "FWKI", "name": "KAMUZU INTL AIRPORT", "state": "MW", "lat": -13.780, "lon": 33.770, "elev": 1229}, +"67588": {"id": "", "name": "N.R.C.", "state": "MW", "lat": -14.170, "lon": 33.670, "elev": 1100}, +"67633": {"id": "", "name": "MONGU", "state": "ZB", "lat": -15.250, "lon": 23.150, "elev": 1052}, +"67666": {"id": "FLLC", "name": "LUSAKA CITY AIRPORT", "state": "ZB", "lat": -15.420, "lon": 28.320, "elev": 1280}, +"67693": {"id": "", "name": "CHILEKA", "state": "ZB", "lat": -15.680, "lon": 34.970, "elev": 767}, +"67741": {"id": "FLSS", "name": "SESHEKE", "state": "ZB", "lat": -17.470, "lon": 24.300, "elev": 951}, +"67761": {"id": "", "name": "KARIBA", "state": "ZW", "lat": -16.520, "lon": 28.770, "elev": 510}, +"67774": {"id": "", "name": "HARARE/BELVEDERE", "state": "ZW", "lat": -17.830, "lon": 31.020, "elev": 1472}, +"67775": {"id": "", "name": "HARARE AIRPORT", "state": "ZW", "lat": -17.930, "lon": 31.100, "elev": 1479}, +"67781": {"id": "", "name": "MTOKO", "state": "ZW", "lat": -17.420, "lon": 32.220, "elev": 1259}, +"67797": {"id": "", "name": "MAKANGA", "state": "MW", "lat": -16.520, "lon": 35.150, "elev": 58}, +"67843": {"id": "", "name": "VICTORIA FALLS", "state": "ZW", "lat": -18.100, "lon": 25.850, "elev": 1062}, +"67853": {"id": "", "name": "DETT", "state": "ZW", "lat": -18.630, "lon": 26.850, "elev": 1094}, +"67861": {"id": "", "name": "GOKWE", "state": "ZW", "lat": -18.220, "lon": 28.930, "elev": 1282}, +"67867": {"id": "", "name": "GWELO", "state": "ZW", "lat": -19.450, "lon": 29.850, "elev": 1429}, +"67869": {"id": "", "name": "KADOMA", "state": "ZW", "lat": -18.320, "lon": 29.880, "elev": 1157}, +"67881": {"id": "", "name": "RUSAPE", "state": "ZW", "lat": -18.530, "lon": 32.130, "elev": 1430}, +"67964": {"id": "", "name": "BULAWAYO/GOETZ", "state": "ZW", "lat": -20.150, "lon": 28.620, "elev": 1344}, +"67975": {"id": "", "name": "FORT VICTORIA", "state": "ZW", "lat": -20.070, "lon": 30.870, "elev": 1097}, +"67977": {"id": "", "name": "BUFFALO RNAGE", "state": "ZW", "lat": -21.020, "lon": 31.580, "elev": 429}, +"67991": {"id": "", "name": "BEITBRIDGE", "state": "ZW", "lat": -22.220, "lon": 30.000, "elev": 456}, +"68010": {"id": "", "name": "OKAUKUEJO", "state": "NM", "lat": -19.180, "lon": 15.920, "elev": 1100}, +"68014": {"id": "", "name": "GROOTFONTEIN", "state": "NM", "lat": -19.600, "lon": 18.120, "elev": 1411}, +"68016": {"id": "", "name": "MOWE", "state": "NM", "lat": -19.330, "lon": 12.720, "elev": 0}, +"68024": {"id": "", "name": "GHANZI", "state": "BC", "lat": -21.700, "lon": 21.650, "elev": 1131}, +"68032": {"id": "FBMN", "name": "MAUN", "state": "BC", "lat": -19.980, "lon": 23.420, "elev": 900}, +"68040": {"id": "FBLT", "name": "LETLHAKANE", "state": "BC", "lat": -21.420, "lon": 25.600, "elev": 985}, +"68054": {"id": "", "name": "FRANCISTOWN", "state": "BC", "lat": -21.220, "lon": 27.500, "elev": 991}, +"68098": {"id": "", "name": "WALVIS BAY AIRPORT", "state": "BC", "lat": -22.960, "lon": 14.660, "elev": 150}, +"68110": {"id": "FYWW", "name": "WINDHOEK/EROS(SAAF)", "state": "NM", "lat": -22.570, "lon": 17.100, "elev": 1725}, +"68112": {"id": "", "name": "STRIJDOM AIRPORT", "state": "NM", "lat": -22.480, "lon": 17.470, "elev": 1715}, +"68155": {"id": "FAER", "name": "ELLISRAS", "state": "ZA", "lat": -23.680, "lon": 27.700, "elev": 839}, +"68174": {"id": "FAPB", "name": "PIETERSBURG (SAAF)", "state": "ZA", "lat": -23.870, "lon": 29.450, "elev": 1222}, +"68240": {"id": "FBSK", "name": "SIR SERETSE KHAMA", "state": "BC", "lat": -24.220, "lon": 25.920, "elev": 1005}, +"68242": {"id": "FAMM", "name": "MMABATHO INTL AIRPORT", "state": "ZA", "lat": -25.780, "lon": 25.530, "elev": 1277}, +"68244": {"id": "", "name": "GABERONES", "state": "ZA", "lat": -24.670, "lon": 25.920, "elev": 983}, +"68252": {"id": "", "name": "RUSTENBURG", "state": "ZA", "lat": -25.720, "lon": 27.300, "elev": 1200}, +"68263": {"id": "FAIR", "name": "PRETORIA/IRENE", "state": "ZA", "lat": -25.920, "lon": 28.220, "elev": 1500}, +"68312": {"id": "", "name": "KEETMENSHOOP", "state": "NM", "lat": -26.530, "lon": 18.120, "elev": 1073}, +"68328": {"id": "FBTS", "name": "TSABONG", "state": "BC", "lat": -26.050, "lon": 22.450, "elev": 1000}, +"68406": {"id": "", "name": "KORT DORN", "state": "ZA", "lat": -28.570, "lon": 16.530, "elev": 27}, +"68424": {"id": "FAUP", "name": "UPINGTON/PIERRE", "state": "ZA", "lat": -28.400, "lon": 21.270, "elev": 836}, +"68438": {"id": "", "name": "KIMBERLEY", "state": "ZA", "lat": -28.800, "lon": 24.770, "elev": 1204}, +"68442": {"id": "FABL", "name": "BLOEMFONTEIN/HERTZOG", "state": "ZA", "lat": -29.100, "lon": 26.300, "elev": 1348}, +"68461": {"id": "FABM", "name": "BETHLEHEM AIRPORT", "state": "ZA", "lat": -28.250, "lon": 28.330, "elev": 1682}, +"68512": {"id": "FASB", "name": "SPRINGBOK", "state": "ZA", "lat": -29.670, "lon": 17.870, "elev": 990}, +"68530": {"id": "", "name": "DOUGLAS", "state": "ZA", "lat": -29.070, "lon": 23.750, "elev": 1000}, +"68538": {"id": "FADY", "name": "DEAAR(UA)", "state": "ZA", "lat": -30.670, "lon": 24.020, "elev": 1287}, +"68580": {"id": "", "name": "CEDARA", "state": "ZA", "lat": -29.530, "lon": 30.280, "elev": 1100}, +"68588": {"id": "FADN", "name": "DURBAN/LOUIS BOTHA", "state": "ZA", "lat": -29.970, "lon": 30.950, "elev": 8}, +"68592": {"id": "FALE", "name": "KING SHAKA INTL AIRPORT", "state": "ZA", "lat": -29.610, "lon": 31.120, "elev": 109}, +"68715": {"id": "", "name": "MALMEBURY", "state": "ZA", "lat": -33.470, "lon": 18.720, "elev": 102}, +"68816": {"id": "FACT", "name": "CAPETOWN/DF MALAN", "state": "ZA", "lat": -33.980, "lon": 18.600, "elev": 42}, +"68828": {"id": "", "name": "GEORGE", "state": "ZA", "lat": -33.970, "lon": 22.420, "elev": 218}, +"68842": {"id": "FAPE", "name": "PORT ELIZABETH", "state": "ZA", "lat": -33.980, "lon": 23.600, "elev": 60}, +"68858": {"id": "", "name": "EAST LONDON", "state": "ZA", "lat": -33.030, "lon": 27.830, "elev": 130}, +"68906": {"id": "FAGE", "name": "GOUGH ISLAND", "state": "ZA", "lat": -40.350, "lon": -9.880, "elev": 0}, +"68916": {"id": "", "name": "CAPE POINT", "state": "ZA", "lat": -34.350, "lon": 18.500, "elev": 226}, +"68994": {"id": "FAME", "name": "MARION ISLAND", "state": "ZA", "lat": -46.880, "lon": 37.870, "elev": 22}, +"69002": {"id": "KHGT", "name": "JOLON/HUNTER LIGGETT MIL,CA US", "state": "317", "lat": 0.000, "lon": 0.000, "elev": 36}, +"69003": {"id": "", "name": "WEST VERTICLE", "state": "UT US", "lat": 40.200, "lon": -111.220, "elev": 1400}, +"69007": {"id": "KOAR", "name": "FORT ORD/FRITZS", "state": "CA US", "lat": 36.680, "lon": -121.770, "elev": 41}, +"69008": {"id": "KSYL", "name": "CAMP ROBERTS", "state": "CA US", "lat": 35.750, "lon": -120.700, "elev": 220}, +"69014": {"id": "KNZJ", "name": "EL TORO/MCAS", "state": "US", "lat": 33.670, "lon": -117.730, "elev": 116}, +"69016": {"id": "KNTK", "name": "TUSTIN/MCAF", "state": "US", "lat": 33.680, "lon": -117.800, "elev": 17}, +"69055": {"id": "KQCA", "name": "GLOGOVAC", "state": "KOSOVO", "lat": 42.370, "lon": 20.550, "elev": 609}, +"69061": {"id": "KQDI", "name": "DAKOVICA NEW AIRPORT", "state": "KOSOVO", "lat": 42.250, "lon": 20.260, "elev": 399}, +"69066": {"id": "KQDN", "name": "ORAHOVAC", "state": "KOSOVO", "lat": 42.180, "lon": 20.360, "elev": 375}, +"69089": {"id": "KQHD", "name": "FARKE", "state": "ALBANIA", "lat": 41.320, "lon": 19.870, "elev": 258}, +"69110": {"id": "KQAA", "name": "PRIZREN", "state": "KOSOVO", "lat": 42.130, "lon": 20.450, "elev": 460}, +"69111": {"id": "KQAB", "name": "TOPLICANE", "state": "KOSOVO", "lat": 42.300, "lon": 20.800, "elev": 366}, +"69145": {"id": "KQDF", "name": "TIRANA USAFE", "state": "ALBANIA", "lat": 41.420, "lon": 19.720, "elev": 89}, +"69149": {"id": "KQKB", "name": "TASZAR USAREUR FWD", "state": "HU", "lat": 46.420, "lon": 17.920, "elev": 160}, +"69154": {"id": "KQKH", "name": "KAPOSJULAK USAREUR FWD", "state": "HU", "lat": 46.370, "lon": 17.820, "elev": 145}, +"69167": {"id": "KQKU", "name": "CAMP ZENICA", "state": "BH", "lat": 44.130, "lon": 17.550, "elev": 316}, +"69170": {"id": "KQKX", "name": "PRILEP MMU", "state": "MACEDONIA", "lat": 41.220, "lon": 21.430, "elev": 640}, +"69189": {"id": "KQLP", "name": "PLANA", "state": "KOSOVO", "lat": 42.510, "lon": 20.560, "elev": 529}, +"69198": {"id": "KQLY", "name": "HQ KFOR PRISTINA", "state": "KOSOVO", "lat": 42.400, "lon": 21.090, "elev": 576}, +"69312": {"id": "KQHL", "name": "CAMP DOAH", "state": "KW", "lat": 29.370, "lon": 47.800, "elev": 12}, +"69635": {"id": "KQOI", "name": "TOMISLAV GRAD", "state": "BH", "lat": 43.430, "lon": 17.150, "elev": 903}, +"69637": {"id": "KQOK", "name": "GARADZE", "state": "BH", "lat": 43.400, "lon": 18.590, "elev": 343}, +"69638": {"id": "KQOL", "name": "KUPRES", "state": "BH", "lat": 44.000, "lon": 17.170, "elev": 1183}, +"69640": {"id": "KQON", "name": "STOLAC", "state": "BH", "lat": 43.050, "lon": 17.580, "elev": 64}, +"69641": {"id": "KQOO", "name": "JUBUSKI", "state": "BH", "lat": 42.130, "lon": 17.320, "elev": 98}, +"69663": {"id": "KQQK", "name": "DOBOJ OJG", "state": "BH", "lat": 44.440, "lon": 18.060, "elev": 182}, +"69665": {"id": "KQQM", "name": "CAMP MCGOVERN", "state": "BH", "lat": 44.530, "lon": 18.410, "elev": 90}, +"69666": {"id": "KQQN", "name": "COMMANCHE BASE", "state": "BH", "lat": 44.280, "lon": 18.400, "elev": 228}, +"69679": {"id": "KQUA", "name": "RAJILOVAK", "state": "BH", "lat": 43.520, "lon": 18.180, "elev": 493}, +"69683": {"id": "KQUE", "name": "CAMP COLT", "state": "BH", "lat": 45.010, "lon": 18.190, "elev": 93}, +"69686": {"id": "KQUH", "name": "CAMP ABLE SENTRY", "state": "MACEDONIA", "lat": 41.580, "lon": 21.380, "elev": 238}, +"69704": {"id": "KQUZ", "name": "AL KHARJI", "state": "SD", "lat": 24.130, "lon": 47.930, "elev": 497}, +"69756": {"id": "KQIZ", "name": "AL JABER", "state": "KW", "lat": 28.930, "lon": 47.780, "elev": 157}, +"70026": {"id": "PBRW", "name": "BARROW/POST-ROG", "state": "AK US", "lat": 71.300, "lon": -156.780, "elev": 3}, +"70086": {"id": "PBTI", "name": "BARTER ISLAND", "state": "AK US", "lat": 70.130, "lon": -143.630, "elev": 15}, +"70117": {"id": "PATC", "name": "TIN CITY AFS", "state": "AK US", "lat": 65.570, "lon": -167.920, "elev": 83}, +"70133": {"id": "POTZ", "name": "KOTZEBUE", "state": "AK US", "lat": 66.870, "lon": -162.630, "elev": 6}, +"70200": {"id": "POME", "name": "NOME", "state": "AK US", "lat": 64.500, "lon": -165.430, "elev": 6}, +"70219": {"id": "PBET", "name": "BETHEL ARPT", "state": "AK US", "lat": 60.780, "lon": -161.800, "elev": 45}, +"70231": {"id": "PMCG", "name": "MCGRATH", "state": "AK US", "lat": 62.970, "lon": -155.620, "elev": 103}, +"70261": {"id": "PFAI", "name": "FAIRBANKS", "state": "AK US", "lat": 64.820, "lon": -147.870, "elev": 138}, +"70266": {"id": "PABG", "name": "FORT GREELY", "state": "AK US", "lat": 63.970, "lon": -145.700, "elev": 398}, +"70270": {"id": "PAFR", "name": "FORT RICHARDSON", "state": "AK US", "lat": 61.270, "lon": -149.650, "elev": 115}, +"70273": {"id": "PANC", "name": "ANCHORAGE INTL", "state": "AK US", "lat": 61.170, "lon": -150.020, "elev": 40}, +"70308": {"id": "PSNP", "name": "SAINT PAUL IS.", "state": "AK US", "lat": 57.150, "lon": -170.220, "elev": 8}, +"70316": {"id": "PCDB", "name": "COLD BAY", "state": "AK US", "lat": 55.200, "lon": -162.720, "elev": 31}, +"70326": {"id": "PAKN", "name": "KING SALMON", "state": "AK US", "lat": 58.680, "lon": -156.650, "elev": 14}, +"70350": {"id": "PADQ", "name": "KODIAK", "state": "AK US", "lat": 57.730, "lon": -152.520, "elev": 33}, +"70361": {"id": "PYAK", "name": "YAKUTAT", "state": "AK US", "lat": 59.500, "lon": -139.680, "elev": 9}, +"70398": {"id": "PANN", "name": "ANNETTE ISLAND", "state": "AK US", "lat": 55.030, "lon": -131.570, "elev": 33}, +"70414": {"id": "PASY", "name": "SHEMYA AFB", "state": "AK US", "lat": 52.720, "lon": -174.120, "elev": 30}, +"70454": {"id": "PADK", "name": "ADAK NS.", "state": "AK US", "lat": 51.880, "lon": -176.650, "elev": 6}, +"71043": {"id": "CYVQ", "name": "NORMAN WELLS", "state": "NT CN", "lat": 65.280, "lon": -126.800, "elev": 73}, +"71069": {"id": "CYZH", "name": "SLAVE LAKE", "state": "AB CN", "lat": 55.280, "lon": -114.770, "elev": 583}, +"71072": {"id": "CYMD", "name": "MOULD BAY", "state": "NT CN", "lat": 76.230, "lon": -119.330, "elev": 12}, +"71078": {"id": "CYYL", "name": "LYNN LAKE AIRPORT", "state": "MB CN", "lat": 56.870, "lon": -101.080, "elev": 357}, +"71079": {"id": "CYTH", "name": "THOMPSON AIRPORT", "state": "MB CN", "lat": 55.800, "lon": -97.870, "elev": 218}, +"71081": {"id": "CYUX", "name": "HALL BEACH", "state": "NT CN", "lat": 68.780, "lon": -81.250, "elev": 8}, +"71082": {"id": "CWLT", "name": "ALERT BAY", "state": "NT CN", "lat": 82.500, "lon": -62.330, "elev": 63}, +"71109": {"id": "CYZT", "name": "PORT HARDY", "state": "BC CN", "lat": 50.680, "lon": -127.370, "elev": 22}, +"71115": {"id": "CWJV", "name": "VERNON", "state": "BC CN", "lat": 50.230, "lon": -119.280, "elev": 555}, +"71119": {"id": "CWSE", "name": "EDMONTON", "state": "AB CN", "lat": 53.550, "lon": -114.100, "elev": 766}, +"71120": {"id": "CYOD", "name": "COLD LAKE", "state": "AB CN", "lat": 54.420, "lon": -110.280, "elev": 544}, +"71121": {"id": "CYED", "name": "EDMONTON", "state": "AB CN", "lat": 53.670, "lon": -113.450, "elev": 688}, +"71124": {"id": "CWIQ", "name": "PRIMROSE LAKE", "state": "AB CN", "lat": 54.750, "lon": -110.050, "elev": 702}, +"71126": {"id": "CZED", "name": "STONY PLAIN", "state": "AB CN", "lat": 53.050, "lon": -113.110, "elev": 766}, +"71129": {"id": "CYKY", "name": "KINDERSLEY", "state": "SK CN", "lat": 51.520, "lon": -109.170, "elev": 694}, +"71135": {"id": "CWKO", "name": "ROCKGLEN (MARS)", "state": "SA CN", "lat": 49.170, "lon": -105.980, "elev": 917}, +"71145": {"id": "CYIV", "name": "ISLAND LAKE", "state": "MN CN", "lat": 53.850, "lon": -94.650, "elev": 236}, +"71146": {"id": "CWLZ", "name": "CANDLE LAKE", "state": "SA CN", "lat": 53.730, "lon": -105.270, "elev": 503}, +"71197": {"id": "CWZB", "name": "PORT BASQUES", "state": "NF CN", "lat": 47.570, "lon": -59.180, "elev": 40}, +"71203": {"id": "CYLW", "name": "KELOWA", "state": "BC CN", "lat": 49.970, "lon": -119.380, "elev": 430}, +"71399": {"id": "CWOS", "name": "SHELBURNE", "state": "NS CN", "lat": 43.720, "lon": -65.250, "elev": 30}, +"71431": {"id": "CWNC", "name": "COBOURG", "state": "ON CN", "lat": 43.950, "lon": -78.170, "elev": 78}, +"71486": {"id": "CWJW", "name": "JASPER WARDEN", "state": "AB CN", "lat": 52.930, "lon": -118.320, "elev": 1020}, +"71488": {"id": "CYKJ", "name": "KEY LAKE", "state": "SA CN", "lat": 57.250, "lon": -105.620, "elev": 509}, +"71569": {"id": "CXBK", "name": "BRATT'S LAKE", "state": "SA CN", "lat": 50.200, "lon": -104.700, "elev": 580}, +"71600": {"id": "CWSA", "name": "SABLE ISLAND", "state": "NS CN", "lat": 43.930, "lon": -60.020, "elev": 4}, +"71603": {"id": "CWQI", "name": "YARMOUTH", "state": "NS CN", "lat": 43.830, "lon": -66.080, "elev": 41}, +"71621": {"id": "CYTR", "name": "TRENTON", "state": "ON CN", "lat": 44.120, "lon": -77.530, "elev": 86}, +"71625": {"id": "CYWA", "name": "PETAWAWA", "state": "ON CN", "lat": 45.950, "lon": -77.320, "elev": 130}, +"71627": {"id": "CYUL", "name": "MONTREAL", "state": "QB CN", "lat": 45.470, "lon": -73.730, "elev": 36}, +"71634": {"id": "CYZR", "name": "SARNIA", "state": "CN", "lat": 43.000, "lon": -82.300, "elev": 181}, +"71638": {"id": "CWTO", "name": "TORONTO AES HQ", "state": "ON CN", "lat": 43.780, "lon": -79.470, "elev": 187}, +"71701": {"id": "CYCX", "name": "GAGETOWN", "state": "NB CN", "lat": 45.830, "lon": -66.430, "elev": 51}, +"71712": {"id": "CWBZ", "name": "ST ANICET", "state": "QB CN", "lat": 45.120, "lon": -74.280, "elev": 49}, +"71716": {"id": "CYOY", "name": "VALCARTIER", "state": "QB CN", "lat": 46.900, "lon": -71.500, "elev": 168}, +"71722": {"id": "CWMW", "name": "MANIWAKI", "state": "QB CN", "lat": 46.380, "lon": -75.970, "elev": 170}, +"71799": {"id": "CYYJ", "name": "VICTORIA INTL AIRPORT", "state": "BC CN", "lat": 48.650, "lon": -123.430, "elev": 19}, +"71800": {"id": "CWRA", "name": "CAPE RACE (MARS)", "state": "NF CN", "lat": 46.650, "lon": -53.070, "elev": 28}, +"71801": {"id": "CYYT", "name": "SAINT JOHN'S", "state": "NF CN", "lat": 47.620, "lon": -52.730, "elev": 140}, +"71802": {"id": "CWDS", "name": "ST LAWRENCE", "state": "NF CN", "lat": 46.920, "lon": -55.380, "elev": 49}, +"71811": {"id": "CYZV", "name": "SEPT-ILES", "state": "QB CN", "lat": 50.220, "lon": -66.270, "elev": 55}, +"71815": {"id": "CYJT", "name": "STEPHENVILLE", "state": "NF CN", "lat": 48.530, "lon": -58.550, "elev": 26}, +"71816": {"id": "CYYR", "name": "GOOSE BAY", "state": "NF CN", "lat": 53.300, "lon": -60.370, "elev": 44}, +"71823": {"id": "CZAG", "name": "LA GRANDE", "state": "QB CN", "lat": 53.750, "lon": -73.670, "elev": 313}, +"71826": {"id": "CWXP", "name": "NITCHEQUON", "state": "QB CN", "lat": 53.200, "lon": -70.900, "elev": 539}, +"71836": {"id": "CWZC", "name": "MOOSONEE", "state": "ON CN", "lat": 51.270, "lon": -80.650, "elev": 10}, +"71843": {"id": "", "name": "WINNIPEG UA", "state": "MB CN", "lat": 49.880, "lon": -97.130, "elev": 251}, +"71845": {"id": "CYPL", "name": "PICKLE LAKE", "state": "ON CN", "lat": 51.470, "lon": -90.200, "elev": 373}, +"71848": {"id": "CWTL", "name": "BIG TROUT LAKE", "state": "ON CN", "lat": 53.830, "lon": -89.870, "elev": 222}, +"71851": {"id": "CYPG", "name": "PORTAGE SOUTPORT", "state": "MN CN", "lat": 49.900, "lon": -98.270, "elev": 269}, +"71852": {"id": "CYWG", "name": "WINNIPEG INTL AIRPORT", "state": "MB CN", "lat": 49.900, "lon": -97.230, "elev": 239}, +"71853": {"id": "CWLO", "name": "SHILO", "state": "MN CN", "lat": 49.780, "lon": -99.630, "elev": 382}, +"71855": {"id": "CYDN", "name": "DAUPHIN LAKE", "state": "MN CN", "lat": 51.100, "lon": -100.050, "elev": 304}, +"71861": {"id": "CWIK", "name": "BROADVIEW", "state": "SA CN", "lat": 50.380, "lon": -102.680, "elev": 602}, +"71863": {"id": "CYQR", "name": "REGINA", "state": "SK CN", "lat": 50.420, "lon": -104.670, "elev": 577}, +"71866": {"id": "CYXE", "name": "SASKATOON", "state": "AB CN", "lat": 52.170, "lon": -106.700, "elev": 504}, +"71867": {"id": "CYQD", "name": "THE PAS", "state": "MN CN", "lat": 53.970, "lon": -101.100, "elev": 271}, +"71869": {"id": "CYPA", "name": "PRINCE ALBERT", "state": "MN CN", "lat": 53.220, "lon": -105.680, "elev": 428}, +"71874": {"id": "CYQL", "name": "LETHBRIDGE", "state": "AB CN", "lat": 49.620, "lon": -112.800, "elev": 929}, +"71876": {"id": "CYOW", "name": "N BATTLEFORD", "state": "SA CN", "lat": 52.770, "lon": -108.250, "elev": 548}, +"71877": {"id": "CYYC", "name": "CALGARY", "state": "AB CN", "lat": 51.120, "lon": -114.020, "elev": 1084}, +"71878": {"id": "CYQF", "name": "RED DEER AIRPORT", "state": "AB CN", "lat": 52.180, "lon": -113.900, "elev": 905}, +"71892": {"id": "CYVR", "name": "VANCOUVER", "state": "BC CN", "lat": 49.180, "lon": -123.180, "elev": 3}, +"71893": {"id": "CYQQ", "name": "COMOX", "state": "BC CN", "lat": 49.720, "lon": -124.900, "elev": 26}, +"71896": {"id": "CYXS", "name": "PRINCE GEORGE", "state": "BC CN", "lat": 53.880, "lon": -122.670, "elev": 691}, +"71906": {"id": "CYVP", "name": "KUUJJUAQ", "state": "QB CN", "lat": 58.100, "lon": -68.420, "elev": 37}, +"71907": {"id": "CWPH", "name": "INUKJUAK", "state": "QB CN", "lat": 58.450, "lon": -78.120, "elev": 3}, +"71908": {"id": "CZXS", "name": "PRINCE GEORGE", "state": "CN", "lat": 53.900, "lon": -122.800, "elev": 601}, +"71909": {"id": "CYFB", "name": "FROBISHER BAY/IQALUIT", "state": "NT CN", "lat": 63.750, "lon": -68.530, "elev": 34}, +"71913": {"id": "CYYQ", "name": "CHURCHILL", "state": "MN CN", "lat": 58.750, "lon": -94.070, "elev": 29}, +"71915": {"id": "CYZS", "name": "CORAL HARBOUR", "state": "NT CN", "lat": 64.200, "lon": -83.370, "elev": 64}, +"71917": {"id": "CWEU", "name": "EUREKA", "state": "NT CN", "lat": 79.980, "lon": -85.930, "elev": 10}, +"71920": {"id": "CWFN", "name": "CREE LAKE", "state": "SK CN", "lat": 57.350, "lon": -107.130, "elev": 499}, +"71924": {"id": "CYRB", "name": "RESOLUTE BAY", "state": "NT CN", "lat": 74.720, "lon": -94.980, "elev": 67}, +"71925": {"id": "CYCB", "name": "CAMBRIDGE BAY", "state": "NT CN", "lat": 69.100, "lon": -105.120, "elev": 27}, +"71926": {"id": "CYBK", "name": "BAKER LAKE", "state": "NT CN", "lat": 64.300, "lon": -96.080, "elev": 18}, +"71932": {"id": "CYMM", "name": "FORT MCMURRAY", "state": "AB CN", "lat": 56.650, "lon": -111.220, "elev": 369}, +"71934": {"id": "CYSM", "name": "FORT SMITH", "state": "NT CN", "lat": 60.020, "lon": -111.950, "elev": 203}, +"71945": {"id": "CYYE", "name": "FORT NELSON", "state": "BC CN", "lat": 58.830, "lon": -122.600, "elev": 382}, +"71954": {"id": "CYUB", "name": "TUKTOYAKTUK", "state": "NT CN", "lat": 69.450, "lon": -133.020, "elev": 5}, +"71957": {"id": "CYEV", "name": "INUVIK", "state": "NT CN", "lat": 68.300, "lon": -133.480, "elev": 68}, +"71964": {"id": "CYXY", "name": "WHITEHORSE", "state": "YK CN", "lat": 60.720, "lon": -135.070, "elev": 703}, +"72201": {"id": "KEYW", "name": "KEY WEST INTL", "state": "FL US", "lat": 24.550, "lon": -81.750, "elev": 6}, +"72202": {"id": "KMIA", "name": "MIAMI INTL", "state": "FL US", "lat": 25.820, "lon": -80.280, "elev": 4}, +"72203": {"id": "KPBI", "name": "WEST PALM BEACH", "state": "FL US", "lat": 26.680, "lon": -80.120, "elev": 6}, +"72206": {"id": "KJAX", "name": "JACKSONVILLE", "state": "FL US", "lat": 30.430, "lon": -81.620, "elev": 9}, +"72207": {"id": "KGRF", "name": "FORT LEWIS", "state": "WA US", "lat": 47.120, "lon": -122.550, "elev": 92}, +"72208": {"id": "KCHS", "name": "CHARLESTON", "state": "SC US", "lat": 32.900, "lon": -80.030, "elev": 14}, +"72210": {"id": "KTBW", "name": "TAMPA BAY", "state": "FL US", "lat": 27.700, "lon": -82.400, "elev": 13}, +"72213": {"id": "KAYS", "name": "WAYCROSS", "state": "GA US", "lat": 31.250, "lon": -82.400, "elev": 46}, +"72214": {"id": "KTLH", "name": "TALLAHASSEE", "state": "FL US", "lat": 30.400, "lon": -84.350, "elev": 18}, +"72215": {"id": "KFFC", "name": "ATLANTA/PEACHTREE CITY", "state": "GA US", "lat": 33.370, "lon": -84.570, "elev": 264}, +"72220": {"id": "KAQQ", "name": "APALACHICOLA", "state": "FL US", "lat": 29.730, "lon": -85.030, "elev": 6}, +"72221": {"id": "KVPS", "name": "EGLIN AFB/VALPARAISO", "state": "FL US", "lat": 30.480, "lon": -86.530, "elev": 26}, +"72225": {"id": "KLSF", "name": "FORT BENNING", "state": "GA US", "lat": 32.330, "lon": -84.830, "elev": 130}, +"72228": {"id": "KTCL", "name": "TUSCALOOSA/MUNICIPAL", "state": "AL US", "lat": 33.220, "lon": -87.620, "elev": 49}, +"72229": {"id": "KCKL", "name": "CENTREVILLE", "state": "AL US", "lat": 32.900, "lon": -87.250, "elev": 140}, +"72230": {"id": "KBMX", "name": "BIRMINGHAM/SHELBY CO.", "state": "AL US", "lat": 33.170, "lon": -86.770, "elev": 178}, +"72232": {"id": "KBVE", "name": "BOOTHVILLE", "state": "LA US", "lat": 29.330, "lon": -89.400, "elev": 0}, +"72233": {"id": "KSIL", "name": "SLIDELL", "state": "LA US", "lat": 30.250, "lon": -89.770, "elev": 3}, +"72235": {"id": "KJAN", "name": "JACKSON", "state": "MS US", "lat": 32.320, "lon": -90.080, "elev": 101}, +"72239": {"id": "KPOE", "name": "FORT POLK", "state": "LA US", "lat": 31.050, "lon": -93.200, "elev": 100}, +"72240": {"id": "KLCH", "name": "LAKE CHARLES", "state": "LA US", "lat": 30.120, "lon": -93.220, "elev": 10}, +"72247": {"id": "KGGG", "name": "LONGVIEW", "state": "TX US", "lat": 32.350, "lon": -94.650, "elev": 124}, +"72248": {"id": "KSHV", "name": "SHREVEPORT WSO", "state": "LA US", "lat": 32.450, "lon": -93.470, "elev": 84}, +"72249": {"id": "KFWD", "name": "FORT WORTH", "state": "TX US", "lat": 32.800, "lon": -97.300, "elev": 196}, +"72250": {"id": "KBRO", "name": "BROWNSVILLE", "state": "TX US", "lat": 25.900, "lon": -97.430, "elev": 6}, +"72251": {"id": "KCRP", "name": "CORPUS CHRISTI", "state": "TX US", "lat": 27.770, "lon": -97.500, "elev": 13}, +"72255": {"id": "KVCT", "name": "VICTORIA", "state": "TX US", "lat": 28.850, "lon": -96.920, "elev": 36}, +"72257": {"id": "KHLR", "name": "FT HOOD AAF/KIL", "state": "TX US", "lat": 31.150, "lon": -97.720, "elev": 286}, +"72260": {"id": "KSEP", "name": "STEPHENVILLE", "state": "TX US", "lat": 32.220, "lon": -98.180, "elev": 402}, +"72261": {"id": "KDRT", "name": "DEL RIO", "state": "TX US", "lat": 29.370, "lon": -100.920, "elev": 313}, +"72265": {"id": "KMAF", "name": "MIDLAND REGIONAL", "state": "TX US", "lat": 31.950, "lon": -102.180, "elev": 872}, +"72269": {"id": "K2C2", "name": "WHITE SANDS", "state": "AZ US", "lat": 32.240, "lon": -106.220, "elev": 1207}, +"72270": {"id": "KELP", "name": "EL PASO", "state": "TX US", "lat": 31.800, "lon": -106.400, "elev": 1194}, +"72273": {"id": "KFHU", "name": "FORT HUACHUCA", "state": "AZ US", "lat": 31.600, "lon": -110.350, "elev": 1438}, +"72274": {"id": "KTUS", "name": "TUCSON", "state": "AZ US", "lat": 32.120, "lon": -110.930, "elev": 779}, +"72280": {"id": "KNYL", "name": "YUMA", "state": "AZ US", "lat": 32.650, "lon": -114.620, "elev": 50}, +"72286": {"id": "KRIV", "name": "RIVERSIDE (MARCH)", "state": "CA US", "lat": 33.880, "lon": -117.270, "elev": 469}, +"72290": {"id": "KNKX", "name": "MIRAMAR NAS", "state": "CA US", "lat": 32.730, "lon": -117.170, "elev": 9}, +"72291": {"id": "KNSI", "name": "SAN NICOLAS IS.", "state": "CA US", "lat": 33.250, "lon": -119.450, "elev": 154}, +"72293": {"id": "KMYF", "name": "SAN DIEGO", "state": "CA US", "lat": 32.820, "lon": -117.130, "elev": 124}, +"72295": {"id": "KLAX", "name": "LOS ANGELES", "state": "CA US", "lat": 33.930, "lon": -118.400, "elev": 34}, +"72304": {"id": "KHAT", "name": "CAPE HATTERAS", "state": "NC US", "lat": 35.270, "lon": -75.550, "elev": 3}, +"72305": {"id": "KMHX", "name": "MOREHEAD/NEWPORT", "state": "NC US", "lat": 34.700, "lon": -76.800, "elev": 11}, +"72309": {"id": "KNKT", "name": "CHERRY POINT", "state": "NC US", "lat": 34.900, "lon": -76.880, "elev": 9}, +"72311": {"id": "KAHN", "name": "ATHENS", "state": "GA US", "lat": 33.950, "lon": -83.320, "elev": 247}, +"72317": {"id": "KGSO", "name": "GREENSBORO-HIGH", "state": "NC US", "lat": 36.080, "lon": -79.950, "elev": 270}, +"72318": {"id": "KRNK", "name": "ROANOKE", "state": "VA US", "lat": 37.200, "lon": -80.420, "elev": 648}, +"72327": {"id": "KBNA", "name": "NASHVILLE", "state": "TN US", "lat": 36.120, "lon": -86.680, "elev": 180}, +"72340": {"id": "KLZK", "name": "LITTLE ROCK", "state": "AR US", "lat": 34.130, "lon": -92.250, "elev": 165}, +"72349": {"id": "KUMN", "name": "MONETT", "state": "MO US", "lat": 36.880, "lon": -93.900, "elev": 437}, +"72353": {"id": "KOKC", "name": "OKLAHOMA CITY", "state": "OK US", "lat": 35.400, "lon": -97.600, "elev": 397}, +"72355": {"id": "KFSI", "name": "FORT SILL", "state": "OK US", "lat": 34.650, "lon": -98.400, "elev": 362}, +"72357": {"id": "KOUN", "name": "NORMAN", "state": "OK US", "lat": 35.230, "lon": -97.470, "elev": 362}, +"72363": {"id": "KAMA", "name": "AMARILLO", "state": "TX US", "lat": 35.230, "lon": -101.700, "elev": 1099}, +"72364": {"id": "KEPZ", "name": "EL PASO", "state": "TX US", "lat": 31.900, "lon": -106.700, "elev": 1377}, +"72365": {"id": "KABQ", "name": "ALBUQUERQUE", "state": "NM US", "lat": 35.050, "lon": -106.600, "elev": 1613}, +"72374": {"id": "KINW", "name": "WINSLOW", "state": "AZ US", "lat": 35.020, "lon": -110.730, "elev": 1488}, +"72376": {"id": "KFSX", "name": "FLAGSTAFF", "state": "AZ US", "lat": 35.230, "lon": -111.820, "elev": 2192}, +"72381": {"id": "KEDW", "name": "EDWARDS AFB", "state": "CA US", "lat": 34.900, "lon": -117.880, "elev": 702}, +"72382": {"id": "KPMD", "name": "PALMDALE", "state": "CA US", "lat": 34.630, "lon": -118.080, "elev": 774}, +"72383": {"id": "KSDB", "name": "SANDBERG", "state": "CA US", "lat": 34.750, "lon": -118.730, "elev": 1379}, +"72387": {"id": "KDRA", "name": "MERCURY/DESERT ROCK", "state": "NV US", "lat": 36.620, "lon": -116.020, "elev": 1009}, +"72388": {"id": "KVEF", "name": "LAS VEGAS", "state": "NV US", "lat": 36.050, "lon": -115.180, "elev": 697}, +"72389": {"id": "KFAT", "name": "FRESNO/AIR TE", "state": "NV US", "lat": 36.770, "lon": -119.720, "elev": 100}, +"72391": {"id": "KNTD", "name": "POINT MAGU", "state": "CA US", "lat": 34.120, "lon": -119.120, "elev": 4}, +"72393": {"id": "KVBG", "name": "VANDENBERG AFB", "state": "CA US", "lat": 34.730, "lon": -120.580, "elev": 112}, +"72394": {"id": "KSMX", "name": "SANTA MARIA", "state": "CA US", "lat": 34.900, "lon": -120.450, "elev": 73}, +"72402": {"id": "KWAL", "name": "WALLOPS ISLAND", "state": "VA US", "lat": 37.850, "lon": -75.480, "elev": 3}, +"72403": {"id": "KIAD", "name": "WASHINGTON/DULLES", "state": "VA US", "lat": 38.950, "lon": -77.450, "elev": 98}, +"72405": {"id": "KDCA", "name": "WASHINGTON/NA", "state": "VA US", "lat": 38.850, "lon": -77.030, "elev": 23}, +"72407": {"id": "KACY", "name": "ATLANTIC CITY", "state": "NJ US", "lat": 39.450, "lon": -74.570, "elev": 20}, +"72408": {"id": "KPHL", "name": "PHILADELPHIA", "state": "PA US", "lat": 39.920, "lon": -75.180, "elev": 5}, +"72424": {"id": "KFTK", "name": "FORT KNOX/GODMA", "state": "KY US", "lat": 37.900, "lon": -85.970, "elev": 230}, +"72425": {"id": "KHTS", "name": "HUNTINGTON", "state": "WV US", "lat": 38.370, "lon": -82.550, "elev": 255}, +"72426": {"id": "KILN", "name": "WILMINGTON", "state": "OH US", "lat": 39.420, "lon": -83.820, "elev": 307}, +"72429": {"id": "KDAY", "name": "DAYTON", "state": "OH US", "lat": 39.900, "lon": -84.200, "elev": 306}, +"72433": {"id": "KSLO", "name": "SALEM-LECKRONE", "state": "IL US", "lat": 38.650, "lon": -88.970, "elev": 177}, +"72435": {"id": "KPAH", "name": "PADUCAH", "state": "KY US", "lat": 37.070, "lon": -88.770, "elev": 125}, +"72440": {"id": "KSGF", "name": "SPRINGFIELD", "state": "MO US", "lat": 37.230, "lon": -93.400, "elev": 392}, +"72451": {"id": "KDDC", "name": "DODGE CITY", "state": "KS US", "lat": 37.770, "lon": -99.970, "elev": 790}, +"72455": {"id": "KFRI", "name": "FORT RILEY", "state": "KS US", "lat": 39.050, "lon": -96.770, "elev": 325}, +"72456": {"id": "KTOP", "name": "TOPEKA/BILLARD", "state": "KS US", "lat": 39.070, "lon": -95.620, "elev": 270}, +"72468": {"id": "KFCS", "name": "FORT CARSON", "state": "CO US", "lat": 38.680, "lon": -104.770, "elev": 1790}, +"72469": {"id": "KDNR", "name": "DENVER", "state": "CO US", "lat": 39.750, "lon": -104.870, "elev": 1625}, +"72475": {"id": "KCDC", "name": "CEDAR CITY", "state": "UT US", "lat": 37.700, "lon": -113.100, "elev": 1714}, +"72476": {"id": "KGJT", "name": "GRAND JUNCTION", "state": "CO US", "lat": 39.120, "lon": -108.530, "elev": 1475}, +"72483": {"id": "KSAC", "name": "SACRAMENTO", "state": "CA US", "lat": 38.520, "lon": -121.500, "elev": 8}, +"72486": {"id": "KELY", "name": "ELY/YELLAND FIELD", "state": "NV US", "lat": 39.280, "lon": -114.850, "elev": 1909}, +"72489": {"id": "KREV", "name": "RENO", "state": "NV US", "lat": 39.570, "lon": -119.780, "elev": 1515}, +"72492": {"id": "KSCK", "name": "STOCKTON", "state": "CA US", "lat": 37.900, "lon": -121.250, "elev": 8}, +"72493": {"id": "KOAK", "name": "OAKLAND", "state": "CA US", "lat": 37.730, "lon": -122.220, "elev": 3}, +"72494": {"id": "KSFO", "name": "SAN FRANCISCO", "state": "CA US", "lat": 37.620, "lon": -122.380, "elev": 5}, +"72501": {"id": "KOKX", "name": "BROOKHAVEN", "state": "NY US", "lat": 40.870, "lon": -72.870, "elev": 20}, +"72518": {"id": "KALB", "name": "ALBANY", "state": "NY US", "lat": 42.750, "lon": -73.800, "elev": 89}, +"72520": {"id": "KPIT", "name": "PITTSBURGH", "state": "PA US", "lat": 40.500, "lon": -80.220, "elev": 373}, +"72528": {"id": "KBUF", "name": "BUFFALO", "state": "NY US", "lat": 42.930, "lon": -78.730, "elev": 215}, +"72532": {"id": "KPIA", "name": "PEORIA", "state": "IL US", "lat": 40.670, "lon": -89.680, "elev": 202}, +"72553": {"id": "KOVN", "name": "NORTH OMAHA", "state": "NE US", "lat": 41.370, "lon": -96.020, "elev": 406}, +"72558": {"id": "KOAX", "name": "OMAHA VALLEY", "state": "NE US", "lat": 41.320, "lon": -96.370, "elev": 350}, +"72562": {"id": "KLBF", "name": "NORTH PLATTE", "state": "NE US", "lat": 41.130, "lon": -100.680, "elev": 849}, +"72572": {"id": "KSLC", "name": "SALT LAKE CITY", "state": "UT US", "lat": 40.780, "lon": -111.970, "elev": 1288}, +"72576": {"id": "KLND", "name": "LANDER/HUNT FIELD", "state": "WY US", "lat": 42.820, "lon": -108.730, "elev": 1694}, +"72579": {"id": "KRXE", "name": "IPCO/REXBURG", "state": "ID US", "lat": 43.900, "lon": -111.680, "elev": 1514}, +"72582": {"id": "KEKO", "name": "ELKO", "state": "NV US", "lat": 40.870, "lon": -115.730, "elev": 1367}, +"72583": {"id": "KWMC", "name": "WINNEMUCCA MUNICIPAL", "state": "NV US", "lat": 40.900, "lon": -117.800, "elev": 1322}, +"72591": {"id": "KRBL", "name": "RED BLUFF", "state": "CA US", "lat": 40.150, "lon": -122.250, "elev": 108}, +"72592": {"id": "KRDD", "name": "REDDING", "state": "CA US", "lat": 40.500, "lon": -122.300, "elev": 153}, +"72597": {"id": "KMFR", "name": "MEDFORD", "state": "OR US", "lat": 42.370, "lon": -122.870, "elev": 405}, +"72606": {"id": "KPWM", "name": "PORTLAND INTL", "state": "ME US", "lat": 43.650, "lon": -70.320, "elev": 19}, +"72632": {"id": "KDTX", "name": "DETROIT", "state": "MI US", "lat": 42.680, "lon": -83.470, "elev": 329}, +"72634": {"id": "KAPX", "name": "GAYLORD", "state": "MI US", "lat": 44.920, "lon": -84.720, "elev": 446}, +"72637": {"id": "KFNT", "name": "FLINT/BISHOP", "state": "MI US", "lat": 42.970, "lon": -83.750, "elev": 233}, +"72645": {"id": "KGRB", "name": "GREEN BAY", "state": "WI US", "lat": 44.480, "lon": -88.130, "elev": 214}, +"72649": {"id": "KMPX", "name": "CHANHASSEN", "state": "MN US", "lat": 44.850, "lon": -93.570, "elev": 287}, +"72654": {"id": "KHON", "name": "HURON REGIONAL", "state": "SD US", "lat": 44.380, "lon": -98.220, "elev": 393}, +"72655": {"id": "KSTC", "name": "ST CLOUD", "state": "MN US", "lat": 45.550, "lon": -94.070, "elev": 312}, +"72659": {"id": "KABR", "name": "ABERDEEN", "state": "SD US", "lat": 45.450, "lon": -98.420, "elev": 397}, +"72662": {"id": "KRAP", "name": "RAPID CITY REGIONAL", "state": "SD US", "lat": 44.050, "lon": -103.070, "elev": 966}, +"72672": {"id": "KRIW", "name": "RIVERTON", "state": "WY US", "lat": 43.000, "lon": -108.500, "elev": 1688}, +"72681": {"id": "KBOI", "name": "BOISE MUNICIPAL", "state": "ID US", "lat": 43.570, "lon": -116.220, "elev": 874}, +"72682": {"id": "KGVL", "name": "IPCO/CROUCH", "state": "ID US", "lat": 44.140, "lon": -115.990, "elev": 1088}, +"72694": {"id": "KSLE", "name": "SALEM/McNARY", "state": "OR US", "lat": 44.920, "lon": -123.000, "elev": 61}, +"72698": {"id": "KPDX", "name": "PORTLAND/INT.", "state": "OR US", "lat": 45.580, "lon": -122.600, "elev": 8}, +"72712": {"id": "KCAR", "name": "CARIBOU MUNICIPAL", "state": "ME US", "lat": 46.870, "lon": -68.020, "elev": 190}, +"72734": {"id": "KSSM", "name": "SAULT STE MARIE", "state": "MI US", "lat": 46.470, "lon": -84.370, "elev": 221}, +"72747": {"id": "KINL", "name": "INTERNATIONAL FALLS", "state": "MN US", "lat": 48.570, "lon": -93.380, "elev": 361}, +"72764": {"id": "KBIS", "name": "BISMARCK", "state": "ND US", "lat": 46.770, "lon": -100.750, "elev": 506}, +"72768": {"id": "KGGW", "name": "GLASGOW", "state": "MT US", "lat": 48.220, "lon": -106.620, "elev": 700}, +"72775": {"id": "KGTF", "name": "GREAT FALLS", "state": "MT US", "lat": 47.480, "lon": -111.370, "elev": 1115}, +"72776": {"id": "KTFX", "name": "GREAT FALLS", "state": "MT US", "lat": 47.450, "lon": -111.380, "elev": 1130}, +"72785": {"id": "KGEG", "name": "SPOKANE INTL", "state": "WA US", "lat": 47.630, "lon": -117.530, "elev": 721}, +"72786": {"id": "KGEG", "name": "SPOKANE INTL", "state": "WA US", "lat": 47.680, "lon": -117.630, "elev": 728}, +"72793": {"id": "KSEA", "name": "SEATTLE", "state": "WA US", "lat": 47.450, "lon": -122.300, "elev": 130}, +"72797": {"id": "KUIL", "name": "QUILLAYUTE", "state": "WA US", "lat": 47.950, "lon": -124.550, "elev": 62}, +"72816": {"id": "CYYR", "name": "GOOSE BAY", "state": "NF CN", "lat": 53.300, "lon": -60.370, "elev": 44}, +"72924": {"id": "CYRB", "name": "RESOLUTE", "state": "NT CN", "lat": 74.720, "lon": -94.980, "elev": 67}, +"72938": {"id": "", "name": "COPPERMINE", "state": "NT CN", "lat": 67.820, "lon": -115.080, "elev": 9}, +"73033": {"id": "CWVK", "name": "VERNON BRIDGE", "state": "BC CN", "lat": 50.220, "lon": -119.270, "elev": 555}, +"74001": {"id": "", "name": "REDSTONE ARSENAL", "state": "AL US", "lat": 34.680, "lon": -86.680, "elev": 209}, +"74002": {"id": "KAPG", "name": "ABERDEEN PROVING", "state": "MD US", "lat": 39.470, "lon": -76.170, "elev": 18}, +"74003": {"id": "KDPG", "name": "DUGWAY PROVING GROUND", "state": "UT US", "lat": 40.180, "lon": -112.920, "elev": 1325}, +"74004": {"id": "K1YZ", "name": "YUMA", "state": "AZ US", "lat": 32.850, "lon": -114.400, "elev": 98}, +"74005": {"id": "K1Y8", "name": "YUMA PG SITE 2", "state": "AZ US", "lat": 32.860, "lon": -114.030, "elev": 231}, +"74006": {"id": "K1Y9", "name": "YUMA PG SITE 3", "state": "AZ US", "lat": 32.920, "lon": -113.800, "elev": 145}, +"74051": {"id": "", "name": "SACHS HARBOUR", "state": "BC CN", "lat": 71.980, "lon": -125.280, "elev": 83}, +"74074": {"id": "", "name": "ISACHSEN", "state": "BC CN", "lat": 78.780, "lon": -103.530, "elev": 31}, +"74090": {"id": "", "name": "CLYLDE", "state": "BC CN", "lat": 70.450, "lon": -68.550, "elev": 16}, +"74109": {"id": "CYZT", "name": "PORT HARDY", "state": "BC CN", "lat": 50.680, "lon": -127.370, "elev": 22}, +"74207": {"id": "KGRF", "name": "FT LEWIS", "state": "WA US", "lat": 47.080, "lon": -122.570, "elev": 92}, +"74370": {"id": "KGTB", "name": "FT DRUM", "state": "NY US", "lat": 44.050, "lon": -75.720, "elev": 210}, +"74389": {"id": "KGYX", "name": "GRAY", "state": "ME US", "lat": 43.880, "lon": -70.250, "elev": 125}, +"74455": {"id": "KDVN", "name": "DAVENPORT/QUAD CITIES", "state": "IA US", "lat": 41.620, "lon": -90.570, "elev": 229}, +"74486": {"id": "KJFK", "name": "NEW YORK/KENNEDY", "state": "NY US", "lat": 40.780, "lon": -73.770, "elev": 8}, +"74494": {"id": "KCHH", "name": "CHATHAM", "state": "MA US", "lat": 41.670, "lon": -69.970, "elev": 16}, +"74504": {"id": "", "name": "PILAR POINT AFS", "state": "CA US", "lat": 37.500, "lon": -122.500, "elev": 49}, +"74505": {"id": "KSJC", "name": "SAN JOSE", "state": "CA US", "lat": 37.370, "lon": -121.930, "elev": 17}, +"74516": {"id": "KSUU", "name": "FAIRFIELD", "state": "CA US", "lat": 38.270, "lon": -121.930, "elev": 19}, +"74530": {"id": "", "name": "GRANADA", "state": "CO US", "lat": 39.780, "lon": -104.980, "elev": 1575}, +"74531": {"id": "KAAF", "name": "U.S. AIR FORCE ACADEMY", "state": "CO US", "lat": 38.970, "lon": -104.820, "elev": 1999}, +"74547": {"id": "KHBK", "name": "HILLSBORO (ARM)", "state": "KS US", "lat": 38.300, "lon": -97.300, "elev": 446}, +"74560": {"id": "KILX", "name": "LINCOLN", "state": "IL US", "lat": 40.150, "lon": -89.330, "elev": 178}, +"74606": {"id": "KVBG", "name": "VANDENBERG AFB", "state": "CA US", "lat": 34.650, "lon": -120.570, "elev": 112}, +"74611": {"id": "KBYS", "name": "BARSTOW", "state": "CA US", "lat": 35.280, "lon": -116.620, "elev": 771}, +"74612": {"id": "KNID", "name": "CHINA LAKE NAF", "state": "CA US", "lat": 35.680, "lon": -117.680, "elev": 696}, +"74619": {"id": "K4SU", "name": "SUPERIOR VALLEY", "state": "CA US", "lat": 36.330, "lon": -117.100, "elev": 1035}, +"74626": {"id": "", "name": "WFO PHOENIX", "state": "AZ US", "lat": 33.450, "lon": -111.950, "elev": 384}, +"74641": {"id": "KVCI", "name": "VICI (ARM)", "state": "OK US", "lat": 36.070, "lon": -99.200, "elev": 621}, +"74646": {"id": "KLMN", "name": "LAMONT (ARM-CART)", "state": "OK US", "lat": 36.620, "lon": -97.480, "elev": 317}, +"74650": {"id": "KMRS", "name": "MORRIS (ARM)", "state": "OK US", "lat": 35.670, "lon": -95.850, "elev": 216}, +"74651": {"id": "KPRC", "name": "PURCELL (ARM)", "state": "OK US", "lat": 34.970, "lon": -97.420, "elev": 343}, +"74671": {"id": "KHOP", "name": "FORT CAMPBELL", "state": "KY US", "lat": 36.670, "lon": -87.500, "elev": 174}, +"74693": {"id": "KFBG", "name": "FORT BRAGG", "state": "NC US", "lat": 35.130, "lon": -78.930, "elev": 74}, +"74702": {"id": "KNLC", "name": "LEMOORE NAS", "state": "CA US", "lat": 36.330, "lon": -119.950, "elev": 71}, +"74724": {"id": "KGBN", "name": "GILA BEND AFB", "state": "AZ US", "lat": 32.430, "lon": -112.680, "elev": 281}, +"74732": {"id": "KHMN", "name": "ALAMOGORDO/HOLLMAN AFB", "state": "NM US", "lat": 32.850, "lon": -106.100, "elev": 1258}, +"74777": {"id": "KHRT", "name": "HURLBURT FIELD", "state": "FL US", "lat": 30.430, "lon": -86.680, "elev": 12}, +"74780": {"id": "", "name": "FORT STEWART RE", "state": "GA US", "lat": 31.900, "lon": -81.630, "elev": 27}, +"74794": {"id": "KXMR", "name": "CAPE CANAVERAL", "state": "FL US", "lat": 28.470, "lon": -80.550, "elev": 3}, +"76151": {"id": "KGUD", "name": "ISLA GUADALUPE", "state": "MX", "lat": 29.170, "lon": -118.320, "elev": 25}, +"76225": {"id": "MMCU", "name": "CHIHUAHUA", "state": "MX", "lat": 28.630, "lon": -106.080, "elev": 1435}, +"76256": {"id": "MGYM", "name": "EMPALME", "state": "MX", "lat": 27.920, "lon": -110.900, "elev": 11}, +"76382": {"id": "MMTC", "name": "TORREJON", "state": "MX", "lat": 25.530, "lon": -103.450, "elev": 1124}, +"76390": {"id": "MMIO", "name": "SALTILLO", "state": "MX", "lat": 25.450, "lon": -100.980, "elev": 1790}, +"76393": {"id": "", "name": "MONTERREY CITY", "state": "MX", "lat": 25.870, "lon": -100.200, "elev": 512}, +"76394": {"id": "MMAN", "name": "MONTERREY INTL AIRPORT", "state": "MX", "lat": 25.870, "lon": -100.230, "elev": 448}, +"76405": {"id": "MMLP", "name": "LA PAZ", "state": "MX", "lat": 24.130, "lon": -110.420, "elev": 27}, +"76458": {"id": "MMMZ", "name": "MAZATLAN", "state": "MX", "lat": 23.200, "lon": -106.420, "elev": 4}, +"76526": {"id": "MZAC", "name": "ZACATECAS", "state": "MX", "lat": 22.750, "lon": -102.500, "elev": 2265}, +"76595": {"id": "MCUN", "name": "CANCUN", "state": "MX", "lat": 21.030, "lon": -87.020, "elev": 10}, +"76612": {"id": "MMGL", "name": "GUADALAJARA", "state": "MX", "lat": 20.670, "lon": -103.380, "elev": 1551}, +"76642": {"id": "", "name": "TRIANGULOS", "state": "MX", "lat": 20.970, "lon": -92.320, "elev": 2}, +"76644": {"id": "MMMD", "name": "MERIDA", "state": "MX", "lat": 20.980, "lon": -89.650, "elev": 9}, +"76654": {"id": "MMZO", "name": "MANZANILLO", "state": "MX", "lat": 19.050, "lon": -104.330, "elev": 3}, +"76679": {"id": "MMMX", "name": "MEXICO CITY", "state": "MX", "lat": 19.430, "lon": -99.080, "elev": 2234}, +"76680": {"id": "", "name": "MEXICO CITY", "state": "MX", "lat": 19.430, "lon": -99.130, "elev": 2237}, +"76692": {"id": "MMVR", "name": "VERACRUZ", "state": "MX", "lat": 19.150, "lon": -96.120, "elev": 35}, +"76723": {"id": "KSOC", "name": "ISLA SOCORRO", "state": "MX", "lat": 18.720, "lon": -110.950, "elev": 35}, +"76726": {"id": "MMCB", "name": "CUERNAVACA", "state": "MX", "lat": 18.920, "lon": -99.250, "elev": 1560}, +"76743": {"id": "", "name": "VILLAHERMOSA", "state": "MX", "lat": 17.980, "lon": -92.920, "elev": 6}, +"76805": {"id": "MMAA", "name": "ACAPULCO", "state": "MX", "lat": 16.830, "lon": -99.930, "elev": 28}, +"76904": {"id": "", "name": "PUERTO MADERO TAPACHULA", "state": "MX", "lat": 14.700, "lon": -92.400, "elev": 3}, +"78016": {"id": "TXKF", "name": "BERMUDA NAS", "state": "BE", "lat": 32.270, "lon": -64.850, "elev": 13}, +"78062": {"id": "MYGF", "name": "FREEPORT INTL AIRPORT", "state": "BA", "lat": 26.550, "lon": -78.700, "elev": 11}, +"78073": {"id": "MYNN", "name": "NASSAU", "state": "BH", "lat": 25.050, "lon": -77.470, "elev": 7}, +"78118": {"id": "", "name": "TURKS ISLAND", "state": "TU", "lat": 21.450, "lon": -71.150, "elev": 9}, +"78325": {"id": "", "name": "HAVANA", "state": "CU", "lat": 23.170, "lon": -82.350, "elev": 50}, +"78355": {"id": "MCAM", "name": "CAMAGUEY", "state": "CU", "lat": 21.400, "lon": -77.920, "elev": 122}, +"78359": {"id": "", "name": "MANSANILLO", "state": "CU", "lat": 20.330, "lon": -77.130, "elev": 10}, +"78367": {"id": "MUGT", "name": "GUANTANAMO BAY", "state": "CU", "lat": 19.900, "lon": -75.150, "elev": 17}, +"78384": {"id": "MWCR", "name": "OWEN ROBERTS", "state": "CI", "lat": 19.280, "lon": -81.350, "elev": 3}, +"78397": {"id": "MKJP", "name": "KINGSTON", "state": "JM", "lat": 17.930, "lon": -76.780, "elev": 14}, +"78486": {"id": "MDSD", "name": "SANTO DOMINGO", "state": "DR", "lat": 18.430, "lon": -69.880, "elev": 14}, +"78501": {"id": "", "name": "SWAN ISLAND", "state": "HO", "lat": 17.400, "lon": -83.930, "elev": 10}, +"78526": {"id": "TJSJ", "name": "SAN JUAN INTL", "state": "PR", "lat": 18.430, "lon": -66.000, "elev": 3}, +"78583": {"id": "MZBZ", "name": "BELIZE", "state": "BE", "lat": 17.530, "lon": -88.300, "elev": 5}, +"78641": {"id": "MGGT", "name": "GUATEMALA", "state": "GU", "lat": 14.580, "lon": -90.520, "elev": 1489}, +"78701": {"id": "MHNJ", "name": "GUANAJA", "state": "HO", "lat": 16.470, "lon": -85.920, "elev": 2}, +"78718": {"id": "MHSC", "name": "SOTO CANO AB", "state": "HO", "lat": 14.380, "lon": -87.620, "elev": 626}, +"78720": {"id": "MHTG", "name": "TEGUCIGALPA", "state": "HO", "lat": 14.050, "lon": -87.220, "elev": 1007}, +"78730": {"id": "", "name": "PUERTO CABEZAS", "state": "NI", "lat": 14.030, "lon": -83.400, "elev": 15}, +"78762": {"id": "MROC", "name": "JUAN SANTAMARIA", "state": "CR", "lat": 10.000, "lon": -84.220, "elev": 939}, +"78806": {"id": "MPHO", "name": "HOWARD AFB", "state": "PN", "lat": 8.920, "lon": -79.600, "elev": 16}, +"78807": {"id": "MPCZ", "name": "HOWARD AFB", "state": "PN", "lat": 8.980, "lon": -79.580, "elev": 19}, +"78808": {"id": "", "name": "COROZL (ALBROOK)", "state": "PN", "lat": 8.980, "lon": -79.580, "elev": 8}, +"78861": {"id": "", "name": "COOLIDGE FIELD", "state": "AN", "lat": 17.120, "lon": -61.780, "elev": 22}, +"78866": {"id": "TNCM", "name": "PHILIPSBURG", "state": "CA", "lat": 18.050, "lon": -63.120, "elev": 9}, +"78894": {"id": "TFFJ", "name": "SAINT BARTHELEMY", "state": "CA", "lat": 17.900, "lon": -62.850, "elev": 15}, +"78897": {"id": "TFFR", "name": "LE RAIZET", "state": "CA", "lat": 16.270, "lon": -61.520, "elev": 8}, +"78925": {"id": "TFFF", "name": "LE LAMENTIN", "state": "MA", "lat": 14.580, "lon": -61.000, "elev": 5}, +"78954": {"id": "TBPB", "name": "GRANTLY ADAMS", "state": "BA", "lat": 13.070, "lon": -59.480, "elev": 56}, +"78955": {"id": "", "name": "CARRIBEAN METEORO INSTITUTE,BA", "state": "113", "lat": 0.000, "lon": 0.000, "elev": 13}, +"78956": {"id": "MKPE", "name": "PEARLS AIRPORT", "state": "GD", "lat": 12.150, "lon": -61.620, "elev": 6}, +"78970": {"id": "TTPP", "name": "PIARCO", "state": "TR", "lat": 10.620, "lon": -61.350, "elev": 15}, +"78988": {"id": "TNCC", "name": "CURACAO HATO", "state": "AC", "lat": 12.200, "lon": -68.970, "elev": 67}, +"80001": {"id": "SKSP", "name": "SAN ANDRES ISLAND", "state": "CO", "lat": 12.580, "lon": -81.720, "elev": 1}, +"80002": {"id": "", "name": "PROVIDENCIA ISLAND", "state": "CO", "lat": 13.370, "lon": -81.350, "elev": 6}, +"80028": {"id": "SKBQ", "name": "BARRANQUILLA/ERNESTO C.", "state": "CO", "lat": 10.880, "lon": -74.780, "elev": 30}, +"80035": {"id": "SKRH", "name": "RIOHACHA/ALMIRANTE", "state": "CO", "lat": 11.530, "lon": -72.930, "elev": 4}, +"80094": {"id": "SKBG", "name": "BUCARAMANGA/PALONEGRO", "state": "CO", "lat": 7.100, "lon": -73.200, "elev": 1189}, +"80222": {"id": "SKBO", "name": "BOGOTA/ELDORADO", "state": "CO", "lat": 4.700, "lon": -74.130, "elev": 2548}, +"80241": {"id": "", "name": "GAVIOTAS", "state": "CO", "lat": 4.550, "lon": -70.920, "elev": 167}, +"80259": {"id": "SKCL", "name": "CALI/ALFONSO BONILLA ARAGON,CO", "state": "969", "lat": 0.000, "lon": 0.000, "elev": 3}, +"80300": {"id": "", "name": "GUAPI", "state": "CO", "lat": 2.580, "lon": -77.900, "elev": 54}, +"80398": {"id": "SKLT", "name": "LETICIA/VASQUEZ COBO", "state": "CO", "lat": -4.170, "lon": -69.950, "elev": 84}, +"80407": {"id": "", "name": "MARACAIBO", "state": "VN", "lat": 10.570, "lon": -71.730, "elev": 66}, +"80410": {"id": "SVBM", "name": "BARQUISIMETO", "state": "VN", "lat": 10.070, "lon": -69.320, "elev": 614}, +"80413": {"id": "SVBS", "name": "MARISCAL SUCRE", "state": "VN", "lat": 10.250, "lon": -67.650, "elev": 437}, +"80415": {"id": "", "name": "MARIPASOULA", "state": "VN", "lat": 3.630, "lon": -54.030, "elev": 106}, +"80444": {"id": "SVCB", "name": "CIUDAD BOLIVAR", "state": "VN", "lat": 8.150, "lon": -63.550, "elev": 48}, +"80447": {"id": "", "name": "SAN ANTONIO DE TACHIRA", "state": "VN", "lat": 7.850, "lon": -72.450, "elev": 405}, +"80450": {"id": "SVSR", "name": "SAN FERNANDO APURE", "state": "VN", "lat": 7.900, "lon": -67.420, "elev": 48}, +"80462": {"id": "", "name": "SANTA ELENA", "state": "VN", "lat": 4.680, "lon": -61.180, "elev": 896}, +"81001": {"id": "SYGT", "name": "GEORGETOWN", "state": "GY", "lat": 6.800, "lon": -58.150, "elev": 2}, +"81401": {"id": "", "name": "ST. LAURENT DU MARONI", "state": "FG", "lat": 5.500, "lon": -54.030, "elev": 4}, +"81405": {"id": "SOCA", "name": "CAYENNE/ROCHAMBEAU", "state": "FG", "lat": 4.830, "lon": -52.370, "elev": 9}, +"81408": {"id": "", "name": "ST. GEORGES", "state": "FG", "lat": 3.880, "lon": -51.800, "elev": 7}, +"81415": {"id": "", "name": "MARIPASSOULA", "state": "FG", "lat": 3.630, "lon": -54.030, "elev": 106}, +"82022": {"id": "SBBV", "name": "BOA VISTA/AIRPORT", "state": "BZ", "lat": 2.830, "lon": -60.700, "elev": 140}, +"82026": {"id": "", "name": "TIRIOS", "state": "BZ", "lat": 2.480, "lon": -55.980, "elev": 326}, +"82099": {"id": "", "name": "MACAPA AIRPORT", "state": "BZ", "lat": 3.050, "lon": -51.070, "elev": 17}, +"82107": {"id": "", "name": "SAO GABRIEL DA CACHOEIRA", "state": "BZ", "lat": -0.120, "lon": -66.970, "elev": 79}, +"82193": {"id": "SBBE", "name": "BELEM/VAL DE CAES", "state": "BZ", "lat": -1.380, "lon": -48.480, "elev": 16}, +"82244": {"id": "SBSN", "name": "SANTAREM/INTL", "state": "BZ", "lat": -2.430, "lon": -54.720, "elev": 72}, +"82280": {"id": "", "name": "SAO LUIZ", "state": "BZ", "lat": -2.530, "lon": -44.280, "elev": 51}, +"82281": {"id": "SBSL", "name": "SAO LUIS/MARECHAL", "state": "BZ", "lat": -2.600, "lon": -44.230, "elev": 53}, +"82332": {"id": "SBMN", "name": "MANAUS/PONTA PELADA", "state": "BZ", "lat": -3.150, "lon": -59.980, "elev": 84}, +"82397": {"id": "", "name": "FORTALEZA", "state": "BZ", "lat": -3.730, "lon": -38.550, "elev": 19}, +"82400": {"id": "SBFN", "name": "FERNANDO DE NORONHA", "state": "BZ", "lat": -3.850, "lon": -32.420, "elev": 56}, +"82411": {"id": "SBTT", "name": "TABATINGA/INTL", "state": "BZ", "lat": -3.670, "lon": -69.670, "elev": 85}, +"82532": {"id": "", "name": "MANICORE (AERO)", "state": "BZ", "lat": -5.820, "lon": -61.280, "elev": 53}, +"82599": {"id": "SBNT", "name": "NATAL/AUGUSTO SEVER", "state": "BZ", "lat": -5.920, "lon": -35.250, "elev": 52}, +"82640": {"id": "", "name": "JACAREACANGA", "state": "BZ", "lat": -6.270, "lon": -57.730, "elev": 98}, +"82678": {"id": "", "name": "FLORIANO", "state": "BZ", "lat": -6.770, "lon": -43.020, "elev": 138}, +"82705": {"id": "", "name": "CRUZEIRO DO SUL (AERO)", "state": "BZ", "lat": -7.580, "lon": -72.760, "elev": 199}, +"82765": {"id": "", "name": "CAROLINA", "state": "BZ", "lat": -7.330, "lon": -47.470, "elev": 212}, +"82824": {"id": "SBPV", "name": "PORTO VELHO (CV/MIL)", "state": "BZ", "lat": -8.770, "lon": -63.920, "elev": 88}, +"82900": {"id": "", "name": "RECIFE/CURADO", "state": "BZ", "lat": -8.050, "lon": -34.920, "elev": 7}, +"82917": {"id": "SBRB", "name": "RIO BRANCO/MEDICI", "state": "BZ", "lat": -10.000, "lon": -67.800, "elev": 143}, +"82965": {"id": "SBAT", "name": "ALTA FLORESTA", "state": "BZ", "lat": -9.870, "lon": -56.100, "elev": 288}, +"82983": {"id": "", "name": "PETROLINA", "state": "BZ", "lat": -9.380, "lon": -40.480, "elev": 370}, +"83063": {"id": "", "name": "PORTO NACIONAL", "state": "BZ", "lat": -10.700, "lon": -48.420, "elev": 290}, +"83096": {"id": "", "name": "ARACAJU", "state": "BZ", "lat": -10.920, "lon": -37.050, "elev": 6}, +"83208": {"id": "SBVH", "name": "VILHENA", "state": "BZ", "lat": -12.730, "lon": -60.130, "elev": 652}, +"83229": {"id": "", "name": "SALVADOR", "state": "BZ", "lat": -13.020, "lon": -38.520, "elev": 51}, +"83288": {"id": "SBLP", "name": "BOM JESUS DA LAPA", "state": "BZ", "lat": -13.270, "lon": -43.420, "elev": 458}, +"83361": {"id": "", "name": "CUIABA", "state": "BZ", "lat": -15.550, "lon": -56.120, "elev": 179}, +"83362": {"id": "SBCY", "name": "CUIABA/MARECHAL", "state": "BZ", "lat": -15.650, "lon": -56.100, "elev": 182}, +"83378": {"id": "SBBR", "name": "BRASILIA (CIV/MIL)", "state": "BZ", "lat": -15.870, "lon": -47.930, "elev": 1061}, +"83424": {"id": "SBGO", "name": "GOIANIA", "state": "BZ", "lat": -16.630, "lon": -49.220, "elev": 747}, +"83498": {"id": "", "name": "CARAVELAS", "state": "BZ", "lat": -17.730, "lon": -39.250, "elev": 3}, +"83525": {"id": "SBUL", "name": "UBERLANDIA", "state": "BZ", "lat": -18.880, "lon": -48.230, "elev": 943}, +"83554": {"id": "SBCR", "name": "CORUMBA", "state": "BZ", "lat": -19.000, "lon": -57.670, "elev": 142}, +"83566": {"id": "", "name": "CONFIS INTNL ARPT", "state": "BZ", "lat": -19.620, "lon": -43.570, "elev": 827}, +"83576": {"id": "", "name": "UBERABA", "state": "BZ", "lat": -19.770, "lon": -47.950, "elev": 809}, +"83612": {"id": "SBCG", "name": "CAMPO GRANDE INTL", "state": "BZ", "lat": -20.470, "lon": -54.670, "elev": 556}, +"83649": {"id": "SBVT", "name": "VITORIA/GOIABEIRAS", "state": "BZ", "lat": -20.270, "lon": -40.280, "elev": 4}, +"83650": {"id": "", "name": "TRINDADE ISLAND", "state": "BZ", "lat": -20.500, "lon": -29.320, "elev": 5}, +"83746": {"id": "SBGL", "name": "GALEAO/RIO(CIV/MIL)", "state": "BZ", "lat": -22.820, "lon": -43.250, "elev": 6}, +"83768": {"id": "SBLO", "name": "LONDRINA AIRPORT", "state": "BZ", "lat": -23.330, "lon": -51.130, "elev": 570}, +"83779": {"id": "", "name": "MARTE APRT", "state": "BZ", "lat": -23.500, "lon": -46.600, "elev": 722}, +"83780": {"id": "SBSP", "name": "SAO PAULO/CONGONHAS", "state": "BZ", "lat": -23.620, "lon": -46.650, "elev": 803}, +"83827": {"id": "SBFI", "name": "FOZ DO IGUACU ARPT", "state": "BZ", "lat": -25.520, "lon": -54.580, "elev": 243}, +"83840": {"id": "SBCT", "name": "CURITIBA/AFONSO PEN", "state": "BZ", "lat": -25.520, "lon": -49.170, "elev": 908}, +"83899": {"id": "SBFL", "name": "FLORIANOPOLIS ARPT", "state": "BZ", "lat": -27.670, "lon": -48.550, "elev": 5}, +"83928": {"id": "SBUG", "name": "URUGUAIANA/RUBEM", "state": "BZ", "lat": -29.780, "lon": -57.030, "elev": 74}, +"83937": {"id": "SBSM", "name": "SANTA MARIA/AIRPORT", "state": "BZ", "lat": -29.720, "lon": -53.700, "elev": 85}, +"83971": {"id": "SBPA", "name": "PORTO ALEGRE/SALGAD", "state": "BZ", "lat": -30.000, "lon": -51.180, "elev": 3}, +"83981": {"id": "", "name": "BAGE", "state": "BZ", "lat": -31.350, "lon": -54.120, "elev": 180}, +"84008": {"id": "SEST", "name": "SAN CRISTOBAL ISL", "state": "EQ", "lat": -0.900, "lon": -89.600, "elev": 6}, +"84135": {"id": "", "name": "PORTO VIEJ", "state": "EQ", "lat": -1.030, "lon": -80.430, "elev": 26}, +"84202": {"id": "", "name": "MILAGRO", "state": "EQ", "lat": -2.120, "lon": -79.600, "elev": 36}, +"84203": {"id": "", "name": "GUAYAQUIL", "state": "EQ", "lat": -2.150, "lon": -79.880, "elev": 100}, +"84378": {"id": "", "name": "MORONA", "state": "PR", "lat": -3.730, "lon": -73.250, "elev": 117}, +"84452": {"id": "", "name": "CHICLAYO", "state": "PR", "lat": -6.780, "lon": -79.830, "elev": 31}, +"84628": {"id": "SPIM", "name": "LIMA/JORGE CHAVEZ", "state": "PR", "lat": -12.000, "lon": -77.120, "elev": 13}, +"84629": {"id": "", "name": "LAS PALMAS", "state": "PR", "lat": -12.150, "lon": -77.000, "elev": 80}, +"84752": {"id": "", "name": "AREQUIPA", "state": "PR", "lat": -16.320, "lon": -71.550, "elev": 2525}, +"85201": {"id": "SLLP", "name": "LA PAZ", "state": "BO", "lat": -16.520, "lon": -68.180, "elev": 4014}, +"85223": {"id": "SLCB", "name": "COCHABAMAB", "state": "BO", "lat": -17.450, "lon": -66.100, "elev": 2531}, +"85364": {"id": "SLTJ", "name": "TARIJA", "state": "BO", "lat": -21.530, "lon": -64.720, "elev": 1858}, +"85394": {"id": "", "name": "BERMEJO", "state": "BO", "lat": -22.770, "lon": -64.320, "elev": 381}, +"85418": {"id": "SCDA", "name": "IQUIQUE", "state": "CH", "lat": -20.530, "lon": -70.180, "elev": 52}, +"85442": {"id": "SCFA", "name": "ANTOFAGASTA/CERRO", "state": "CH", "lat": -23.430, "lon": -70.430, "elev": 120}, +"85460": {"id": "SCRA", "name": "CHANARAL", "state": "CH", "lat": -26.320, "lon": -70.620, "elev": 30}, +"85469": {"id": "SCIP", "name": "EASTER ISLAND", "state": "CH", "lat": -27.150, "lon": -109.420, "elev": 47}, +"85470": {"id": "SCHA", "name": "COPIAPO", "state": "CH", "lat": -27.300, "lon": -70.420, "elev": 291}, +"85543": {"id": "SCER", "name": "QUINTERO (MIL)", "state": "CH", "lat": -32.780, "lon": -71.520, "elev": 8}, +"85574": {"id": "SCEL", "name": "PUDAHUEL/ARTURO", "state": "CH", "lat": -33.380, "lon": -70.780, "elev": 476}, +"85577": {"id": "", "name": "SANTIAGO Q. NORMAL", "state": "CH", "lat": -33.430, "lon": -70.680, "elev": 520}, +"85586": {"id": "SCSN", "name": "SANTO DOMINGO", "state": "CH", "lat": -33.650, "lon": -71.620, "elev": 75}, +"85799": {"id": "SCTE", "name": "PUERTO MONTT/TEPUAL", "state": "CH", "lat": -41.420, "lon": -73.080, "elev": 86}, +"85834": {"id": "", "name": "ISLA HUAFO", "state": "CH", "lat": -43.570, "lon": -74.830, "elev": 140}, +"85934": {"id": "SCCI", "name": "PUNTA ARENAS", "state": "CH", "lat": -53.000, "lon": -70.850, "elev": 37}, +"86218": {"id": "SGAS", "name": "ASUNCION", "state": "PY", "lat": -25.270, "lon": -57.630, "elev": 101}, +"86294": {"id": "", "name": "CAP.MIRANDA", "state": "PY", "lat": -27.280, "lon": -58.270, "elev": 16}, +"86560": {"id": "SUCA", "name": "COLONIA", "state": "UY", "lat": -34.450, "lon": -57.830, "elev": 23}, +"87047": {"id": "SASA", "name": "SALTA AIRPORT", "state": "AG", "lat": -24.850, "lon": -65.480, "elev": 1216}, +"87155": {"id": "SARE", "name": "RESISTENCIA AIRPORT", "state": "AG", "lat": -27.450, "lon": -59.050, "elev": 52}, +"87178": {"id": "SARP", "name": "POSADAS AIRPORT", "state": "AG", "lat": -27.370, "lon": -55.970, "elev": 125}, +"87344": {"id": "SACO", "name": "CORDOBA AIRPORT", "state": "AG", "lat": -31.320, "lon": -64.220, "elev": 474}, +"87418": {"id": "SAME", "name": "MENDOZA/EL PLUMERIL", "state": "AG", "lat": -32.830, "lon": -68.780, "elev": 704}, +"87576": {"id": "SAEZ", "name": "BUENOS AIRES/EZEIZA", "state": "AG", "lat": -34.820, "lon": -58.530, "elev": 20}, +"87623": {"id": "SAZR", "name": "SANTA ROSA AIRPORT", "state": "AG", "lat": -36.570, "lon": -64.270, "elev": 191}, +"87715": {"id": "SAZN", "name": "NEUQUEN AIRPORT", "state": "AG", "lat": -38.950, "lon": -68.130, "elev": 271}, +"87748": {"id": "", "name": "B.A. COMANDANTE ESPORA", "state": "AG", "lat": -38.730, "lon": -62.170, "elev": 75}, +"87860": {"id": "SAVC", "name": "COMODORO RIVADAVIA", "state": "AG", "lat": -45.780, "lon": -67.500, "elev": 46}, +"88889": {"id": "EGYP", "name": "MOUNT PLEASANT ARPT", "state": "FK", "lat": -51.820, "lon": -58.450, "elev": 73}, +"88903": {"id": "", "name": "GRYTVIKEN", "state": "FK", "lat": -54.270, "lon": -36.500, "elev": 3}, +"88968": {"id": "", "name": "ISLAS ORCADAS", "state": "AG", "lat": -60.750, "lon": -44.720, "elev": 6}, +"89001": {"id": "", "name": "SANAE SAF-BASE", "state": "ZA", "lat": -70.300, "lon": -2.350, "elev": 62}, +"89002": {"id": "", "name": "VON-NEUMAYER G-BASE", "state": "DL", "lat": -70.670, "lon": -8.250, "elev": 40}, +"89009": {"id": "NZSP", "name": "AMUNDSEN-SCOTT", "state": "US", "lat": -90.000, "lon": 0.000, "elev": 2830}, +"89022": {"id": "", "name": "HALLEY BRI-BASE", "state": "UK", "lat": -75.500, "lon": -26.650, "elev": 30}, +"89050": {"id": "", "name": "BELLINGSHAUSEN", "state": "RA", "lat": -62.200, "lon": -58.930, "elev": 16}, +"89055": {"id": "", "name": "VICECOMODORO MARAM", "state": "AG", "lat": -64.230, "lon": -56.720, "elev": 198}, +"89062": {"id": "", "name": "ROTHERA PT BRI-BASE", "state": "UK", "lat": -67.570, "lon": -68.130, "elev": 16}, +"89512": {"id": "", "name": "NOVOLAZAREVSKAJA", "state": "RA", "lat": -70.770, "lon": 11.830, "elev": 102}, +"89532": {"id": "", "name": "SYOWA JAPAN-BASE", "state": "JP", "lat": -69.000, "lon": 39.580, "elev": 21}, +"89542": {"id": "", "name": "MOLODEZNAJA", "state": "RA", "lat": -67.670, "lon": 45.850, "elev": 40}, +"89564": {"id": "", "name": "MAWSON AUS-BASE", "state": "AU", "lat": -68.600, "lon": 62.870, "elev": 16}, +"89571": {"id": "", "name": "DAVIS AUS-BASE", "state": "AU", "lat": -68.570, "lon": 77.950, "elev": 13}, +"89577": {"id": "", "name": "DAVIS", "state": "AU", "lat": -75.850, "lon": 71.480, "elev": 2356}, +"89592": {"id": "", "name": "MIRNYJ SOVIET-BASE", "state": "RA", "lat": -66.550, "lon": 93.020, "elev": 30}, +"89606": {"id": "", "name": "VOSTOK SOVIET-BASE", "state": "RA", "lat": -78.450, "lon": 106.870, "elev": 3420}, +"89611": {"id": "", "name": "CASEY AUS-BASE", "state": "AU", "lat": -66.280, "lon": 110.520, "elev": 41}, +"89625": {"id": "", "name": "CONCORDIA/ANTARTIKA", "state": "IY", "lat": -75.170, "lon": 123.690, "elev": 3234}, +"89642": {"id": "", "name": "DUMONT D'URVILLE", "state": "FR", "lat": -66.670, "lon": 140.020, "elev": 43}, +"89662": {"id": "", "name": "BAIA TERRA NOVA", "state": "IY", "lat": -74.700, "lon": 164.100, "elev": 80}, +"89663": {"id": "", "name": "LAKE VANDA", "state": "NZ", "lat": -77.530, "lon": 161.670, "elev": 15}, +"89664": {"id": "", "name": "MCMURDO USA-BASE", "state": "US", "lat": -77.850, "lon": 166.670, "elev": 34}, +"89671": {"id": "", "name": "HALLETT", "state": "US/NZ", "lat": -72.300, "lon": 170.300, "elev": 5}, +"91066": {"id": "PMDY", "name": "MIDWAY ISLAND", "state": "HI", "lat": 28.220, "lon": -177.350, "elev": 9}, +"91115": {"id": "KFFS", "name": "FRENCH FRIGATE SHOAL", "state": "US", "lat": 23.870, "lon": -166.280, "elev": 6}, +"91162": {"id": "KBKH", "name": "BARKING SANDS", "state": "US", "lat": 22.030, "lon": -159.780, "elev": 5}, +"91163": {"id": "", "name": "PORT ALLEN AIRPORT", "state": "HI US", "lat": 21.900, "lon": -159.600, "elev": 7}, +"91165": {"id": "PLIH", "name": "LIHUE-ON-KAUAI", "state": "HI US", "lat": 21.980, "lon": -159.350, "elev": 45}, +"91168": {"id": "", "name": "KAPAA/KAUIAI", "state": "HI US", "lat": 22.070, "lon": -159.400, "elev": 162}, +"91169": {"id": "", "name": "WAIALEE", "state": "HI US", "lat": 21.680, "lon": -158.030, "elev": 10}, +"91170": {"id": "PHHI", "name": "WHEELER AFB", "state": "HI US", "lat": 21.480, "lon": -158.030, "elev": 255}, +"91176": {"id": "KHNG", "name": "KANEOHE BAY MCAS", "state": "US", "lat": 21.450, "lon": -157.780, "elev": 3}, +"91182": {"id": "KHIK", "name": "HONOLULU/(HICKAM FLD)", "state": "US", "lat": 21.330, "lon": -157.950, "elev": 8}, +"91190": {"id": "", "name": "PUUNENE", "state": "US", "lat": 20.900, "lon": -156.430, "elev": 20}, +"91199": {"id": "KNPS", "name": "PEARL HARBOR", "state": "US", "lat": 21.350, "lon": -157.950, "elev": 7}, +"91212": {"id": "PGUM", "name": "AGANA (NWS)", "state": "MY", "lat": 13.480, "lon": 144.800, "elev": 91}, +"91217": {"id": "PGAC", "name": "GUAM MARIANA ISLAND", "state": "MY", "lat": 13.550, "lon": 144.830, "elev": 111}, +"91232": {"id": "PGSN", "name": "SAIPAN", "state": "MY", "lat": 15.120, "lon": 145.730, "elev": 65}, +"91245": {"id": "PWAK", "name": "WAKE ISLAND", "state": "WK", "lat": 12.280, "lon": 166.650, "elev": 4}, +"91275": {"id": "PJON", "name": "JOHNSTON ISLAND", "state": "PN", "lat": 16.730, "lon": -169.520, "elev": 5}, +"91285": {"id": "PHTO", "name": "HILO/LYMAN", "state": "HI US", "lat": 19.720, "lon": -155.070, "elev": 11}, +"91287": {"id": "", "name": "CAPE KUMUKAHI", "state": "HI US", "lat": 19.520, "lon": -154.820, "elev": 49}, +"91289": {"id": "", "name": "HALEMAUMAU", "state": "HI US", "lat": 19.400, "lon": -155.280, "elev": 1112}, +"91328": {"id": "", "name": "ULUI ATOLL", "state": "KA", "lat": 8.700, "lon": 149.700, "elev": 2}, +"91334": {"id": "PTKK", "name": "CHUUK/EAST CAROLINE IS", "state": "KA", "lat": 7.450, "lon": 151.830, "elev": 3}, +"91347": {"id": "", "name": "POHNPEI", "state": "KA", "lat": 6.970, "lon": 158.230, "elev": 37}, +"91348": {"id": "PTPN", "name": "PONAPE/EAST CAROLINE IS", "state": "KA", "lat": 6.970, "lon": 158.220, "elev": 39}, +"91364": {"id": "", "name": "ROI NAMUR", "state": "PN", "lat": 9.400, "lon": 167.470, "elev": 4}, +"91366": {"id": "PKWA", "name": "KWAJALEIN ISLAND", "state": "MH", "lat": 8.730, "lon": 167.730, "elev": 8}, +"91368": {"id": "", "name": "AILINGLAPALAP", "state": "MH", "lat": 7.280, "lon": 168.850, "elev": 2}, +"91376": {"id": "PKMJ", "name": "MARSHALL ISLANDS", "state": "MM", "lat": 7.080, "lon": 171.380, "elev": 3}, +"91378": {"id": "", "name": "MILI ATOLL", "state": "MH", "lat": 6.080, "lon": 171.730, "elev": 4}, +"91385": {"id": "", "name": "PALMYRA ISLAND", "state": "PN", "lat": 5.880, "lon": -162.050, "elev": 2}, +"91408": {"id": "PTRO", "name": "PALAU/WEST CAROLINE IS", "state": "KA", "lat": 7.330, "lon": 134.480, "elev": 33}, +"91413": {"id": "PTYA", "name": "LUWEECH/WEST CAROLINE IS", "state": "KA", "lat": 9.480, "lon": 138.080, "elev": 17}, +"91434": {"id": "", "name": "KAPINGAMARANGI", "state": "KA", "lat": 1.080, "lon": 154.770, "elev": 3}, +"91492": {"id": "", "name": "CHRISTMAS ISLAND", "state": "KA", "lat": 2.000, "lon": -157.400, "elev": 3}, +"91517": {"id": "", "name": "HONIARA", "state": "SO", "lat": -9.420, "lon": 159.970, "elev": 56}, +"91532": {"id": "", "name": "NAURU ARC-2", "state": "SW-PAC", "lat": -0.520, "lon": 166.920, "elev": 7}, +"91557": {"id": "NVVV", "name": "BAUERFIELD", "state": "NH", "lat": -17.700, "lon": 168.300, "elev": 21}, +"91577": {"id": "NWWK", "name": "KOUMAC", "state": "NC", "lat": -20.570, "lon": 164.280, "elev": 18}, +"91582": {"id": "", "name": "OUANAHAM", "state": "NC", "lat": -20.770, "lon": 167.230, "elev": 29}, +"91592": {"id": "NWWN", "name": "NOUMEA", "state": "NC", "lat": -22.270, "lon": 166.450, "elev": 72}, +"91610": {"id": "NGTA", "name": "TARAWA/BONRIKI INTL", "state": "KB", "lat": 1.350, "lon": 172.920, "elev": 4}, +"91643": {"id": "NGFU", "name": "FUNAFUTI INTL ARPT", "state": "TV", "lat": -8.520, "lon": 179.220, "elev": 2}, +"91644": {"id": "", "name": "FUNAFUTI", "state": "TV", "lat": -8.520, "lon": 179.230, "elev": 0}, +"91650": {"id": "NFNR", "name": "ROTUMA ISLAND", "state": "FJ", "lat": -12.500, "lon": 177.050, "elev": 26}, +"91680": {"id": "NFFN", "name": "NANDI/NADI INTL", "state": "FJ", "lat": -17.750, "lon": 177.450, "elev": 18}, +"91700": {"id": "KCIS", "name": "CANTON ISLAND", "state": "CT", "lat": -2.770, "lon": -171.720, "elev": 4}, +"91701": {"id": "KCIS", "name": "CANTON ISLAND", "state": "CT", "lat": -2.770, "lon": -171.720, "elev": 4}, +"91727": {"id": "", "name": "FENUAFALA", "state": "NZ", "lat": -9.380, "lon": -171.270, "elev": 4}, +"91765": {"id": "NSTU", "name": "PAGO PAGO", "state": "ZM", "lat": -14.330, "lon": -170.720, "elev": 3}, +"91788": {"id": "", "name": "NUKUALDFA", "state": "TO", "lat": -21.130, "lon": -175.180, "elev": 3}, +"91792": {"id": "NFTF", "name": "FUA'AMOTU", "state": "TO", "lat": -21.230, "lon": -175.150, "elev": 41}, +"91801": {"id": "", "name": "PENRHYN ISLAND", "state": "KU", "lat": -9.000, "lon": -158.050, "elev": 1}, +"91809": {"id": "", "name": "MANIHIKI AWS", "state": "KU", "lat": -10.430, "lon": -161.020, "elev": 2}, +"91811": {"id": "", "name": "PUKAPUKA ATOLL", "state": "KU", "lat": -10.880, "lon": -165.820, "elev": 3}, +"91830": {"id": "", "name": "AITUTAKI", "state": "KU", "lat": -18.830, "lon": -159.770, "elev": 6}, +"91843": {"id": "NCRG", "name": "AVARUA/RAROTONGA IL", "state": "KU", "lat": -21.200, "lon": -159.820, "elev": 7}, +"91848": {"id": "", "name": "MANGAIA AWS", "state": "KU", "lat": -21.920, "lon": -157.950, "elev": 41}, +"91925": {"id": "", "name": "ATUONA", "state": "PF", "lat": -9.800, "lon": -139.030, "elev": 52}, +"91929": {"id": "", "name": "BORA BORA", "state": "PF", "lat": -16.450, "lon": -151.750, "elev": 3}, +"91938": {"id": "NTAA", "name": "TAHITI ISLAND/FAAA", "state": "PF", "lat": -17.550, "lon": -149.620, "elev": 2}, +"91943": {"id": "", "name": "TAKAROA ATOLL", "state": "PF", "lat": -14.480, "lon": -145.030, "elev": 3}, +"91944": {"id": "NTTO", "name": "HAO/TUAMOTO", "state": "PF", "lat": -18.070, "lon": -140.950, "elev": 3}, +"91945": {"id": "", "name": "HEREHERETUE ISLAND", "state": "PF", "lat": -19.870, "lon": -145.000, "elev": 3}, +"91948": {"id": "", "name": "RIKITEA", "state": "PF", "lat": -23.130, "lon": -134.970, "elev": 89}, +"91952": {"id": "NTTX", "name": "MUROROA ATOLL", "state": "PF", "lat": -21.820, "lon": -138.800, "elev": 3}, +"91954": {"id": "NTAT", "name": "TUBUAI ISLAND", "state": "PF", "lat": -23.350, "lon": -149.480, "elev": 3}, +"91958": {"id": "", "name": "RAPA ISLAND", "state": "PF", "lat": -27.620, "lon": -144.330, "elev": 2}, +"92014": {"id": "AYMD", "name": "MADANG W.O.", "state": "NG", "lat": -5.220, "lon": 145.780, "elev": 4}, +"92035": {"id": "AYPY", "name": "PORT MORESBY", "state": "NG", "lat": -9.380, "lon": 147.220, "elev": 58}, +"92044": {"id": "", "name": "MOMOTE W.O.", "state": "NG", "lat": -2.070, "lon": 147.430, "elev": 5}, +"92076": {"id": "", "name": "KAVIENG W.O.", "state": "NG", "lat": -2.580, "lon": 150.800, "elev": 4}, +"92087": {"id": "", "name": "MISIMA W.O.", "state": "NG", "lat": -10.680, "lon": 152.830, "elev": 6}, +"93072": {"id": "", "name": "MT. TAMAHUNGA", "state": "NZ", "lat": -36.300, "lon": 174.700, "elev": 452}, +"93112": {"id": "NZWP", "name": "WHENUAPAI (NZ-AFB)", "state": "NZ", "lat": -36.780, "lon": 174.630, "elev": 27}, +"93185": {"id": "", "name": "TAURANGA AERO", "state": "NZ", "lat": -37.670, "lon": 176.200, "elev": 5}, +"93291": {"id": "NZGS", "name": "GISBORNE AERO", "state": "NZ", "lat": -38.670, "lon": 177.980, "elev": 8}, +"93308": {"id": "NZNP", "name": "NEW PLYMOUTH AERO", "state": "NZ", "lat": -39.020, "lon": 174.180, "elev": 36}, +"93336": {"id": "", "name": "WAIOURU AERO", "state": "NZ", "lat": -39.450, "lon": 175.670, "elev": 823}, +"93417": {"id": "NZPP", "name": "PARAPARAUMU AERO", "state": "NZ", "lat": -40.900, "lon": 174.980, "elev": 12}, +"93431": {"id": "", "name": "OUTLOOK HILL", "state": "NZ", "lat": -41.300, "lon": 174.630, "elev": 548}, +"93515": {"id": "", "name": "WESTPORT AERO", "state": "NZ", "lat": -41.730, "lon": 171.570, "elev": 18}, +"93577": {"id": "", "name": "BLENHEIM AERO", "state": "NZ", "lat": -41.520, "lon": 173.870, "elev": 32}, +"93614": {"id": "NZHK", "name": "HOKITIKA", "state": "NZ", "lat": -42.720, "lon": 170.980, "elev": 40}, +"93769": {"id": "", "name": "RAKAIA", "state": "NZ", "lat": -43.780, "lon": 172.020, "elev": 124}, +"93772": {"id": "", "name": "TIMARU AERO", "state": "NZ", "lat": -44.300, "lon": 171.220, "elev": 27}, +"93780": {"id": "NZCH", "name": "CHRISTCHURCH", "state": "NZ", "lat": -43.480, "lon": 172.550, "elev": 34}, +"93844": {"id": "NZNV", "name": "INVERCARGILL AERO", "state": "NZ", "lat": -46.420, "lon": 168.330, "elev": 1}, +"93845": {"id": "", "name": "INVERCARGILL", "state": "NZ", "lat": -46.420, "lon": 168.330, "elev": 2}, +"93944": {"id": "", "name": "CAMPBELL ISLAND", "state": "NZ", "lat": -52.550, "lon": 169.150, "elev": 15}, +"93986": {"id": "NZCI", "name": "CHATHAM ISL/TUUTA", "state": "NZ", "lat": -43.950, "lon": -176.570, "elev": 48}, +"93997": {"id": "NZRN", "name": "RAOUL ISL/KERMADEC", "state": "NZ", "lat": -29.250, "lon": -177.920, "elev": 49}, +"94014": {"id": "", "name": "MADANG", "state": "AU", "lat": -5.220, "lon": 145.800, "elev": 6}, +"94085": {"id": "", "name": "RABAUL", "state": "AU", "lat": -4.220, "lon": 152.180, "elev": 6}, +"94102": {"id": "", "name": "TROUGHTON ISLAND", "state": "WE AU", "lat": -13.750, "lon": 126.130, "elev": 8}, +"94120": {"id": "YPDN", "name": "DARWIN (CIV/MIL)", "state": "NT AU", "lat": -12.400, "lon": 130.870, "elev": 30}, +"94135": {"id": "", "name": "MC CLER ISLAND", "state": "NT AU", "lat": -11.070, "lon": 132.980, "elev": 5}, +"94150": {"id": "YDGV", "name": "GOVE AIRPORT", "state": "NT AU", "lat": -12.270, "lon": 136.820, "elev": 54}, +"94170": {"id": "YBWP", "name": "WEIPA AERO", "state": "QU AU", "lat": -12.630, "lon": 141.900, "elev": 0}, +"94175": {"id": "", "name": "THURSDAY", "state": "QU AU", "lat": -10.580, "lon": 142.220, "elev": 61}, +"94203": {"id": "YBRM", "name": "BROOME AIRPORT", "state": "WE AU", "lat": -17.950, "lon": 122.220, "elev": 9}, +"94212": {"id": "YHLC", "name": "HALLS CREEK AMO", "state": "WE AU", "lat": -18.230, "lon": 127.670, "elev": 422}, +"94218": {"id": "", "name": "INVERWAY", "state": "NT AU", "lat": -17.850, "lon": 129.630, "elev": 403}, +"94237": {"id": "", "name": "LARRIMAH", "state": "NT AU", "lat": -15.580, "lon": 133.220, "elev": 181}, +"94238": {"id": "YTNK", "name": "TENNANT CREEK", "state": "NT AU", "lat": -19.630, "lon": 134.180, "elev": 375}, +"94287": {"id": "YBCS", "name": "CAIRNS AIRPORT", "state": "QU AU", "lat": -16.880, "lon": 145.750, "elev": 7}, +"94292": {"id": "", "name": "CARDWELL", "state": "QU AU", "lat": -18.250, "lon": 146.020, "elev": 7}, +"94294": {"id": "YBTL", "name": "TOWNSVILLE(CIV/MIL)", "state": "QU AU", "lat": -19.250, "lon": 146.750, "elev": 6}, +"94299": {"id": "", "name": "WILLIS ISLAND", "state": "QU AU", "lat": -16.300, "lon": 149.980, "elev": 9}, +"94300": {"id": "YCAR", "name": "CARNARVON AIRPORT", "state": "WE AU", "lat": -24.880, "lon": 113.670, "elev": 4}, +"94302": {"id": "YPLM", "name": "LEARMONTH", "state": "WE AU", "lat": -22.230, "lon": 114.080, "elev": 6}, +"94312": {"id": "YPPD", "name": "PORT HEDLAND ARRT", "state": "WE AU", "lat": -20.370, "lon": 118.620, "elev": 6}, +"94326": {"id": "YBAS", "name": "ALICE SPRINGS ARPT", "state": "NT AU", "lat": -23.800, "lon": 133.900, "elev": 541}, +"94332": {"id": "YBMA", "name": "MOUNT ISA AIRPORT", "state": "QU AU", "lat": -20.670, "lon": 139.480, "elev": 344}, +"94337": {"id": "", "name": "JULIA CREEK", "state": "QU AU", "lat": -20.650, "lon": 141.730, "elev": 124}, +"94346": {"id": "YBLR", "name": "LONGREACH AIRPORT", "state": "QU AU", "lat": -23.430, "lon": 144.270, "elev": 193}, +"94367": {"id": "YBMK", "name": "MACKAY", "state": "QU AU", "lat": -21.120, "lon": 149.220, "elev": 33}, +"94374": {"id": "YBRK", "name": "ROCKHAMPTON AIRPORT", "state": "QU AU", "lat": -23.380, "lon": 150.470, "elev": 14}, +"94380": {"id": "YBGL", "name": "GLADSTONE", "state": "QU AU", "lat": -23.850, "lon": 151.250, "elev": 75}, +"94401": {"id": "", "name": "KALBARRI", "state": "WE AU", "lat": -27.720, "lon": 114.170, "elev": 9}, +"94403": {"id": "YPGN", "name": "GERALDTON AIRPORT", "state": "WE AU", "lat": -28.780, "lon": 114.700, "elev": 34}, +"94430": {"id": "YPMR", "name": "MEEKATHARRA AIRPORT", "state": "WE AU", "lat": -26.600, "lon": 118.530, "elev": 518}, +"94461": {"id": "YGLS", "name": "GILES MET STATION", "state": "WE AU", "lat": -25.030, "lon": 128.280, "elev": 599}, +"94510": {"id": "YBCV", "name": "CHARLEVILLE ARPT", "state": "QU AU", "lat": -26.400, "lon": 146.270, "elev": 304}, +"94511": {"id": "", "name": "INJUNE", "state": "QU AU", "lat": -25.850, "lon": 148.570, "elev": 389}, +"94527": {"id": "", "name": "MOREE", "state": "NW AU", "lat": -29.470, "lon": 149.850, "elev": 212}, +"94578": {"id": "YBBN", "name": "BRISBANE INTL ARPT", "state": "QU AU", "lat": -27.380, "lon": 153.100, "elev": 5}, +"94610": {"id": "YPPH", "name": "PERTH INTL/BELMONT", "state": "WE AU", "lat": -31.930, "lon": 115.950, "elev": 29}, +"94637": {"id": "YPKG", "name": "KALGOORLIE/BOULDER", "state": "WE AU", "lat": -30.770, "lon": 121.450, "elev": 360}, +"94638": {"id": "YESP", "name": "ESPERANCE", "state": "WE AU", "lat": -33.820, "lon": 121.880, "elev": 26}, +"94646": {"id": "", "name": "FORREST", "state": "WE AU", "lat": -30.830, "lon": 128.100, "elev": 156}, +"94647": {"id": "", "name": "EUCLA", "state": "WE AU", "lat": -31.670, "lon": 128.880, "elev": 99}, +"94653": {"id": "YPCD", "name": "CEDUNA AIRPORT", "state": "SA AU", "lat": -32.120, "lon": 133.700, "elev": 17}, +"94659": {"id": "YPWR", "name": "WOOMERA (AUS-AFB)", "state": "SA AU", "lat": -31.130, "lon": 136.820, "elev": 167}, +"94662": {"id": "", "name": "WHYALLA AIRPORT", "state": "SA AU", "lat": -33.050, "lon": 137.500, "elev": 11}, +"94672": {"id": "YPAD", "name": "ADELAIDE INTL ARPT", "state": "SA AU", "lat": -34.930, "lon": 138.520, "elev": 4}, +"94693": {"id": "YMMI", "name": "MILDURA AIRPORT", "state": "VC AU", "lat": -34.230, "lon": 142.080, "elev": 50}, +"94711": {"id": "YCBA", "name": "COBAR", "state": "NW AU", "lat": -31.480, "lon": 145.820, "elev": 265}, +"94750": {"id": "YSNW", "name": "NOWRA(AUS-NAVY)", "state": "NW AU", "lat": -34.950, "lon": 150.530, "elev": 110}, +"94767": {"id": "YSSY", "name": "SYDNEY INTL AIRPORT", "state": "NW AU", "lat": -33.950, "lon": 151.180, "elev": 3}, +"94776": {"id": "YSWM", "name": "WILLIAMTOWN(AUS-AB)", "state": "NW AU", "lat": -32.780, "lon": 151.820, "elev": 8}, +"94791": {"id": "YSCH", "name": "COFFS HARBOUR AIRPORT", "state": "NW AU", "lat": -30.320, "lon": 153.120, "elev": 5}, +"94802": {"id": "YPAL", "name": "ALBANY AIRPORT", "state": "WE AU", "lat": -34.930, "lon": 117.800, "elev": 69}, +"94821": {"id": "YMMG", "name": "MOUNT GAMBIER ARPT", "state": "SA AU", "lat": -37.730, "lon": 140.780, "elev": 69}, +"94827": {"id": "", "name": "NHILL", "state": "VC AU", "lat": -36.350, "lon": 141.650, "elev": 130}, +"94857": {"id": "", "name": "GEELONG ARPT", "state": "VC AU", "lat": -38.230, "lon": 144.330, "elev": 35}, +"94865": {"id": "YMLV", "name": "LAVERTON AERO", "state": "VC AU", "lat": -37.850, "lon": 144.730, "elev": 21}, +"94866": {"id": "YMML", "name": "MELBOURNE INTL ARPT", "state": "VC AU", "lat": -37.670, "lon": 144.830, "elev": 141}, +"94907": {"id": "YMES", "name": "EAST SALE", "state": "VC AU", "lat": -38.100, "lon": 147.130, "elev": 8}, +"94910": {"id": "YSWG", "name": "WAGGA WAGGA(CV/MIL)", "state": "NW AU", "lat": -35.170, "lon": 147.450, "elev": 212}, +"94911": {"id": "", "name": "OMEO", "state": "VC AU", "lat": -37.080, "lon": 147.600, "elev": 679}, +"94922": {"id": "", "name": "TIDBINBILLA", "state": "NW AU", "lat": -35.450, "lon": 148.930, "elev": 226}, +"94926": {"id": "YSCB", "name": "CANBERRA", "state": "CT AU", "lat": -35.300, "lon": 149.200, "elev": 580}, +"94968": {"id": "YMLT", "name": "LAUNCESTON AIRPORT", "state": "TA AU", "lat": -41.530, "lon": 147.200, "elev": 178}, +"94975": {"id": "YMHB", "name": "HOBART AIRPORT", "state": "TA AU", "lat": -42.830, "lon": 147.480, "elev": 27}, +"94995": {"id": "YLHI", "name": "LORD HOWE ISLAND", "state": "AU", "lat": -31.530, "lon": 159.070, "elev": 6}, +"94996": {"id": "YSNF", "name": "NORFOLK ISLAND ARPT", "state": "AU", "lat": -29.030, "lon": 167.930, "elev": 109}, +"94998": {"id": "YMMQ", "name": "MACQUARIE ISLAND", "state": "AU", "lat": -54.480, "lon": 158.930, "elev": 6}, +"95308": {"id": "", "name": "KARRATHA", "state": "WE AU", "lat": -20.620, "lon": 116.750, "elev": 16}, +"95527": {"id": "YMOR", "name": "MOREE AMO", "state": "NW AU", "lat": -29.500, "lon": 149.830, "elev": 214}, +"95591": {"id": "", "name": "IPSWICH", "state": "QU AU", "lat": -27.620, "lon": 152.750, "elev": 40}, +"95605": {"id": "", "name": "FREMANTLE", "state": "WE AU", "lat": -32.050, "lon": 115.770, "elev": 4}, +"95610": {"id": "", "name": "KALAMUNDA", "state": "WE AU", "lat": -32.020, "lon": 116.130, "elev": 384}, +"96009": {"id": "WITM", "name": "LHOKSEUMAWE/MALIKUSSALEH", "state": "ID", "lat": 5.230, "lon": 97.200, "elev": 87}, +"96011": {"id": "WITT", "name": "BANDA ACEH/BLANGBINTANG", "state": "ID", "lat": 5.520, "lon": 95.420, "elev": 21}, +"96015": {"id": "WITC", "name": "MEULABOH/CUT NYAK", "state": "ID", "lat": 4.250, "lon": 96.120, "elev": 90}, +"96033": {"id": "", "name": "BELAWAN", "state": "ID", "lat": 3.800, "lon": 98.700, "elev": 3}, +"96035": {"id": "WIMM", "name": "MEDAN/POLONIA (MIL)", "state": "ID", "lat": 3.570, "lon": 98.680, "elev": 25}, +"96039": {"id": "", "name": "TEBINGTINGGI", "state": "ID", "lat": 3.370, "lon": 99.120, "elev": 7}, +"96073": {"id": "WIMS", "name": "SIBOLGA/PINANGS", "state": "ID", "lat": 1.550, "lon": 98.880, "elev": 3}, +"96075": {"id": "WIMB", "name": "GUNUNG SITOLI/BINAKA", "state": "ID", "lat": 1.500, "lon": 97.630, "elev": 6}, +"96091": {"id": "WIKN", "name": "TANJUNGPINANG", "state": "ID", "lat": 0.920, "lon": 104.530, "elev": 18}, +"96109": {"id": "WIBB", "name": "PAKANBARU", "state": "ID", "lat": 0.470, "lon": 101.450, "elev": 31}, +"96145": {"id": "", "name": "TAREMPA", "state": "ID", "lat": 3.200, "lon": 106.250, "elev": 3}, +"96147": {"id": "WION", "name": "RANAI (MIL/CIV)", "state": "ID", "lat": 3.950, "lon": 108.380, "elev": 2}, +"96163": {"id": "WIMG", "name": "PADANG/TABING", "state": "ID", "lat": -0.880, "lon": 100.350, "elev": 3}, +"96171": {"id": "", "name": "RENGAT", "state": "ID", "lat": 0.500, "lon": 102.300, "elev": 46}, +"96179": {"id": "WIKS", "name": "SINGKEP/DABO", "state": "ID", "lat": -0.480, "lon": 104.580, "elev": 31}, +"96195": {"id": "WIPA", "name": "JAMBI/SULTAN", "state": "ID", "lat": -1.630, "lon": 103.650, "elev": 25}, +"96207": {"id": "", "name": "KERINCI/DEPATI PARBO", "state": "ID", "lat": -2.460, "lon": 101.220, "elev": 782}, +"96221": {"id": "WIPP", "name": "PALEMBANG/ST.N.BADARUDIN", "state": "ID", "lat": -2.900, "lon": 104.700, "elev": 10}, +"96237": {"id": "WIKK", "name": "PANGKAL-PINANG", "state": "ID", "lat": -2.170, "lon": 106.130, "elev": 33}, +"96249": {"id": "WIKD", "name": "TANJUNG PANDAN", "state": "ID", "lat": -2.750, "lon": 107.750, "elev": 44}, +"96253": {"id": "WIPL", "name": "BENGKULA/PADANG", "state": "ID", "lat": -3.880, "lon": 102.330, "elev": 16}, +"96295": {"id": "", "name": "TAJUNG KARANG/RADIN", "state": "ID", "lat": -5.010, "lon": 115.180, "elev": 96}, +"96315": {"id": "WBSB", "name": "BRUNEI AIRPORT", "state": "BF", "lat": 4.930, "lon": 114.930, "elev": 15}, +"96323": {"id": "", "name": "KUALA BELAIT", "state": "ID", "lat": 4.580, "lon": 114.200, "elev": 3}, +"96413": {"id": "WBGG", "name": "KUCHING (CIV/MIL)", "state": "MS", "lat": 1.480, "lon": 110.330, "elev": 27}, +"96421": {"id": "WBGS", "name": "SIBU", "state": "MS", "lat": 2.330, "lon": 111.830, "elev": 8}, +"96441": {"id": "WBGB", "name": "BINTULU/KALIMANTAN", "state": "MS", "lat": 3.200, "lon": 113.030, "elev": 5}, +"96471": {"id": "WBKK", "name": "KOTA KINABALU INTL", "state": "MS", "lat": 5.930, "lon": 116.050, "elev": 3}, +"96481": {"id": "WBKW", "name": "TAWAU/KALIMANTAN IL", "state": "MS", "lat": 4.270, "lon": 117.880, "elev": 20}, +"96491": {"id": "WBKS", "name": "SANDAKAN/KALIMANTAN", "state": "MS", "lat": 5.900, "lon": 118.070, "elev": 13}, +"96509": {"id": "WRLR", "name": "TARAKAN/JUWATA", "state": "ID", "lat": 3.330, "lon": 117.570, "elev": 6}, +"96529": {"id": "WRLK", "name": "TANJUNG REDEP/BERAU", "state": "ID", "lat": 2.120, "lon": 117.450, "elev": 26}, +"96535": {"id": "WIOH", "name": "PALOH", "state": "ID", "lat": 1.700, "lon": 109.300, "elev": 15}, +"96565": {"id": "", "name": "PUTUSIBAU", "state": "ID", "lat": 0.880, "lon": 112.930, "elev": 17}, +"96581": {"id": "WIOO", "name": "PONTIANAK", "state": "ID", "lat": -0.150, "lon": 109.400, "elev": 3}, +"96595": {"id": "", "name": "MUARATEWE", "state": "ID", "lat": -0.950, "lon": 114.900, "elev": 60}, +"96607": {"id": "", "name": "SAMARINDA", "state": "ID", "lat": -0.620, "lon": 117.150, "elev": 230}, +"96633": {"id": "WRLL", "name": "BALIKPAPAN/SEPINGGAN", "state": "ID", "lat": -1.270, "lon": 116.900, "elev": 3}, +"96645": {"id": "WRBI", "name": "PANGKALAN", "state": "ID", "lat": -2.700, "lon": 111.700, "elev": 25}, +"96651": {"id": "", "name": "SAMPIT/HAJI", "state": "ID", "lat": -2.520, "lon": 112.950, "elev": 27}, +"96655": {"id": "WRBP", "name": "PANARUNG", "state": "ID", "lat": -1.000, "lon": 114.000, "elev": 27}, +"96685": {"id": "WRBB", "name": "BANJARMASIN NOOR", "state": "ID", "lat": -3.430, "lon": 114.750, "elev": 20}, +"96737": {"id": "", "name": "SERANG", "state": "ID", "lat": -6.120, "lon": 106.130, "elev": 40}, +"96739": {"id": "WIIA", "name": "CURUG/BUDIARTO", "state": "ID", "lat": -6.230, "lon": 106.650, "elev": 46}, +"96749": {"id": "WIII", "name": "SOEKARNO-HATTA INTL", "state": "ID", "lat": -6.120, "lon": 106.650, "elev": 8}, +"96781": {"id": "", "name": "BANDUNG/HUSEIN", "state": "ID", "lat": -6.540, "lon": 107.350, "elev": 740}, +"96791": {"id": "", "name": "CIREBON/JATIWANGI", "state": "ID", "lat": -6.450, "lon": 108.160, "elev": 50}, +"96797": {"id": "", "name": "TEGAL", "state": "ID", "lat": -6.510, "lon": 109.090, "elev": 10}, +"96801": {"id": "WIAM", "name": "CIBEUREM", "state": "ID", "lat": -7.330, "lon": 108.250, "elev": 335}, +"96805": {"id": "WIIL", "name": "CILACAP", "state": "ID", "lat": -7.730, "lon": 109.020, "elev": 6}, +"96839": {"id": "WIIS", "name": "SEMARANG/AHMAD YANI", "state": "ID", "lat": -6.980, "lon": 110.380, "elev": 3}, +"96845": {"id": "", "name": "SURAKARTA PANASAN", "state": "ID", "lat": -7.870, "lon": 110.920, "elev": 104}, +"96925": {"id": "", "name": "SANGKAPURA/BAWEAN IS.", "state": "ID", "lat": -5.800, "lon": 112.600, "elev": 3}, +"96933": {"id": "", "name": "SUBABAJA/PERAK", "state": "ID", "lat": -7.220, "lon": 112.720, "elev": 3}, +"96935": {"id": "WRSJ", "name": "SURABAYA/JUANDA MIL", "state": "ID", "lat": -7.370, "lon": 112.770, "elev": 3}, +"96973": {"id": "", "name": "KALIANGET (MADURA IS.)", "state": "ID", "lat": -7.050, "lon": 113.970, "elev": 3}, +"96987": {"id": "", "name": "BANYUWANGI", "state": "ID", "lat": -8.370, "lon": 114.630, "elev": 5}, +"96996": {"id": "YPCC", "name": "COCOS ISL INTL ARPT", "state": "AU", "lat": -12.180, "lon": 96.820, "elev": 3}, +"97008": {"id": "WAMH", "name": "NAHA/TAHUNA", "state": "ID", "lat": 3.580, "lon": 12.470, "elev": 38}, +"97014": {"id": "WAMM", "name": "MENADO/SAM RATULANG", "state": "ID", "lat": 1.530, "lon": 124.920, "elev": 80}, +"97028": {"id": "WAMI", "name": "TOLI-TOLI/LALOS", "state": "ID", "lat": 1.020, "lon": 120.800, "elev": 2}, +"97048": {"id": "WAMG", "name": "GORONTALO/JALALUDDIN", "state": "ID", "lat": 0.520, "lon": 123.070, "elev": 2}, +"97072": {"id": "WAML", "name": "PALU/MUTIARA", "state": "ID", "lat": -0.680, "lon": 119.730, "elev": 6}, +"97086": {"id": "WAMW", "name": "LUWUK/BUBUNG", "state": "ID", "lat": -0.900, "lon": 122.780, "elev": 2}, +"97096": {"id": "WAMP", "name": "POSO", "state": "ID", "lat": -1.380, "lon": 120.730, "elev": 2}, +"97100": {"id": "", "name": "KOLONDALE", "state": "ID", "lat": -2.000, "lon": 121.320, "elev": 2}, +"97120": {"id": "", "name": "MAJENE", "state": "ID", "lat": -2.500, "lon": 119.000, "elev": 8}, +"97146": {"id": "", "name": "KENDARI", "state": "ID", "lat": -4.100, "lon": 122.430, "elev": 50}, +"97180": {"id": "WAAA", "name": "HASANUDDIN/UJUNG", "state": "ID", "lat": -5.070, "lon": 119.550, "elev": 14}, +"97182": {"id": "", "name": "UJANG PANDANG", "state": "ID", "lat": -5.070, "lon": 119.570, "elev": 14}, +"97192": {"id": "WAAB", "name": "BAU-BAU/BETO AMBIRI", "state": "ID", "lat": -5.470, "lon": 122.620, "elev": 2}, +"97230": {"id": "WRRR", "name": "DENPASAR/NGURAH RAI", "state": "ID", "lat": -8.450, "lon": 115.100, "elev": 1}, +"97240": {"id": "", "name": "MATARAM/SELAPARANG", "state": "ID", "lat": -8.880, "lon": 116.120, "elev": 3}, +"97260": {"id": "WRRS", "name": "SUMBAWA BESAR/BRAGBIJI", "state": "ID", "lat": -8.430, "lon": 117.420, "elev": 3}, +"97270": {"id": "WRRB", "name": "BIMA/M.SALAHUDDIN", "state": "ID", "lat": -8.550, "lon": 118.700, "elev": 2}, +"97300": {"id": "WRKC", "name": "MAUMERE/WAI", "state": "ID", "lat": -8.630, "lon": 122.250, "elev": 3}, +"97310": {"id": "WRKL", "name": "LARANTUKA", "state": "ID", "lat": -8.270, "lon": 122.970, "elev": 9}, +"97340": {"id": "WRRW", "name": "WAINGAPU/MAU-HAU", "state": "ID", "lat": -9.670, "lon": 120.330, "elev": 12}, +"97372": {"id": "WRKK", "name": "KUPANG/EL TARI", "state": "ID", "lat": -10.170, "lon": 123.670, "elev": 108}, +"97390": {"id": "", "name": "DILI", "state": "ID", "lat": -8.570, "lon": 125.570, "elev": 4}, +"97430": {"id": "WAMT", "name": "TERNATE/BABULIAH", "state": "ID", "lat": 0.780, "lon": 127.380, "elev": 23}, +"97460": {"id": "", "name": "LABUHA/OESMAN SADIK", "state": "ID", "lat": -0.630, "lon": 127.500, "elev": 3}, +"97502": {"id": "WASS", "name": "SORONG/JEFMAN", "state": "ID", "lat": -0.930, "lon": 131.120, "elev": 3}, +"97530": {"id": "WASR", "name": "MANOKWARI", "state": "ID", "lat": -0.880, "lon": 134.050, "elev": 3}, +"97560": {"id": "WABB", "name": "BIAK/FRANS KAISIEPO", "state": "ID", "lat": -1.180, "lon": 136.120, "elev": 11}, +"97580": {"id": "WAJI", "name": "SARMI/ORAI", "state": "ID", "lat": -1.830, "lon": 138.720, "elev": 3}, +"97600": {"id": "WAPN", "name": "SANANA", "state": "ID", "lat": -2.080, "lon": 126.000, "elev": 2}, +"97686": {"id": "WAJW", "name": "WAMENA", "state": "ID", "lat": -4.070, "lon": 138.950, "elev": 1660}, +"97690": {"id": "WAJJ", "name": "JAYAPURA/SENTANI", "state": "ID", "lat": -2.570, "lon": 140.480, "elev": 99}, +"97724": {"id": "WAPP", "name": "AMBON/PATTIMURA", "state": "ID", "lat": -3.700, "lon": 128.080, "elev": 12}, +"97748": {"id": "", "name": "GESER", "state": "ID", "lat": -3.800, "lon": 130.830, "elev": 3}, +"97760": {"id": "WASK", "name": "KAIMANA", "state": "ID", "lat": -3.670, "lon": 133.750, "elev": 3}, +"97810": {"id": "", "name": "TUAL/DUMATUBUN", "state": "ID", "lat": -5.680, "lon": 132.750, "elev": 12}, +"97876": {"id": "WAKT", "name": "TANAH MERAH", "state": "ID", "lat": -6.100, "lon": 140.300, "elev": 16}, +"97900": {"id": "WAPI", "name": "SAUMLAKI/OLILIT", "state": "ID", "lat": -7.980, "lon": 131.300, "elev": 24}, +"97980": {"id": "WAKK", "name": "MERAUKE/MOPAH", "state": "ID", "lat": -8.470, "lon": 140.380, "elev": 3}, +"98223": {"id": "RPLI", "name": "LAOAG INTL(PH-ARMY)", "state": "PH", "lat": 18.180, "lon": 120.530, "elev": 5}, +"98233": {"id": "RPUT", "name": "TUGUEGARAO", "state": "PH", "lat": 17.640, "lon": 121.750, "elev": 62}, +"98327": {"id": "RPMK", "name": "CLARK AB", "state": "PH", "lat": 14.800, "lon": 120.270, "elev": 17}, +"98328": {"id": "RPUB", "name": "BAGUIO", "state": "PH", "lat": 16.420, "lon": 120.600, "elev": 1501}, +"98426": {"id": "RPLB", "name": "OLONGAPO", "state": "PH", "lat": 14.800, "lon": 120.270, "elev": 17}, +"98433": {"id": "", "name": "TANAY", "state": "PH", "lat": 14.570, "lon": 121.370, "elev": 614}, +"98444": {"id": "RPMP", "name": "LEGAZPI/LUZON ISL", "state": "PH", "lat": 13.130, "lon": 123.730, "elev": 17}, +"98618": {"id": "", "name": "PUERTO PRINCESA", "state": "PH", "lat": 9.750, "lon": 118.730, "elev": 15}, +"98630": {"id": "", "name": "CUYO", "state": "PH", "lat": 10.850, "lon": 121.030, "elev": 4}, +"98646": {"id": "RPMT", "name": "MACTAN INTL(CIV/AF)", "state": "PH", "lat": 10.300, "lon": 123.970, "elev": 24}, +"98653": {"id": "", "name": "SURIGAO", "state": "PH", "lat": 9.800, "lon": 125.500, "elev": 22}, +"98747": {"id": "", "name": "LUMBIA AIRPORT", "state": "PH", "lat": 8.410, "lon": 124.610, "elev": 188}, +"98753": {"id": "", "name": "DAVAO AIRPORT", "state": "PH", "lat": 7.120, "lon": 125.650, "elev": 18}, +"98836": {"id": "", "name": "ZAMBOANGA MINDANAO", "state": "PH", "lat": 6.900, "lon": 122.070, "elev": 6}, +"99017": {"id": "C7L", "name": "SHIP LIMA", "state": "UK", "lat": 57.000, "lon": -20.000, "elev": 0}, +"99018": {"id": "LDWR", "name": "SHIP MIKE", "state": "NO", "lat": 66.000, "lon": 2.000, "elev": 0}, +"99023": {"id": "GACA", "name": "SHIP ROMEO", "state": "FR", "lat": 47.000, "lon": 17.000, "elev": 0} +} diff --git a/share/raob_stations_small.json b/share/raob_stations_small.json new file mode 100644 index 000000000..b109ce88a --- /dev/null +++ b/share/raob_stations_small.json @@ -0,0 +1,8 @@ +{ +"72469": {"id": "KDNR", "name": "DENVER/STAPLETON", "state": "CO", "lat": 39.75, "lon": -104.87, "elev": 1608}, +"72403": {"id": "KIAD", "name": "WASH DC/DULLES", "state": "VA", "lat": 38.93, "lon": -77.45, "elev": 93}, +"17064": {"id": "", "name": "ISTANBUL/KARTAL", "state": "TU", "lat": 40.90, "lon": 29.15, "elev": 18}, +"83840": {"id": "SBCT", "name": "CURITIBA/AFONSO PEN", "state": "BZ", "lat": -25.52, "lon": -49.17, "elev": 908}, +"68263": {"id": "FAIR", "name": "PRETORIA/IRENE", "state": "ZA", "lat": -25.92, "lon": 28.22, "elev": 1500}, +"27713": {"id": "UUDD", "name": "MOSCOW POLGOPRUDNYJ", "state": "RS", "lat": 55.40, "lon": 37.90, "elev": 179} +} diff --git a/src/bufr/BufrParser/BufrCollectors/BufrAccumulator.cpp b/src/bufr/BufrParser/BufrCollectors/BufrAccumulator.cpp deleted file mode 100644 index fb51ce680..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrAccumulator.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "BufrAccumulator.h" - - -#include "eckit/exception/Exceptions.h" - - -namespace Ingester -{ - BufrAccumulator::BufrAccumulator(Eigen::Index numColumns, Eigen::Index blockSize) : - dataArray_(blockSize, numColumns), - numColumns_(numColumns), - numDataRows_(0), - blockSize_(blockSize) - { - } - - void BufrAccumulator::addRow(std::vector& newRow) - { - if (numDataRows_ + 1 > dataArray_.rows()) - { - dataArray_.conservativeResize(dataArray_.rows() + blockSize_, numColumns_); - } - - dataArray_.row(numDataRows_) = Eigen::Map(newRow.data(), 1, numColumns_); - numDataRows_++; - } - - IngesterArray BufrAccumulator::getData(Eigen::Index elementPos, - Eigen::Index numElementsPerSet, - const Channels& indices) - { - if (std::find_if(indices.begin(), indices.end(), - [](const auto x){ return x < 1; }) != indices.end()) - { - throw eckit::BadParameter("All channel numbers must be >= 1."); - } - - IngesterArray resultArr(numDataRows_, indices.size()); - unsigned int colIdx = 0; - for (auto col : indices) - { - unsigned int offset = elementPos + numElementsPerSet * (col - 1); - resultArr.block(0, colIdx, numDataRows_, 1) = - dataArray_.block(0, offset, numDataRows_, 1); - - colIdx++; - } - - return resultArr; - } - - void BufrAccumulator::reset() - { - numDataRows_ = 0; - } -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrCollectors/BufrAccumulator.h b/src/bufr/BufrParser/BufrCollectors/BufrAccumulator.h deleted file mode 100644 index d272928f5..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrAccumulator.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include - -#include "Eigen/Dense" - -#include "BufrParser/BufrTypes.h" - -namespace Ingester -{ - /// \brief Accumulates provided data into a dynamically expanding Eigen Array - class BufrAccumulator - { - public: - /// \param numColumns Width of collected data. - /// \param blockSize The amount to allocate when we need to extend the Eigen Array - explicit BufrAccumulator(Eigen::Index numColumns, Eigen::Index blockSize = 50000); - - /// \brief Add row of data to the internal data structure - /// \param newRow Collection of values to add (size must match the number of columns) - void addRow(std::vector& newRow); - - /// \brief Get an Eigen Array that contains a slice of the collected data. - /// \param elementPos Position of mnemonic in the list of read mnemonics. - /// \param numElementsPerSet Number of mnemonics in the list. - /// \param indices indices to collect starting from the start position - IngesterArray getData(Eigen::Index elementPos, - Eigen::Index numElementsPerSet, - const Channels& indices = {1}); - - /// \brief Start over - void reset(); - - // Getters - inline Eigen::Index getNumColumns() const { return numColumns_; } - - private: - /// \brief Eigen Array that holds the accumulated data - IngesterArray dataArray_; - - /// \brief Total number of columns (width of data structure) - Eigen::Index numColumns_; - - /// \brief Number of data rows of collected data. - Eigen::Index numDataRows_; - - /// \brief Amount to allocate when we need to extend the Eigen Array - Eigen::Index blockSize_; - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrCollectors/BufrCollector.cpp b/src/bufr/BufrParser/BufrCollectors/BufrCollector.cpp deleted file mode 100644 index 427f70154..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrCollector.cpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "BufrCollector.h" - - -namespace Ingester -{ - BufrCollector::BufrCollector(const int fortranFileId, const BufrMnemonicSet mnemonicSet) : - fortranFileId_(fortranFileId), - accumulator_(BufrAccumulator(mnemonicSet.getSize() * mnemonicSet.getMaxColumn())), - mnemonicSet_(mnemonicSet) - { - } - - BufrDataMap BufrCollector::finalize() - { - IngesterArrayMap dataMap; - size_t fieldIdx = 0; - for (const auto &fieldName : mnemonicSet_.getMnemonics()) - { - IngesterArray dataArr = accumulator_.getData(fieldIdx, - mnemonicSet_.getSize(), - mnemonicSet_.getChannels()); - - dataMap.insert({fieldName, dataArr}); - fieldIdx++; - } - - accumulator_.reset(); - - return dataMap; - } -} // namespace Ingester - diff --git a/src/bufr/BufrParser/BufrCollectors/BufrCollector.h b/src/bufr/BufrParser/BufrCollectors/BufrCollector.h deleted file mode 100644 index fcf6a8ae7..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrCollector.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include - -#include "Eigen/Dense" - -#include "BufrParser/BufrTypes.h" -#include "BufrParser/BufrMnemonicSet.h" -#include "BufrAccumulator.h" - - -namespace Ingester -{ - /// \brief Collectors know how to use the BUFR interface to grab data associated with - /// configured mnemonicSets. - class BufrCollector - { - public: - BufrCollector(const int fortranFileId, const BufrMnemonicSet mnemonicSet); - virtual ~BufrCollector() = default; - - /// \brief Grab the data - virtual void collect() = 0; - - /// \brief Get the data we want from the accumulator and make our data map. Resets - /// the accumulator. - BufrDataMap finalize(); - - protected: - /// \brief Fortran file ID for the open BUFR file - const int fortranFileId_; - - /// \brief Accumulator to collect the data we are collecting - BufrAccumulator accumulator_; - - /// \brief Specifies the mnemonics and channels this collector gets from the BUFR file. - const BufrMnemonicSet mnemonicSet_; - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrCollectors/BufrCollectors.cpp b/src/bufr/BufrParser/BufrCollectors/BufrCollectors.cpp deleted file mode 100644 index b06e27343..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrCollectors.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include - -#include - -#include "BufrCollectors.h" -#include "BufrIntCollector.h" -#include "BufrRepCollector.h" -#include "DataContainer.h" - - -namespace Ingester -{ - BufrCollectors::BufrCollectors(unsigned int fortranFileId) : - fortranFileId_(fortranFileId) - { - } - - void BufrCollectors::addMnemonicSets(const std::vector &mnemonicSets) - { - for (const auto &set : mnemonicSets) - { - addMnemonicSet(set); - } - } - - void BufrCollectors::addMnemonicSet(const BufrMnemonicSet &mnemonicSet) - { - if (mnemonicSet.getMaxColumn() == 1) - { - collectors_.push_back(std::make_shared(fortranFileId_, mnemonicSet)); - } - else - { - collectors_.push_back(std::make_shared(fortranFileId_, mnemonicSet)); - } - } - - void BufrCollectors::collect() - { - for (const auto &collector : collectors_) - { - collector->collect(); - } - } - - BufrDataMap BufrCollectors::finalize() - { - auto dataMap = BufrDataMap(); - - for (const auto &collector : collectors_) - { - IngesterArrayMap collectorDataMap = collector->finalize(); - dataMap.insert(collectorDataMap.begin(), collectorDataMap.end()); - } - - return dataMap; - } -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrCollectors/BufrCollectors.h b/src/bufr/BufrParser/BufrCollectors/BufrCollectors.h deleted file mode 100644 index 78d62b065..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrCollectors.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include -#include -#include - -#include "BufrParser/BufrTypes.h" - - -namespace Ingester -{ - class BufrMnemonicSet; - class BufrAccumulator; - class BufrCollector; - - /// \brief Manager of collectors. - class BufrCollectors - { - public: - explicit BufrCollectors(unsigned int fortranFileId); - ~BufrCollectors() = default; - - /// \brief Add collectors for mnemonic sets - /// \param mnemonicSets list of BufrMnemonicSet to use - void addMnemonicSets(const std::vector& mnemonicSets); - - /// \brief Add collector for a mnemonic set - /// \param mnemonicSet BufrMnemonicSet to use - void addMnemonicSet(const BufrMnemonicSet& mnemonicSet); - - /// \brief Cause all the collectors to grab the next peaces of data from the BUFR file. - void collect(); - - /// \brief Finalize all the collectors and assemble the resulting data into a map. - BufrDataMap finalize(); - - private: - /// \brief Fortran file ID for the open BUFR file - unsigned int fortranFileId_; - - /// \brief Collection of all the collectors being managed. - std::vector> collectors_; - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrCollectors/BufrIntCollector.cpp b/src/bufr/BufrParser/BufrCollectors/BufrIntCollector.cpp deleted file mode 100644 index afab02e2b..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrIntCollector.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#if __has_include("bufr_interface.h") // TODO(rmclaren): Remove this in future - #include "bufr_interface.h" -#else - #include "bufr.interface.h" -#endif - -#include "BufrIntCollector.h" - - -namespace Ingester -{ - BufrIntCollector::BufrIntCollector(const int fortranFileId, const BufrMnemonicSet& mnemonicSet): - BufrCollector(fortranFileId, mnemonicSet) - { - scratchData_.resize(accumulator_.getNumColumns()); - floatTypeScratchData_.resize(accumulator_.getNumColumns()); - } - - void BufrIntCollector::collect() - { - double* scratchDataPtr = scratchData_.data(); - - int result; - ufbint_f(fortranFileId_, - reinterpret_cast (&scratchDataPtr), - mnemonicSet_.getSize(), - mnemonicSet_.getMaxColumn(), - &result, - mnemonicSet_.getMnemonicsStr().c_str()); - - for (Eigen::Index colIdx = 0; colIdx < accumulator_.getNumColumns(); colIdx++) - { - floatTypeScratchData_[colIdx] = static_cast(scratchData_[colIdx]); - } - - accumulator_.addRow(floatTypeScratchData_); - } -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrCollectors/BufrIntCollector.h b/src/bufr/BufrParser/BufrCollectors/BufrIntCollector.h deleted file mode 100644 index feb98ae75..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrIntCollector.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "BufrAccumulator.h" -#include "BufrCollector.h" -#include "BufrParser/BufrMnemonicSet.h" -#include "BufrParser/BufrTypes.h" - -#pragma once - -namespace Ingester -{ - /// \brief Collector that uses the BUFR interface ufbint call to grab data (single col data). - class BufrIntCollector: public BufrCollector - { - public: - explicit BufrIntCollector(const int fortranFileId, const BufrMnemonicSet& mnemonicSet); - ~BufrIntCollector() = default; - - /// \brief Grab the next section of data - void collect() final; - - private: - /// \brief Pre-allocated buffer to hand to the Fortran interface. - std::vector scratchData_; - std::vector floatTypeScratchData_; - }; -} // namespace Ingester - diff --git a/src/bufr/BufrParser/BufrCollectors/BufrRepCollector.cpp b/src/bufr/BufrParser/BufrCollectors/BufrRepCollector.cpp deleted file mode 100644 index 3b2886f7c..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrRepCollector.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#if __has_include("bufr_interface.h") // TODO(rmclaren): Remove this in future - #include "bufr_interface.h" -#else - #include "bufr.interface.h" -#endif - -#include "BufrRepCollector.h" - - -namespace Ingester -{ - BufrRepCollector::BufrRepCollector(const int fortranFileId, const BufrMnemonicSet& mnemonicSet): - BufrCollector(fortranFileId, mnemonicSet) - { - scratchData_.resize(accumulator_.getNumColumns()); - floatTypeScratchData_.resize(accumulator_.getNumColumns()); - } - - void BufrRepCollector::collect() - { - double* scratchDataPtr = scratchData_.data(); - - int result; - ufbrep_f(fortranFileId_, - reinterpret_cast(&scratchDataPtr), - mnemonicSet_.getSize(), - mnemonicSet_.getMaxColumn(), - &result, - mnemonicSet_.getMnemonicsStr().c_str()); - - for (Eigen::Index colIdx = 0; colIdx < accumulator_.getNumColumns(); colIdx++) - { - floatTypeScratchData_[colIdx] = static_cast(scratchData_[colIdx]); - } - - accumulator_.addRow(floatTypeScratchData_); - } -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrCollectors/BufrRepCollector.h b/src/bufr/BufrParser/BufrCollectors/BufrRepCollector.h deleted file mode 100644 index 6ebecd57d..000000000 --- a/src/bufr/BufrParser/BufrCollectors/BufrRepCollector.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include "BufrAccumulator.h" -#include "BufrCollector.h" -#include "BufrParser/BufrMnemonicSet.h" -#include "BufrParser/BufrTypes.h" - - -namespace Ingester -{ - /// \brief Collector that uses the BUFR interface ufbrep call to grab data (multi col data). - class BufrRepCollector : public BufrCollector - { - public: - explicit BufrRepCollector(const int fortranFileId, const BufrMnemonicSet& mnemonicSet); - ~BufrRepCollector() = default; - - /// \brief Grab the next section of data - void collect() final; - - private: - /// \brief Pre-allocated buffer to hand to the Fortran interface. - std::vector scratchData_; - std::vector floatTypeScratchData_; - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrDescription.cpp b/src/bufr/BufrParser/BufrDescription.cpp index ccf9bb42c..57464abd2 100644 --- a/src/bufr/BufrParser/BufrDescription.cpp +++ b/src/bufr/BufrParser/BufrDescription.cpp @@ -12,8 +12,6 @@ #include "oops/util/IntSetParser.h" #include "BufrDescription.h" -#include "BufrMnemonicSet.h" -#include "BufrTypes.h" namespace @@ -23,9 +21,6 @@ namespace const char* Filename = "obsdatain"; const char* IsWmoFormat = "isWmoFormat"; const char* TablePath = "tablepath"; - const char* MnemonicSets = "mnemonicSets"; - const char* Mnemonics = "mnemonics"; - const char* Channels = "channels"; const char* Exports = "exports"; } // namespace ConfKeys } // namespace @@ -54,30 +49,5 @@ namespace Ingester { setTablepath(""); } - - if (conf.getSubConfigurations(ConfKeys::MnemonicSets).size() == 0) - { - std::stringstream errStr; - errStr << "bufr::mnemonicSets must contain a list of objects!"; - throw eckit::BadParameter(errStr.str()); - } - - for (const auto& mnemonicSetConf : conf.getSubConfigurations(ConfKeys::MnemonicSets)) - { - Channels channels = {1}; - if (mnemonicSetConf.has(ConfKeys::Channels)) - { - auto intChannels = oops::parseIntSet(mnemonicSetConf.getString(ConfKeys::Channels)); - channels = Channels(intChannels.begin(), intChannels.end()); - } - - addMnemonicSet(BufrMnemonicSet( - mnemonicSetConf.getStringVector(ConfKeys::Mnemonics), channels)); - } - } - - void BufrDescription::addMnemonicSet(const BufrMnemonicSet& mnemonicSet) - { - mnemonicSets_.push_back(mnemonicSet); } } // namespace Ingester diff --git a/src/bufr/BufrParser/BufrDescription.h b/src/bufr/BufrParser/BufrDescription.h index e94ec7ae8..0069cd8a3 100644 --- a/src/bufr/BufrParser/BufrDescription.h +++ b/src/bufr/BufrParser/BufrDescription.h @@ -7,13 +7,11 @@ #pragma once -#include #include #include #include "eckit/config/LocalConfiguration.h" -#include "BufrTypes.h" #include "Exports/Export.h" @@ -27,7 +25,6 @@ namespace Ingester class BufrDescription { public: - BufrDescription() = default; explicit BufrDescription(const eckit::Configuration &conf); /// \brief Add a BufrMnemonicSet to the description. @@ -41,16 +38,12 @@ namespace Ingester inline void setExport(const Export& newExport) { export_ = newExport; } // Getters - inline std::vector getMnemonicSets() const { return mnemonicSets_; } inline std::string filepath() const { return filepath_; } inline bool isWmoFormat() const { return isWmoFormat_; } inline std::string tablepath() const { return tablepath_; } inline Export getExport() const { return export_; } private: - /// \brief Sets of mnemonic strings for the data to read. - std::vector mnemonicSets_; - /// \brief Specifies the relative path to the BUFR file to read. std::string filepath_; diff --git a/src/bufr/BufrParser/BufrMnemonicSet.cpp b/src/bufr/BufrParser/BufrMnemonicSet.cpp deleted file mode 100644 index 3ddf803f2..000000000 --- a/src/bufr/BufrParser/BufrMnemonicSet.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include -#include - -#include "eckit/exception/Exceptions.h" - -#include "BufrMnemonicSet.h" -#include "BufrTypes.h" - -namespace Ingester -{ - BufrMnemonicSet::BufrMnemonicSet(const std::vector& mnemonics, - const Channels& channels) : - mnemonics_(mnemonics), - mnemonicsStr_(makeMnemonicsStr(mnemonics)), - channels_(channels), - maxColumn_(*std::max_element(channels.begin(), channels.end())) - { - if (std::find_if(channels.begin(), channels.end(), [](const auto x){ return x < 1; }) \ - != channels.end()) - { - throw eckit::BadParameter("All channel numbers must be >= 1."); - } - } - - std::string BufrMnemonicSet::makeMnemonicsStr(std::vector mnemonics) - { - std::ostringstream mnemonicsStrStream; - for (auto mnemonicsIt = mnemonics.begin(); - mnemonicsIt < mnemonics.end(); - mnemonicsIt++) - { - mnemonicsStrStream << *mnemonicsIt; - - if (mnemonicsIt != mnemonics.end() - 1) - { - mnemonicsStrStream << " "; - } - } - - return mnemonicsStrStream.str(); - } -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrMnemonicSet.h b/src/bufr/BufrParser/BufrMnemonicSet.h deleted file mode 100644 index bfe16409e..000000000 --- a/src/bufr/BufrParser/BufrMnemonicSet.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include - -#include "BufrTypes.h" - -namespace Ingester -{ - /// \brief Defenition of BUFR mnemonics and associated channels of interest. - class BufrMnemonicSet - { - public: - explicit BufrMnemonicSet(const std::vector& mnemonics, - const Channels& channels = {1}); - - // Getters - inline std::vector getMnemonics() const { return mnemonics_; } - inline std::string getMnemonicsStr() const { return mnemonicsStr_; } - inline Channels getChannels() const { return channels_; } - inline size_t getMaxColumn() const { return maxColumn_; } - inline size_t getSize() const { return mnemonics_.size(); } - - private: - /// \brief Collection of BUFR mnemonics. - const std::vector mnemonics_; - - /// \brief String of assembled mnemonics to use when reading from the buffer interface - const std::string mnemonicsStr_; - - /// \brief Collection of channels to read for each mnemonic - const Channels channels_; - - /// \brief The value of the greatest channel. - const size_t maxColumn_; - - /// \brief Concatinates mnemonics into a space seperated string. - static std::string makeMnemonicsStr(std::vector mnemonics); - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/BufrParser.cpp b/src/bufr/BufrParser/BufrParser.cpp index 669824ae8..00160e0d7 100644 --- a/src/bufr/BufrParser/BufrParser.cpp +++ b/src/bufr/BufrParser/BufrParser.cpp @@ -7,121 +7,125 @@ #include "BufrParser.h" -#include #include #include +#include // NOLINT #include "eckit/exception/Exceptions.h" +#include "oops/util/Logger.h" -#if __has_include("bufr_interface.h") // TODO(rmclaren): Remove this in future - #include "bufr_interface.h" -#else - #include "bufr.interface.h" -#endif - -#include "BufrParser/BufrCollectors/BufrCollectors.h" -#include "BufrMnemonicSet.h" #include "DataContainer.h" +#include "DataObject.h" #include "Exports/Export.h" #include "Exports/Splits/Split.h" +#include "Query/QuerySet.h" + -namespace Ingester -{ +namespace Ingester { BufrParser::BufrParser(const BufrDescription &description) : - description_(description), - fortranFileId_(0), - table1FileId_(0), - table2FileId_(0) + description_(description), + file_(bufr::File(description_.filepath(), + description_.isWmoFormat(), + description_.tablepath())) { - reset(); } - BufrParser::BufrParser(const eckit::Configuration& conf) : - description_(BufrDescription(conf)), - fortranFileId_(0), - table1FileId_(0), - table2FileId_(0) + BufrParser::BufrParser(const eckit::Configuration &conf) : + description_(BufrDescription(conf)), + file_(bufr::File(description_.filepath(), + description_.isWmoFormat(), + description_.tablepath())) { - reset(); } BufrParser::~BufrParser() { - closeBufrFile(); + file_.close(); } - std::shared_ptr BufrParser::parse(const size_t maxMsgsToParse) + std::shared_ptr BufrParser::parse(const size_t maxMsgsToParse) { - const unsigned int SubsetStringLength = 25; + auto startTime = std::chrono::steady_clock::now(); - if (fortranFileId_ <= 10) + auto querySet = bufr::QuerySet(description_.getExport().getSubsets()); + + for (const auto &var : description_.getExport().getVariables()) { - throw eckit::BadValue("Fortran File ID is an invalid number (must be > 10)."); + for (const auto &queryPair : var->getQueryList()) + { + querySet.add(queryPair.name, queryPair.query); + } } - auto collectors = BufrCollectors(fortranFileId_); - collectors.addMnemonicSets(description_.getMnemonicSets()); - - char subset[SubsetStringLength]; - int iddate; + oops::Log::info() << "Executing Queries" << std::endl; + const auto resultSet = file_.execute(querySet, maxMsgsToParse); - unsigned int messageNum = 0; - while (ireadmg_f(fortranFileId_, subset, &iddate, SubsetStringLength) == 0) + oops::Log::info() << "Building Bufr Data" << std::endl; + auto srcData = BufrDataMap(); + for (const auto& var : description_.getExport().getVariables()) { - while (ireadsb_f(fortranFileId_) == 0) + for (const auto& queryInfo : var->getQueryList()) { - collectors.collect(); + srcData[queryInfo.name] = resultSet.get( + queryInfo.name, queryInfo.groupByField, queryInfo.type); } - - if (maxMsgsToParse > 0 && ++messageNum >= maxMsgsToParse) break; } - return exportData(collectors.finalize()); + oops::Log::info() << "Exporting Data" << std::endl; + auto exportedData = exportData(srcData); + + auto timeElapsed = std::chrono::steady_clock::now() - startTime; + auto timeElapsedDuration = std::chrono::duration_cast + (timeElapsed); + oops::Log::info() << "Finished " + << "[" << timeElapsedDuration.count() / 1000.0 << "s]" + << std::endl; + + return exportedData; } - std::shared_ptr BufrParser::exportData(const BufrDataMap& srcData) - { + std::shared_ptr BufrParser::exportData(const BufrDataMap &srcData) { auto exportDescription = description_.getExport(); auto filters = exportDescription.getFilters(); - auto splitMap = exportDescription.getSplits(); - auto varMap = exportDescription.getVariables(); + auto splits = exportDescription.getSplits(); + auto vars = exportDescription.getVariables(); // Filter BufrDataMap dataCopy = srcData; // make mutable copy - for (const auto& filter : filters) + for (const auto &filter : filters) { filter->apply(dataCopy); } // Split CategoryMap catMap; - for (const auto& splitPair : splitMap) + for (const auto &split : splits) { std::ostringstream catName; - catName << "splits/" << splitPair.first; - catMap.insert({catName.str(), splitPair.second->subCategories(dataCopy)}); + catName << "splits/" << split->getName(); + catMap.insert({catName.str(), split->subCategories(dataCopy)}); } BufrParser::CatDataMap splitDataMaps; splitDataMaps.insert({std::vector(), dataCopy}); - for (const auto& splitPair : splitMap) + for (const auto &split : splits) { - splitDataMaps = splitData(splitDataMaps, *splitPair.second); + splitDataMaps = splitData(splitDataMaps, *split); } // Export - auto exportData = std::make_shared(catMap); - for (const auto& dataPair : splitDataMaps) + auto exportData = std::make_shared(catMap); + for (const auto &dataPair : splitDataMaps) { - for (const auto& varPair : varMap) + for (const auto &var : vars) { std::ostringstream pathStr; - pathStr << "variables/" << varPair.first; + pathStr << "variables/" << var->getExportName(); exportData->add(pathStr.str(), - varPair.second->exportData(dataPair.second), + var->exportData(dataPair.second), dataPair.first); } } @@ -129,15 +133,15 @@ namespace Ingester return exportData; } - BufrParser::CatDataMap BufrParser::splitData(BufrParser::CatDataMap& splitMaps, Split& split) + BufrParser::CatDataMap BufrParser::splitData(BufrParser::CatDataMap &splitMaps, Split &split) { CatDataMap splitDataMap; - for (const auto& splitMapPair : splitMaps) + for (const auto &splitMapPair : splitMaps) { auto newData = split.split(splitMapPair.second); - for (const auto& newDataPair : newData) + for (const auto &newDataPair : newData) { auto catVect = splitMapPair.first; catVect.push_back(newDataPair.first); @@ -148,52 +152,12 @@ namespace Ingester return splitDataMap; } - void BufrParser::openBufrFile(const std::string& filepath, - bool isWmoFormat, - const std::string& tablepath) - { - fortranFileId_ = 11; // Fortran file id must be a integer > 10 - open_f(fortranFileId_, filepath.c_str()); - - if (!isWmoFormat) - { - openbf_f(fortranFileId_, "IN", fortranFileId_); - } - else - { - openbf_f(fortranFileId_, "SEC3", fortranFileId_); - - if (!tablepath.empty()) // else use the default tables - { - table1FileId_ = fortranFileId_ + 1; - table2FileId_ = fortranFileId_ + 2; - mtinfo_f(tablepath.c_str(), table1FileId_, table2FileId_); - } - } - } - - void BufrParser::closeBufrFile() - { - exitbufr_f(); - - fortranFileId_ = 0; - table1FileId_ = 0; - table2FileId_ = 0; - } - void BufrParser::reset() { - if (fortranFileId_ != 0) - { - closeBufrFile(); - } - - openBufrFile(description_.filepath(), - description_.isWmoFormat(), - description_.tablepath()); + file_.rewind(); } - void BufrParser::printMap(const BufrParser::CatDataMap& map) + void BufrParser::printMap(const BufrParser::CatDataMap &map) { for (const auto &mp : map) { @@ -206,7 +170,7 @@ namespace Ingester std::cout << " subkeys: "; for (const auto &m2p : mp.second) { - std::cout << m2p.first << " " << m2p.second.rows() << " "; + std::cout << m2p.first << " " << m2p.second->getDims()[0] << " "; } std::cout << std::endl; diff --git a/src/bufr/BufrParser/BufrParser.h b/src/bufr/BufrParser/BufrParser.h index bead6eaec..9e3a9dcd0 100644 --- a/src/bufr/BufrParser/BufrParser.h +++ b/src/bufr/BufrParser/BufrParser.h @@ -16,14 +16,12 @@ #include "eckit/config/LocalConfiguration.h" +#include "Query/File.h" #include "Parser.h" -#include "BufrTypes.h" #include "BufrDescription.h" -namespace Ingester -{ - class BufrMnemonicSet; +namespace Ingester { class DataContainer; /// \brief Uses a BufrDescription and helper classes to parse the contents of a BUFR file. @@ -48,14 +46,8 @@ namespace Ingester /// \brief The description the defines what to parse from the BUFR file BufrDescription description_; - /// \brief The Fortran file ID to an open BUFR file (0 when no file open) - unsigned int fortranFileId_; - - /// \brief The Fortran file ID to an open BUFR file (0 when no file open) - unsigned int table1FileId_; - - /// \brief The Fortran file ID to an open BUFR file (0 when no file open) - unsigned int table2FileId_; + /// \brief The Bufr file object we are working with + bufr::File file_; /// \brief Exports collected data into a DataContainer /// \param srcData Data to export @@ -72,11 +64,6 @@ namespace Ingester /// \param filepath Path to bufr file. /// \param isWmoFormat _optional_ Bufr file is in the standard format. /// \param tablepath _optional_ Path to WMO master tables (needed for standard bufr files). - void - openBufrFile(const std::string& filepath, bool isWmoFormat, const std::string& tablepath); - - /// \brief Closes the open BUFR file. - void closeBufrFile(); /// \brief Convenience method to print the Categorical data map to stdout. void printMap(const CatDataMap& map); diff --git a/src/bufr/BufrParser/BufrTypes.h b/src/bufr/BufrParser/BufrTypes.h deleted file mode 100644 index d00dc22fa..000000000 --- a/src/bufr/BufrParser/BufrTypes.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include -#include - -#include "IngesterTypes.h" - -namespace Ingester -{ - class Variable; - - typedef std::set Channels; - typedef IngesterArrayMap BufrDataMap; -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Export.cpp b/src/bufr/BufrParser/Exports/Export.cpp index 564db6bdf..c08b0bc5c 100644 --- a/src/bufr/BufrParser/Exports/Export.cpp +++ b/src/bufr/BufrParser/Exports/Export.cpp @@ -14,8 +14,9 @@ #include "Filters/BoundingFilter.h" #include "Splits/CategorySplit.h" -#include "Variables/MnemonicVariable.h" +#include "Variables/QueryVariable.h" #include "Variables/DatetimeVariable.h" +#include "Variables/TimeoffsetVariable.h" #include "Variables/Transforms/Transform.h" #include "Variables/Transforms/TransformBuilder.h" @@ -27,23 +28,28 @@ namespace const char* Filters = "filters"; const char* Splits = "splits"; const char* Variables = "variables"; + const char* GroupByVariable = "group_by_variable"; + const char* Subsets = "subsets"; namespace Variable { const char* Datetime = "datetime"; - const char* Mnemonic = "mnemonic"; + const char* Timeoffset = "timeoffset"; + const char* Query = "query"; + const char* GroupByField = "group_by"; // Deprecated + const char* Type = "type"; } // namespace Variable namespace Split { const char* Category = "category"; - const char* Mnemonic = "mnemonic"; + const char* Variable = "variable"; const char* NameMap = "map"; } // namespace Split namespace Filter { - const char* Mnemonic = "mnemonic"; + const char* Variable = "variable"; const char* Bounding = "bounding"; const char* UpperBound = "upperBound"; const char* LowerBound = "lowerBound"; @@ -66,17 +72,49 @@ namespace Ingester addSplits(conf.getSubConfiguration(ConfKeys::Splits)); } + std::string groupByVariable; + if (conf.has(ConfKeys::GroupByVariable)) // Optional + { + groupByVariable = conf.getString(ConfKeys::GroupByVariable); + } + + if (conf.has(ConfKeys::Subsets)) + { + subsets_ = conf.getStringVector(ConfKeys::Subsets); + } + if (conf.has(ConfKeys::Variables)) { - addVariables(conf.getSubConfiguration(ConfKeys::Variables)); + addVariables(conf.getSubConfiguration(ConfKeys::Variables), + groupByVariable); } else { throw eckit::BadParameter("Missing export::variables section in configuration."); } + + // Make sure the groupByVariable field is valid. + if (conf.has(ConfKeys::GroupByVariable)) + { + auto groupByFound = false; + for (const auto &var : variables_) + { + if (var->getExportName() == groupByVariable) + { + groupByFound = true; + break; + } + } + + if (!groupByFound) + { + throw eckit::BadParameter( + "Group by variable not found in export::variables section."); + } + } } - void Export::addVariables(const eckit::Configuration &conf) + void Export::addVariables(const eckit::Configuration &conf, const std::string& groupByField) { if (conf.keys().size() == 0) { @@ -93,13 +131,38 @@ namespace Ingester if (subConf.has(ConfKeys::Variable::Datetime)) { auto dtconf = subConf.getSubConfiguration(ConfKeys::Variable::Datetime); - variable = std::make_shared(dtconf); + variable = std::make_shared(key, groupByField, dtconf); } - else if (subConf.has(ConfKeys::Variable::Mnemonic)) + else if (subConf.has(ConfKeys::Variable::Timeoffset)) + { + auto dtconf = subConf.getSubConfiguration(ConfKeys::Variable::Timeoffset); + variable = std::make_shared(key, groupByField, dtconf); + } + else if (subConf.has(ConfKeys::Variable::Query)) { Transforms transforms = TransformBuilder::makeTransforms(subConf); - variable = std::make_shared( - subConf.getString(ConfKeys::Variable::Mnemonic), transforms); + const auto& query = subConf.getString(ConfKeys::Variable::Query); + + if (subConf.has(ConfKeys::Variable::GroupByField)) + { + std::ostringstream errMsg; + errMsg << "Obsolete format::exports::variable of group_by field for key"; + errMsg << key << std::endl; + errMsg << "Use \"query:\" instead."; + throw eckit::BadParameter(errMsg.str()); + } + + std::string type = ""; + if (subConf.has(ConfKeys::Variable::Type)) + { + type = subConf.getString(ConfKeys::Variable::Type); + } + + variable = std::make_shared(key, + query, + groupByField, + type, + transforms); } else { @@ -108,7 +171,7 @@ namespace Ingester throw eckit::BadParameter(errMsg.str()); } - variables_.insert({key, variable}); + variables_.push_back(variable); } } @@ -142,9 +205,9 @@ namespace Ingester } } - split = std::make_shared( - catConf.getString(ConfKeys::Split::Mnemonic), - nameMap); + split = std::make_shared(key, + catConf.getString(ConfKeys::Split::Variable), + nameMap); } else { @@ -153,7 +216,7 @@ namespace Ingester throw eckit::BadParameter(errMsg.str()); } - splits_.insert({key, split}); + splits_.push_back(split); } } @@ -190,7 +253,7 @@ namespace Ingester } filter = std::make_shared( - filterConf.getString(ConfKeys::Filter::Mnemonic), + filterConf.getString(ConfKeys::Filter::Variable), lowerBound, upperBound); } diff --git a/src/bufr/BufrParser/Exports/Export.h b/src/bufr/BufrParser/Exports/Export.h index 4020beee9..baed395d5 100644 --- a/src/bufr/BufrParser/Exports/Export.h +++ b/src/bufr/BufrParser/Exports/Export.h @@ -8,7 +8,6 @@ #pragma once #include -#include #include #include "eckit/config/LocalConfiguration.h" @@ -24,8 +23,8 @@ namespace Ingester class Export { public: - typedef std::map> Splits; - typedef std::map> Variables; + typedef std::vector> Splits; + typedef std::vector> Variables; typedef std::vector> Filters; /// \brief Constructor @@ -36,14 +35,17 @@ namespace Ingester inline Splits getSplits() const { return splits_; } inline Variables getVariables() const { return variables_; } inline Filters getFilters() const { return filters_; } + inline std::vector getSubsets() const { return subsets_; } private: Splits splits_; Variables variables_; Filters filters_; + std::vector subsets_; /// \brief Create Variables exports from config. - void addVariables(const eckit::Configuration &conf); + void addVariables(const eckit::Configuration &conf, + const std::string& groupByVariable = ""); /// \brief Create Splits exports from config. void addSplits(const eckit::Configuration &conf); diff --git a/src/bufr/BufrParser/Exports/Filters/BoundingFilter.cpp b/src/bufr/BufrParser/Exports/Filters/BoundingFilter.cpp index 79674db8a..5f1f646eb 100644 --- a/src/bufr/BufrParser/Exports/Filters/BoundingFilter.cpp +++ b/src/bufr/BufrParser/Exports/Filters/BoundingFilter.cpp @@ -11,14 +11,19 @@ #include "eckit/exception/Exceptions.h" -#include "../RowSlice.h" +#include "DataObject.h" + +#include "Eigen/Dense" + namespace Ingester { - BoundingFilter::BoundingFilter(const std::string& mnemonic, + typedef Eigen::Array EigArray; + + BoundingFilter::BoundingFilter(const std::string& variable, std::shared_ptr lowerBound, std::shared_ptr upperBound) : - mnemonic_(mnemonic), + variable_(variable), lowerBound_(lowerBound), upperBound_(upperBound) { @@ -40,40 +45,60 @@ namespace Ingester void BoundingFilter::apply(BufrDataMap& dataMap) { std::vector validRows; - if (dataMap.find(mnemonic_) == dataMap.end()) + if (dataMap.find(variable_) == dataMap.end()) { std::ostringstream errStr; - errStr << "Unknown mnemonic " << mnemonic_ << " found in bounding filter."; + errStr << "Unknown variable " << variable_ << " found in bounding filter."; throw eckit::BadParameter(errStr.str()); } - const auto& array = dataMap.at(mnemonic_); - for (size_t rowIdx = 0; rowIdx < static_cast(array.rows()); rowIdx++) + if (const auto& var = std::dynamic_pointer_cast>(dataMap.at(variable_))) { - if (lowerBound_ && upperBound_) + auto dims = var->getDims(); + size_t extraDims = 1; + + for (size_t dimIdx = 1; dimIdx < dims.size(); ++dimIdx) + { + extraDims *= dims[dimIdx]; + } + + // Make local copy otherwise you get weird memory corruption issue. + auto rawData = var->getRawData(); + auto array = Eigen::Map (rawData.data(), dims[0], extraDims); + + for (auto rowIdx = 0; rowIdx < dims[0]; rowIdx++) { - if ((array.row(rowIdx) >= *lowerBound_).all() && - (array.row(rowIdx) <= *upperBound_).all()) + if (lowerBound_ && upperBound_) { - validRows.push_back(rowIdx); + if ((array.row(rowIdx) >= *lowerBound_).all() && + (array.row(rowIdx) <= *upperBound_).all()) + { + validRows.push_back(rowIdx); + } + } + else + { + if ((lowerBound_ && (array.row(rowIdx) >= *lowerBound_).all()) || + (upperBound_ && (array.row(rowIdx) <= *upperBound_).all())) + { + validRows.push_back(rowIdx); + } } } - else + + if (validRows.size() != static_cast(dims[0])) { - if ((lowerBound_ && (array.row(rowIdx) >= *lowerBound_).all()) || - (upperBound_ && (array.row(rowIdx) <= *upperBound_).all())) + for (const auto& dataPair : dataMap) { - validRows.push_back(rowIdx); + dataMap[dataPair.first] = dataPair.second->slice(validRows); } } } - - if (validRows.size() != static_cast(array.rows())) + else { - for (const auto& dataPair : dataMap) - { - dataMap[dataPair.first] = rowSlice(dataPair.second, validRows); - } + std::stringstream errStr; + errStr << "BoundingFilter variable must be a array of numbers (found list of strings)."; + throw eckit::BadParameter(errStr.str()); } } } // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Filters/BoundingFilter.h b/src/bufr/BufrParser/Exports/Filters/BoundingFilter.h index 3e1cb1d6a..c1213ce82 100644 --- a/src/bufr/BufrParser/Exports/Filters/BoundingFilter.h +++ b/src/bufr/BufrParser/Exports/Filters/BoundingFilter.h @@ -13,8 +13,6 @@ #include #include -#include "BufrParser/BufrTypes.h" - namespace Ingester { @@ -23,19 +21,21 @@ namespace Ingester { public: /// \brief Constructor - /// \param mnemonic BUFR Mnemonic to filter on + /// \param variable Variable to filter /// \param lowerBound Lowest allowable value /// \param upperBound Highest allowable value - BoundingFilter(const std::string& mnemonic, + BoundingFilter(const std::string& variable, std::shared_ptr lowerBound, std::shared_ptr upperBound); - /// \brief Apply the filter to the data - /// \param dataMap Map to modify by filtering out relevant data. + virtual ~BoundingFilter() = default; + + /// \brief Apply the filter to the dataevant data. void apply(BufrDataMap& dataMap) final; + /// \param dataMap Map to modify by filtering out rel private: - const std::string mnemonic_; + const std::string variable_; const std::shared_ptr lowerBound_; const std::shared_ptr upperBound_; }; diff --git a/src/bufr/BufrParser/Exports/Filters/Filter.h b/src/bufr/BufrParser/Exports/Filters/Filter.h index 73b3825a3..0ccfcf6ff 100644 --- a/src/bufr/BufrParser/Exports/Filters/Filter.h +++ b/src/bufr/BufrParser/Exports/Filters/Filter.h @@ -7,8 +7,6 @@ #pragma once #include "IngesterTypes.h" -#include "BufrParser/BufrTypes.h" - namespace Ingester { diff --git a/src/bufr/BufrParser/Exports/RowSlice.h b/src/bufr/BufrParser/Exports/RowSlice.h deleted file mode 100644 index 4a228bf77..000000000 --- a/src/bufr/BufrParser/Exports/RowSlice.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include -#include - - -/// Collection of template methods that are used to slice array and vector data. - -/// \brief Slice eigen array using indicies of eigen arrays. -template -EigenType rowSlice(const EigenType& arr, const EigenIdxType& idxVec) -{ - EigenType result(idxVec.rows(), arr.cols()); - - for (size_t rowIdx = 0; rowIdx < idxVec.rows(); rowIdx++) - { - result.row(rowIdx) = arr.row(idxVec.at(rowIdx)(0)); - } - - return result; -} - -/// \brief Slice eigen array using std::vector of indicies. -template -EigenType rowSlice(const EigenType& arr, const std::vector& idxVec) -{ - EigenType result(idxVec.size(), arr.cols()); - - for (size_t rowIdx = 0; rowIdx < idxVec.size(); rowIdx++) - { - result.row(rowIdx) = arr.row(static_cast(idxVec[rowIdx])); - } - - return result; -} - -/// \brief Slice string vector using indicies of eigen arrays. -template -std::vector rowSlice(const std::vector& arr, - const EigenIdxType& idxVec) -{ - std::vector result; - result.resize(idxVec.rows()); - - for (Eigen::Index rowIdx = 0; rowIdx < idxVec.rows(); rowIdx++) - { - result[rowIdx] = arr[idxVec.at(rowIdx)(0)]; - } - - return result; -} - -/// \brief Slice string vector using std::vector of indicies. -template -std::vector rowSlice(const std::vector& arr, - const std::vector& idxVec) -{ - std::vector result; - result.resize(idxVec.size()); - - for (Eigen::Index rowIdx = 0; rowIdx < idxVec.size(); rowIdx++) - { - result[rowIdx] = arr[static_cast(idxVec[rowIdx])]; - } - - return result; -} - diff --git a/src/bufr/BufrParser/Exports/Splits/CategorySplit.cpp b/src/bufr/BufrParser/Exports/Splits/CategorySplit.cpp index 861ef7fe0..8d30aa319 100644 --- a/src/bufr/BufrParser/Exports/Splits/CategorySplit.cpp +++ b/src/bufr/BufrParser/Exports/Splits/CategorySplit.cpp @@ -11,14 +11,15 @@ #include "eckit/exception/Exceptions.h" -#include "../RowSlice.h" - namespace Ingester { - CategorySplit::CategorySplit(const std::string& mnemonic, const NameMap& nameMap) : - nameMap_(nameMap), - mnemonic_(mnemonic) + CategorySplit::CategorySplit(const std::string& name, + const std::string& variable, + const NameMap& nameMap) : + Split(name), + variable_(variable), + nameMap_(nameMap) { } @@ -35,23 +36,24 @@ namespace Ingester return categories; } - std::map CategorySplit::split(const BufrDataMap &dataMap) + std::unordered_map CategorySplit::split(const BufrDataMap &dataMap) { updateNameMap(dataMap); - std::map dataMaps; + std::unordered_map dataMaps; - const IngesterArray& mnemonicArr = dataMap.at(mnemonic_); + const auto& dataObject = dataMap.at(variable_); for (const auto& mapPair : nameMap_) { // Find matching rows std::vector indexVec; - for (int rowIdx = 0; - rowIdx < static_cast(dataMap.at(mnemonic_).rows()); - rowIdx++) + for (auto rowIdx = 0; rowIdx < dataObject->getDims()[0]; rowIdx++) { - if (mnemonicArr.row(rowIdx)[0] == mapPair.first) + auto location = Location(dataObject->getDims().size(), 0); + location[0] = rowIdx; + + if (dataObject->getAsInt(location) == mapPair.first) { indexVec.push_back(rowIdx); } @@ -61,7 +63,7 @@ namespace Ingester BufrDataMap newDataMap; for (const auto& dataPair : dataMap) { - const auto newArr = rowSlice(dataPair.second, indexVec); + const auto newArr = dataPair.second->slice(indexVec); newDataMap.insert({dataPair.first, newArr}); } @@ -75,19 +77,21 @@ namespace Ingester { if (nameMap_.empty()) { - auto& array = dataMap.at(mnemonic_); - for (auto rowIdx = 0; rowIdx < array.rows(); rowIdx++) + const auto& dataObject = dataMap.at(variable_); + for (auto rowIdx = 0; rowIdx < dataObject->getDims()[0]; rowIdx++) { - auto itemVal = array.row(rowIdx)[0]; - if (trunc(itemVal) == itemVal) + auto location = Location(dataObject->getDims().size(), 0); + location[0] = rowIdx; + + if (auto dat = std::dynamic_pointer_cast> (dataObject)) { - nameMap_.insert({static_cast (itemVal), - std::to_string(static_cast (itemVal))}); + auto itemVal = dat->get(location); + nameMap_.insert({itemVal, std::to_string(itemVal)}); } else { std::stringstream errStr; - errStr << "Can't turn " << mnemonic_ << " into a category as it contains "; + errStr << "Can't turn " << variable_ << " into a category as it contains "; errStr << "non-integer values."; throw eckit::BadParameter(errStr.str()); } @@ -97,7 +101,7 @@ namespace Ingester if (nameMap_.empty()) { std::stringstream errStr; - errStr << "No categories could be identified for " << mnemonic_ << "."; + errStr << "No categories could be identified for " << variable_ << "."; throw eckit::BadParameter(errStr.str()); } } diff --git a/src/bufr/BufrParser/Exports/Splits/CategorySplit.h b/src/bufr/BufrParser/Exports/Splits/CategorySplit.h index 96d7222ce..d80144bdf 100644 --- a/src/bufr/BufrParser/Exports/Splits/CategorySplit.h +++ b/src/bufr/BufrParser/Exports/Splits/CategorySplit.h @@ -11,24 +11,24 @@ #include #include -#include +#include namespace Ingester { /// \brief Data splitter class that splits data according to a predefined categories. /// \details This class sub-divides data into sub-categories depending on the value of a - /// mnemonic. It is assumed that the mnemonic values are integers which represent - /// separate categories of data. An example is Satellite ID (mnemonic: SAID) where each + /// variable. It is assumed that the variable values are integers which represent + /// separate categories of data. An example is Satellite ID (variable: SAID) where each /// possible satellite has its own unique integer ID. /// The subcategories this Split divides into can either be manually specified by a /// NameMap (map) or be automatically determined (if the given NameMap /// is found to be empty). An example NameMap might look like this: /// { 257 : GEOS-13, /// 259 : GEOS-15 } - /// This NameMap tells the splitter to divide by the values of the given mnemonic + /// This NameMap tells the splitter to divide by the values of the given variable /// (SAID for this example) into two named groups (GEOS-13 and GEOS-15). Data - /// associated with mnemonic values not specified in the map are discarded. If the + /// associated with variable values not specified in the map are discarded. If the /// NameMap were empty (unspecified) then this splitter will use the data to to /// determine all all the possible values to split on automatically. Each split would /// then be named according to its integer value (ex: 257, 259, 270, 271, ....). @@ -40,11 +40,11 @@ namespace Ingester typedef std::map NameMap; /// \brief constructor - /// \param mnemonic BUFR mnemonic to base the split on. + /// \param variable Variable to base the split on. /// \param map Name of the created categories from the integer BUFR values. May be an /// empty map in which case subcategories are automatically determined from the /// data. - CategorySplit(const std::string& mnemonic, const NameMap& map); + CategorySplit(const std::string& name, const std::string& variable, const NameMap& map); /// \brief Get list of sub categories this split will create /// \result Set of unique strings. @@ -53,14 +53,13 @@ namespace Ingester /// \brief Split the data according to internal rules /// \param dataMap Data to be split /// \result map of split data where the category is the key - std::map split(const BufrDataMap& dataMap) final; - - // Getters - inline std::string getMnemonic() { return mnemonic_; } + std::unordered_map split(const BufrDataMap& dataMap) final; private: + const std::string variable_; + NameMap nameMap_; - const std::string mnemonic_; + /// \brief Adds values to nameMap_ using the data if nameMap_ is empty. /// \param dataMap Data to be split diff --git a/src/bufr/BufrParser/Exports/Splits/Split.cpp b/src/bufr/BufrParser/Exports/Splits/Split.cpp new file mode 100644 index 000000000..da66dafba --- /dev/null +++ b/src/bufr/BufrParser/Exports/Splits/Split.cpp @@ -0,0 +1,17 @@ +/* + * (C) Copyright 2021 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + + +#include "Split.h" + +namespace Ingester +{ + Split::Split(const std::string& name) : + name_(name) + { + } +} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Splits/Split.h b/src/bufr/BufrParser/Exports/Splits/Split.h index 67c5d9d55..dbfbe98b1 100644 --- a/src/bufr/BufrParser/Exports/Splits/Split.h +++ b/src/bufr/BufrParser/Exports/Splits/Split.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -8,9 +8,10 @@ #pragma once #include +#include #include -#include "BufrParser/BufrTypes.h" +#include "IngesterTypes.h" namespace Ingester { @@ -18,7 +19,8 @@ namespace Ingester class Split { public: - Split() = default; + explicit Split(const std::string& name); + virtual ~Split() = default; /// \brief Get set of sub categories this split will create /// \param dataMap The data we will split on. @@ -28,8 +30,13 @@ namespace Ingester /// \brief Split the data according to internal rules /// \param dataMap Data to be split /// \result map of split data where the category is the key - virtual std::map split(const BufrDataMap& dataMap) = 0; - }; -} // namespace Ingester + virtual std::unordered_map split(const BufrDataMap& dataMap) = 0; + /// \brief Get the split name + inline std::string getName() const { return name_; } + private: + /// \brief The name of the split as defined by the key in the YAML file. + const std::string name_; + }; +} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Transforms/OffsetTransform.cpp b/src/bufr/BufrParser/Exports/Transforms/OffsetTransform.cpp deleted file mode 100644 index 98ca3acbb..000000000 --- a/src/bufr/BufrParser/Exports/Transforms/OffsetTransform.cpp +++ /dev/null @@ -1,23 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "OffsetTransform.h" - - -namespace Ingester -{ - OffsetTransform::OffsetTransform(const double offset) : - offset_(offset) - { - } - - void OffsetTransform::apply(IngesterArray& array) - { - array = array + offset_; - } - -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Transforms/OffsetTransform.h b/src/bufr/BufrParser/Exports/Transforms/OffsetTransform.h deleted file mode 100644 index f1556a707..000000000 --- a/src/bufr/BufrParser/Exports/Transforms/OffsetTransform.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include "Transform.h" - - -namespace Ingester -{ - /// \brief Add a floating point offset to to the data. - class OffsetTransform : public Transform - { - public: - explicit OffsetTransform(const double offset); - ~OffsetTransform() = default; - - /// \brief Apply transform to the given data. - void apply(IngesterArray& array) override; - - private: - const double offset_; - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Transforms/ScalingTransform.cpp b/src/bufr/BufrParser/Exports/Transforms/ScalingTransform.cpp deleted file mode 100644 index a6d0aa1c5..000000000 --- a/src/bufr/BufrParser/Exports/Transforms/ScalingTransform.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "ScalingTransform.h" - - -namespace Ingester -{ - ScalingTransform::ScalingTransform(const double scaling) : - scaling_(scaling) - { - } - - void ScalingTransform::apply(IngesterArray& array) - { - array = array * scaling_; - } -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Transforms/ScalingTransform.h b/src/bufr/BufrParser/Exports/Transforms/ScalingTransform.h deleted file mode 100644 index fb7ebcde0..000000000 --- a/src/bufr/BufrParser/Exports/Transforms/ScalingTransform.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include "Transform.h" - - -namespace Ingester -{ - /// \brief Multiply a floating point scaling factor to to the data. - class ScalingTransform : public Transform - { - public: - explicit ScalingTransform(const double scaling_); - ~ScalingTransform() = default; - - /// \brief Apply transform to the given data. - void apply(IngesterArray& array) override; - - private: - const double scaling_; - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Transforms/Transform.h b/src/bufr/BufrParser/Exports/Transforms/Transform.h deleted file mode 100644 index 9d1d9e65c..000000000 --- a/src/bufr/BufrParser/Exports/Transforms/Transform.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include - -#include "IngesterTypes.h" - -namespace Ingester -{ - /// \brief Base class for Transforms which are used to transform data. Transforms are useful - /// for getting data into the right units (for example you can convert Kelvin to Celsius) - class Transform - { - public: - ~Transform() = default; - - /// \brief Apply transform to the given data. - virtual void apply(IngesterArray& array) = 0; - }; - - typedef std::vector > Transforms; -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Transforms/TransformBuilder.cpp b/src/bufr/BufrParser/Exports/Transforms/TransformBuilder.cpp deleted file mode 100644 index f167aa6cc..000000000 --- a/src/bufr/BufrParser/Exports/Transforms/TransformBuilder.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "TransformBuilder.h" - -#include "eckit/exception/Exceptions.h" - -#include "ScalingTransform.h" -#include "OffsetTransform.h" - - -static const char* TRANSFORMS_SECTION = "transforms"; -static const char* OFFSET_KEY = "offset"; -static const char* SCALE_KEY = "scale"; - -namespace Ingester -{ - std::shared_ptr TransformBuilder::makeTransform(const eckit::Configuration& conf) - { - std::shared_ptr transform; - if (conf.has(OFFSET_KEY)) - { - transform = std::make_shared(conf.getFloat(OFFSET_KEY)); - } - else if (conf.has(SCALE_KEY)) - { - transform = std::make_shared(conf.getFloat(SCALE_KEY)); - } - else - { - throw eckit::BadParameter("Tried to create unknown export transform type. " - "Check your configuration."); - } - - return transform; - } - - Transforms TransformBuilder::makeTransforms(const eckit::Configuration& conf) - { - Transforms transforms; - if (conf.has(TRANSFORMS_SECTION)) - { - for (const auto& transformConf : conf.getSubConfigurations(TRANSFORMS_SECTION)) - { - transforms.push_back(makeTransform(transformConf)); - } - } - - return transforms; - } -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Transforms/TransformBuilder.h b/src/bufr/BufrParser/Exports/Transforms/TransformBuilder.h deleted file mode 100644 index 059f169e6..000000000 --- a/src/bufr/BufrParser/Exports/Transforms/TransformBuilder.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include - -#include "eckit/config/LocalConfiguration.h" - -#include "Transform.h" - - -namespace Ingester -{ - /// \brief Convenience class used to create transforms from configuration data. - class TransformBuilder - { - public: - /// \brief Create transform given the configuration - static std::shared_ptr makeTransform(const eckit::Configuration& conf); - - /// \brief Uses makeTransform to loop through the list of transforms in the configuration - static Transforms makeTransforms(const eckit::Configuration& conf); - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/DatetimeVariable.cpp b/src/bufr/BufrParser/Exports/Variables/DatetimeVariable.cpp index df1c3736d..c27f49661 100644 --- a/src/bufr/BufrParser/Exports/Variables/DatetimeVariable.cpp +++ b/src/bufr/BufrParser/Exports/Variables/DatetimeVariable.cpp @@ -8,12 +8,16 @@ #include #include #include +#include + #include #include #include #include "eckit/exception/Exceptions.h" +#include "oops/util/Logger.h" +#include "DataObject.h" #include "DatetimeVariable.h" @@ -28,25 +32,28 @@ namespace const char* Minute = "minute"; const char* Second = "second"; const char* HoursFromUtc = "hoursFromUtc"; - const char* Utc = "isUTC"; // deprecated + const char* GroupByField = "group_by"; } // namespace ConfKeys } // namespace namespace Ingester { - DatetimeVariable::DatetimeVariable(const eckit::Configuration& conf) : - yearKey_(conf.getString(ConfKeys::Year)), - monthKey_(conf.getString(ConfKeys::Month)), - dayKey_(conf.getString(ConfKeys::Day)), - hourKey_(conf.getString(ConfKeys::Hour)), - minuteKey_(conf.getString(ConfKeys::Minute)), - secondKey_(""), + DatetimeVariable::DatetimeVariable(const std::string& exportName, + const std::string& groupByField, + const eckit::Configuration &conf) : + Variable(exportName), + yearQuery_(conf.getString(ConfKeys::Year)), + monthQuery_(conf.getString(ConfKeys::Month)), + dayQuery_(conf.getString(ConfKeys::Day)), + hourQuery_(conf.getString(ConfKeys::Hour)), + minuteQuery_(conf.getString(ConfKeys::Minute)), + groupByField_(groupByField), hoursFromUtc_(0) { if (conf.has(ConfKeys::Second)) { - secondKey_ = conf.getString(ConfKeys::Second); + secondQuery_ = conf.getString(ConfKeys::Second); } if (conf.has(ConfKeys::HoursFromUtc)) @@ -54,25 +61,20 @@ namespace Ingester hoursFromUtc_ = conf.getInt(ConfKeys::HoursFromUtc); } - if (conf.has(ConfKeys::Utc)) + if (conf.has(ConfKeys::GroupByField)) { - std::cout << "WARNING: usage of " \ - << ConfKeys::Utc \ - << " in datetime is depricated!" \ - << std::endl; - std::cout << "Use the optional parameter " << ConfKeys::HoursFromUtc << " instead."; + groupByField_ = conf.getString(ConfKeys::GroupByField); } + + initQueryMap(); } - std::shared_ptr DatetimeVariable::exportData(const BufrDataMap& map) + std::shared_ptr DatetimeVariable::exportData(const BufrDataMap& map) { checkKeys(map); - static const float missing = 1.e+11; - static const int64_t missing_int = INT_MIN; - - std::vector timeOffsets; - timeOffsets.reserve(map.at(yearKey_).size()); + static const int missingInt = DataObject::missingValue(); + setenv("TZ", "UTC", 1); // Force UTC time zone std::tm tm{}; // zero initialise tm.tm_year = 1970-1900; // 1970 tm.tm_mon = 0; // Jan=0, Feb=1, ... @@ -82,68 +84,113 @@ namespace Ingester tm.tm_sec = 0; tm.tm_isdst = 0; // Not daylight saving std::time_t epochDt = std::mktime(&tm); - std::time_t this_time = std::mktime(&tm); - int64_t diff_time; - for (unsigned int idx = 0; idx < map.at(yearKey_).size(); idx++) + std::vector timeOffsets; + timeOffsets.reserve(map.at(getExportKey(ConfKeys::Year))->size()); + + // Validation + if (map.at(getExportKey(ConfKeys::Year))->getDims().size() != 1 || + map.at(getExportKey(ConfKeys::Month))->getDims().size() != 1 || + map.at(getExportKey(ConfKeys::Day))->getDims().size() != 1 || + (!minuteQuery_.empty() && + map.at(getExportKey(ConfKeys::Minute))->getDims().size() != 1) || + (!secondQuery_.empty() && + map.at(getExportKey(ConfKeys::Second))->getDims().size() != 1)) { - diff_time = missing_int; - if (map.at(yearKey_)(idx) != missing - && map.at(monthKey_)(idx) != missing - && map.at(dayKey_)(idx) !=missing - && map.at(hourKey_)(idx) !=missing - && map.at(minuteKey_)(idx) !=missing) + std::ostringstream errStr; + errStr << "Datetime variables must be 1 dimensional."; + throw eckit::BadParameter(errStr.str()); + } + + for (unsigned int idx = 0; idx < map.at(getExportKey(ConfKeys::Year))->size(); idx++) + { + int year = map.at(getExportKey(ConfKeys::Year))->getAsInt(idx); + int month = map.at(getExportKey(ConfKeys::Month))->getAsInt(idx); + int day = map.at(getExportKey(ConfKeys::Day))->getAsInt(idx); + int hour = map.at(getExportKey(ConfKeys::Hour))->getAsInt(idx); + int minutes = 0; + int seconds = 0; + + auto diff_time = DataObject::missingValue(); + if (year != missingInt && + month != missingInt && + day != missingInt && + hour != missingInt) { - tm.tm_year = map.at(yearKey_)(idx) - 1900; - tm.tm_mon = map.at(monthKey_)(idx) - 1; - tm.tm_mday = map.at(dayKey_)(idx); - tm.tm_hour = map.at(hourKey_)(idx); - tm.tm_min = map.at(minuteKey_)(idx); + tm.tm_year = year - 1900; + tm.tm_mon = month - 1; + tm.tm_mday = day; + tm.tm_hour = hour; + tm.tm_min = 0; tm.tm_sec = 0; tm.tm_isdst = 0; - if (!secondKey_.empty()) + if (!minuteQuery_.empty()) { - if (map.at(secondKey_)(idx) >= 0 && map.at(secondKey_)(idx) < 60) + minutes = map.at(getExportKey(ConfKeys::Minute))->getAsInt(idx); + + if (minutes >= 0 && minutes < 60) + { + tm.tm_min = minutes; + } + } + + if (!secondQuery_.empty()) + { + seconds = map.at(getExportKey(ConfKeys::Second))->getAsInt(idx); + + if (seconds >= 0 && seconds < 60) { - tm.tm_sec = map.at(secondKey_)(idx); + tm.tm_sec = seconds; } } - this_time = std::mktime(&tm); - if (this_time < 0) + // Be careful with mktime as it can be very slow. + auto thisTime = std::mktime(&tm); + if (thisTime < 0) { - std::cout << "Caution, date suspicious date (year, month, day): " - << map.at(yearKey_)(idx) << ", " - << map.at(monthKey_)(idx) << ", " - << map.at(dayKey_)(idx) << std::endl; + oops::Log::warning() << "Caution, date suspicious date (year, month, day): " + << year << ", " + << month << ", " + << day << std::endl; } - diff_time = static_cast(difftime(this_time, epochDt) - + hoursFromUtc_*3600); + + diff_time = static_cast(difftime(thisTime, epochDt) + + hoursFromUtc_ * 3600); } + timeOffsets.push_back(diff_time); } - return std::make_shared(timeOffsets); + + Dimensions dims = {static_cast(timeOffsets.size())}; + + return std::make_shared>( + timeOffsets, + getExportName(), + groupByField_, + dims, + map.at(getExportKey(ConfKeys::Year))->getPath(), + map.at(getExportKey(ConfKeys::Year))->getDimPaths()); } void DatetimeVariable::checkKeys(const BufrDataMap& map) { - std::vector requiredKeys = {yearKey_, - monthKey_, - dayKey_, - hourKey_, - minuteKey_}; + std::vector requiredKeys = {getExportKey(ConfKeys::Year), + getExportKey(ConfKeys::Month), + getExportKey(ConfKeys::Day), + getExportKey(ConfKeys::Hour), + getExportKey(ConfKeys::Minute)}; - if (!secondKey_.empty()) + if (!secondQuery_.empty()) { - requiredKeys.push_back(secondKey_); + requiredKeys.push_back(getExportKey(ConfKeys::Second)); } std::stringstream errStr; - errStr << "Mnemonic "; + errStr << "Query "; bool isKeyMissing = false; - for (auto key : requiredKeys) + for (const auto& key : requiredKeys) { if (map.find(key) == map.end()) { @@ -160,4 +207,65 @@ namespace Ingester throw eckit::BadParameter(errStr.str()); } } + + QueryList DatetimeVariable::makeQueryList() const + { + auto queries = QueryList(); + + { // Year + QueryInfo info; + info.name = getExportKey(ConfKeys::Year); + info.query = yearQuery_; + info.groupByField = groupByField_; + queries.push_back(info); + } + + { // Month + QueryInfo info; + info.name = getExportKey(ConfKeys::Month); + info.query = monthQuery_; + info.groupByField = groupByField_; + queries.push_back(info); + } + + { // Day + QueryInfo info; + info.name = getExportKey(ConfKeys::Day); + info.query = dayQuery_; + info.groupByField = groupByField_; + queries.push_back(info); + } + + { // Hour + QueryInfo info; + info.name = getExportKey(ConfKeys::Hour); + info.query = hourQuery_; + info.groupByField = groupByField_; + queries.push_back(info); + } + + { // Minute + QueryInfo info; + info.name = getExportKey(ConfKeys::Minute); + info.query = minuteQuery_; + info.groupByField = groupByField_; + queries.push_back(info); + } + + if (!secondQuery_.empty()) // Second + { + QueryInfo info; + info.name = getExportKey(ConfKeys::Second); + info.query = secondQuery_; + info.groupByField = groupByField_; + queries.push_back(info); + } + + return queries; + } + + std::string DatetimeVariable::getExportKey(const char* name) const + { + return getExportName() + "_" + name; + } } // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/DatetimeVariable.h b/src/bufr/BufrParser/Exports/Variables/DatetimeVariable.h index 32d74df1c..df03ccdc0 100644 --- a/src/bufr/BufrParser/Exports/Variables/DatetimeVariable.h +++ b/src/bufr/BufrParser/Exports/Variables/DatetimeVariable.h @@ -9,12 +9,10 @@ #include #include - +#include #include "eckit/config/LocalConfiguration.h" -#include "BufrParser/BufrTypes.h" -#include "DataObject/Int64VecDataObject.h" #include "Variable.h" @@ -24,36 +22,48 @@ namespace Ingester class DatetimeVariable final : public Variable { public: - explicit DatetimeVariable(const eckit::Configuration& conf); + DatetimeVariable() = delete; + DatetimeVariable(const std::string& exportName, + const std::string& groupByField, + const eckit::Configuration& conf); ~DatetimeVariable() final = default; /// \brief Get the configured mnemonics and turn them into datetime strings /// \param map BufrDataMap that contains the parsed data for each mnemonic - std::shared_ptr exportData(const BufrDataMap& map) final; + std::shared_ptr exportData(const BufrDataMap& map) final; + + /// \brief Get a list of queries for this variable + QueryList makeQueryList() const final; private: - /// \brief Mnemonic for year - const std::string yearKey_; + /// \brief Query for year + const std::string yearQuery_; - /// \brief Mnemonic for month - const std::string monthKey_; + /// \brief Query for month + const std::string monthQuery_; - /// \brief Mnemonic for day - const std::string dayKey_; + /// \brief Query for day + const std::string dayQuery_; - /// \brief Mnemonic for hour - const std::string hourKey_; + /// \brief Query for hour + const std::string hourQuery_; - /// \brief Mnemonic for minute - const std::string minuteKey_; + /// \brief Query for minute + const std::string minuteQuery_; - /// \brief Mnemonic for second (optional) - std::string secondKey_; + /// \brief Query for second (optional) + std::string secondQuery_; + + /// \brief For field (optional) + std::string groupByField_; /// \brief Hours to offset from UTC (optional) int hoursFromUtc_; /// \brief makes sure the bufr data map has all the required keys. void checkKeys(const BufrDataMap& map); + + /// \brief get the export key string + std::string getExportKey(const char* name) const; }; } // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/MnemonicVariable.cpp b/src/bufr/BufrParser/Exports/Variables/MnemonicVariable.cpp deleted file mode 100644 index 0f386187c..000000000 --- a/src/bufr/BufrParser/Exports/Variables/MnemonicVariable.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "MnemonicVariable.h" - -#include - -#include "eckit/exception/Exceptions.h" - -#include "IngesterTypes.h" - - -namespace Ingester -{ - MnemonicVariable::MnemonicVariable(const std::string& mnemonic, const Transforms& transforms) : - mnemonic_(mnemonic), - transforms_(transforms) - { - } - - std::shared_ptr MnemonicVariable::exportData(const BufrDataMap& map) - { - if (map.find(mnemonic_) == map.end()) - { - std::stringstream errStr; - errStr << "Mnemonic " << mnemonic_; - errStr << " could not be found during export."; - - eckit::BadParameter(errStr.str()); - } - - auto data = map.at(mnemonic_); - applyTransforms(data); - return std::make_shared(data); - } - - void MnemonicVariable::applyTransforms(IngesterArray& data) - { - for (auto transform : transforms_) - { - transform->apply(data); - } - } -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/MnemonicVariable.h b/src/bufr/BufrParser/Exports/Variables/MnemonicVariable.h deleted file mode 100644 index f411ad2f3..000000000 --- a/src/bufr/BufrParser/Exports/Variables/MnemonicVariable.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include - -#include "eckit/config/LocalConfiguration.h" - -#include "Variable.h" -#include "IngesterTypes.h" -#include "DataObject/ArrayDataObject.h" -#include "Transforms/Transform.h" - - -namespace Ingester -{ - /// \brief Exports parsed data associated with a mnemonic (ex: "CLAT") - class MnemonicVariable final : public Variable - { - public: - explicit MnemonicVariable(const std::string& mnemonicStr, const Transforms& transforms); - ~MnemonicVariable() final = default; - - /// \brief Gets the requested data, applies transforms, and returns the requested data - /// \param map BufrDataMap that contains the parsed data for each mnemonic - std::shared_ptr exportData(const BufrDataMap& map) final; - - private: - /// \brief The BUFR mnemonic of interest - std::string mnemonic_; - - /// \brief Collection of transforms to apply to the data during export - Transforms transforms_; - - /// \brief Apply the transforms - /// \param data Eigen Array data to apply the transform to. - void applyTransforms(IngesterArray& data); - }; -} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/QueryVariable.cpp b/src/bufr/BufrParser/Exports/Variables/QueryVariable.cpp new file mode 100644 index 000000000..b0033c748 --- /dev/null +++ b/src/bufr/BufrParser/Exports/Variables/QueryVariable.cpp @@ -0,0 +1,66 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "QueryVariable.h" + +#include + +#include "eckit/exception/Exceptions.h" + +#include "IngesterTypes.h" + + +namespace Ingester +{ + QueryVariable::QueryVariable(const std::string& exportName, + const std::string& query, + const std::string& groupByField, + const std::string& type, + const Transforms& transforms) : + Variable(exportName), + query_(query), + groupByField_(groupByField), + type_(type), + transforms_(transforms) + { + initQueryMap(); + } + + std::shared_ptr QueryVariable::exportData(const BufrDataMap& map) + { + if (map.find(getExportName()) == map.end()) + { + std::stringstream errStr; + errStr << "Export named " << getExportName(); + errStr << " could not be found during export."; + throw eckit::BadParameter(errStr.str()); + } + + auto dataObject = map.at(getExportName()); + + for (auto transform : transforms_) + { + transform->apply(dataObject); + } + + return dataObject; + } + + QueryList QueryVariable::makeQueryList() const + { + auto queries = QueryList(); + + QueryInfo info; + info.name = getExportName(); + info.query = query_; + info.groupByField = groupByField_; + info.type = type_; + queries.push_back(info); + + return queries; + } +} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/QueryVariable.h b/src/bufr/BufrParser/Exports/Variables/QueryVariable.h new file mode 100644 index 000000000..f809bc6e8 --- /dev/null +++ b/src/bufr/BufrParser/Exports/Variables/QueryVariable.h @@ -0,0 +1,56 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" + +#include "Variable.h" +#include "IngesterTypes.h" +#include "DataObject.h" +#include "Transforms/Transform.h" + + +namespace Ingester +{ + /// \brief Exports parsed data associated with a mnemonic (ex: "CLAT") + class QueryVariable final : public Variable + { + public: + explicit QueryVariable(const std::string& exportName, + const std::string& query, + const std::string& groupByField, + const std::string& type, + const Transforms& transforms); + + ~QueryVariable() final = default; + + /// \brief Gets the requested data, applies transforms, and returns the requested data + /// \param map BufrDataMap that contains the parsed data for each mnemonic + std::shared_ptr exportData(const BufrDataMap& map) final; + + /// \brief Get a list of queries for this variable + QueryList makeQueryList() const final; + + private: + /// \brief The query of interest + std::string query_; + + /// \brief The for field of interest + std::string groupByField_; + + /// \brief Optional type override string + std::string type_; + + /// \brief Collection of transforms to apply to the data during export + Transforms transforms_; + }; +} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/TimeoffsetVariable.cpp b/src/bufr/BufrParser/Exports/Variables/TimeoffsetVariable.cpp new file mode 100644 index 000000000..5a4fa6bf1 --- /dev/null +++ b/src/bufr/BufrParser/Exports/Variables/TimeoffsetVariable.cpp @@ -0,0 +1,152 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "eckit/exception/Exceptions.h" +#include "oops/util/Logger.h" + +#include "DataObject.h" +#include "TimeoffsetVariable.h" +#include "Transforms/TransformBuilder.h" + + +namespace +{ + namespace ConfKeys + { + const char* Timeoffset = "timeOffset"; + const char* Referencetime = "referenceTime"; + const char* Transforms = "transforms"; + } // namespace ConfKeys +} // namespace + + +namespace Ingester +{ + TimeoffsetVariable::TimeoffsetVariable(const std::string& exportName, + const std::string& groupByField, + const eckit::LocalConfiguration &conf) : + Variable(exportName), + groupByField_(groupByField), + conf_(conf) + { + initQueryMap(); + } + + std::shared_ptr TimeoffsetVariable::exportData(const BufrDataMap& map) + { + checkKeys(map); + + std::tm tm{}; // zero initialise + tm.tm_year = 1970 - 1900; // 1970 + tm.tm_mon = 0; // Jan=0, Feb=1, ... + tm.tm_mday = 1; // 1st + tm.tm_hour = 0; // midnight + tm.tm_min = 0; + tm.tm_sec = 0; + tm.tm_isdst = 0; // Not daylight saving + std::time_t epochDt = timegm(&tm); + + // Convert the reference time (ISO8601 string) to time struct + std::tm ref_time = {}; + std::istringstream ss(conf_.getString(ConfKeys::Referencetime)); + + ss >> std::get_time(&ref_time, "%Y-%m-%dT%H:%M:%S"); + if (ss.fail()) + { + std::ostringstream errStr; + errStr << "Reference time MUST be formatted like 2021-11-29T22:43:51Z"; + throw eckit::BadParameter(errStr.str()); + } + + auto timeOffsets = map.at(getExportKey(ConfKeys::Timeoffset)); + if (conf_.has(ConfKeys::Transforms)) + { + auto transforms = TransformBuilder::makeTransforms(conf_); + for (const auto &transform : transforms) + { + transform->apply(timeOffsets); + } + } + + auto timeDiffs = std::vector (timeOffsets->size()); + for (size_t idx = 0; idx < timeOffsets->size(); ++idx) + { + auto diff_time = DataObject::missingValue(); + if (!timeOffsets->isMissing(idx)) + { + auto obs_tm = ref_time; + obs_tm.tm_sec = ref_time.tm_sec + timeOffsets->getAsInt(idx); + auto thisTime = timegm(&obs_tm); + diff_time = static_cast(difftime(thisTime, epochDt)); + } + + timeDiffs[idx] = diff_time; + } + + return std::make_shared>(timeDiffs, + getExportName(), + groupByField_, + timeOffsets->getDims(), + timeOffsets->getPath(), + timeOffsets->getDimPaths()); + } + + void TimeoffsetVariable::checkKeys(const BufrDataMap& map) + { + std::vector requiredKeys = {getExportKey(ConfKeys::Timeoffset)}; + + std::stringstream errStr; + errStr << "Query "; + + bool isKeyMissing = false; + for (const auto& key : requiredKeys) + { + if (map.find(key) == map.end()) + { + isKeyMissing = true; + errStr << key; + break; + } + } + + errStr << " could not be found during export of datetime object."; + + if (isKeyMissing) + { + throw eckit::BadParameter(errStr.str()); + } + } + + QueryList TimeoffsetVariable::makeQueryList() const + { + auto queries = QueryList(); + + { // Timeoffset + QueryInfo info; + info.name = getExportKey(ConfKeys::Timeoffset); + info.query = conf_.getString(ConfKeys::Timeoffset); + queries.push_back(info); + } + + // The reference time string is a single scalar variable in YAML, not a query variable. + + return queries; + } + + std::string TimeoffsetVariable::getExportKey(const char* name) const + { + return getExportName() + "_" + name; + } +} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/TimeoffsetVariable.h b/src/bufr/BufrParser/Exports/Variables/TimeoffsetVariable.h new file mode 100644 index 000000000..3ef8aa414 --- /dev/null +++ b/src/bufr/BufrParser/Exports/Variables/TimeoffsetVariable.h @@ -0,0 +1,51 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" + +#include "Variable.h" + + +namespace Ingester +{ + /// \brief Exports parsed data as datetimes using speciefied Mnemonics + class TimeoffsetVariable final : public Variable + { + public: + TimeoffsetVariable() = delete; + TimeoffsetVariable(const std::string& exportName, + const std::string& groupByField, + const eckit::LocalConfiguration& conf); + ~TimeoffsetVariable() final = default; + + /// \brief Get the configured mnemonics and turn them into datetime strings + /// \param map BufrDataMap that contains the parsed data for each mnemonic + std::shared_ptr exportData(const BufrDataMap& map) final; + + /// \brief Get a list of queries for this variable + QueryList makeQueryList() const final; + + private: + // \\brief The field to group by. + std::string groupByField_; + + /// \brief Configuration for this variable + const eckit::LocalConfiguration conf_; + + /// \brief makes sure the bufr data map has all the required keys. + void checkKeys(const BufrDataMap& map); + + /// \brief get the export key string + std::string getExportKey(const char* name) const; + }; +} // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.cpp b/src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.cpp index 98ca3acbb..5b048f44a 100644 --- a/src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.cpp +++ b/src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.cpp @@ -6,6 +6,8 @@ */ #include "OffsetTransform.h" +#include "IngesterTypes.h" +#include "BufrParser/Query/Constants.h" namespace Ingester @@ -15,9 +17,9 @@ namespace Ingester { } - void OffsetTransform::apply(IngesterArray& array) + void OffsetTransform::apply(std::shared_ptr& dataObject) { - array = array + offset_; + dataObject->offsetBy(offset_); } } // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.h b/src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.h index d4a4059ba..1124b4fd8 100644 --- a/src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.h +++ b/src/bufr/BufrParser/Exports/Variables/Transforms/OffsetTransform.h @@ -23,7 +23,7 @@ namespace Ingester /// \brief Modify data according to the rules of the transform. /// \param array Array of data to modify. - void apply(IngesterArray& array) override; + void apply(std::shared_ptr& dataObject) override; private: const double offset_; diff --git a/src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.cpp b/src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.cpp index a6d0aa1c5..c895acb96 100644 --- a/src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.cpp +++ b/src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.cpp @@ -6,7 +6,8 @@ */ #include "ScalingTransform.h" - +#include "IngesterTypes.h" +#include "BufrParser/Query/Constants.h" namespace Ingester { @@ -15,8 +16,8 @@ namespace Ingester { } - void ScalingTransform::apply(IngesterArray& array) + void ScalingTransform::apply(std::shared_ptr& dataObject) { - array = array * scaling_; + dataObject->multiplyBy(scaling_); } } // namespace Ingester diff --git a/src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.h b/src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.h index 8b3b25119..9935151df 100644 --- a/src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.h +++ b/src/bufr/BufrParser/Exports/Variables/Transforms/ScalingTransform.h @@ -23,7 +23,7 @@ namespace Ingester /// \brief Modify data according to the rules of the transform. /// \param array Array of data to modify. - void apply(IngesterArray& array) override; + void apply(std::shared_ptr& dataObject) override; private: const double scaling_; diff --git a/src/bufr/BufrParser/Exports/Variables/Transforms/Transform.h b/src/bufr/BufrParser/Exports/Variables/Transforms/Transform.h index ca5a8a32e..d9dcf3ff5 100644 --- a/src/bufr/BufrParser/Exports/Variables/Transforms/Transform.h +++ b/src/bufr/BufrParser/Exports/Variables/Transforms/Transform.h @@ -10,7 +10,7 @@ #include #include -#include "IngesterTypes.h" +#include "DataObject.h" namespace Ingester { @@ -18,11 +18,11 @@ namespace Ingester class Transform { public: - ~Transform() = default; + virtual ~Transform() = default; /// \brief Modify data according to the rules of the transform. /// \param array Array of data to modify. - virtual void apply(IngesterArray& array) = 0; + virtual void apply(std::shared_ptr& dataObject) = 0; }; typedef std::vector > Transforms; diff --git a/src/bufr/BufrParser/Exports/Variables/Variable.h b/src/bufr/BufrParser/Exports/Variables/Variable.h index dc0191bbe..1c82883da 100644 --- a/src/bufr/BufrParser/Exports/Variables/Variable.h +++ b/src/bufr/BufrParser/Exports/Variables/Variable.h @@ -11,19 +11,53 @@ #include #include -#include "BufrParser/BufrTypes.h" -#include "DataObject/DataObject.h" +#include "IngesterTypes.h" +#include "DataObject.h" namespace Ingester { + struct QueryInfo + { + std::string name; + std::string query; + std::string groupByField; + std::string type; + }; + + typedef std::string QueryName; + typedef std::vector QueryList; + /// \brief Abstract base class for all Exports. class Variable { public: + Variable() = delete; + + explicit Variable(const std::string& exportName) : exportName_(exportName) {} + virtual ~Variable() = default; /// \brief Variable data objects for previously parsed data from BufrDataMap. - virtual std::shared_ptr exportData(const BufrDataMap& map) = 0; + virtual std::shared_ptr exportData(const BufrDataMap& dataMap) = 0; + + /// \brief Get Query List + inline QueryList getQueryList() { return queryList_; } + + /// \brief Get Export Name + inline std::string getExportName() const { return exportName_; } + + protected: + inline void initQueryMap() { queryList_ = makeQueryList(); } + + /// \brief Make a map of name and queries + virtual QueryList makeQueryList() const = 0; + + private: + /// \brief Name used to export this variable + std::string exportName_; + + /// \brief The query map for all the queries this variable needs + QueryList queryList_; }; } // namespace Ingester diff --git a/src/bufr/BufrParser/Query/Constants.h b/src/bufr/BufrParser/Query/Constants.h new file mode 100644 index 000000000..b67dd6cf5 --- /dev/null +++ b/src/bufr/BufrParser/Query/Constants.h @@ -0,0 +1,14 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ +#pragma once + +namespace Ingester { +namespace bufr { + /// \brief The missing data value for all BUFR data. + const double MissingValue = 10.0e10; +} // Ingester +} // bufr diff --git a/src/bufr/BufrParser/Query/DataProvider.cpp b/src/bufr/BufrParser/Query/DataProvider.cpp new file mode 100644 index 000000000..2506c6dbb --- /dev/null +++ b/src/bufr/BufrParser/Query/DataProvider.cpp @@ -0,0 +1,160 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "DataProvider.h" +#include "bufr_interface.h" + +#include +#include +#include + +#include "eckit/exception/Exceptions.h" + +namespace +{ + const char* Subset = "SUB"; + const char* DelayedRep = "DRP"; + const char* FixedRep = "REP"; + const char* DelayedRepStacked = "DRS"; + const char* DelayedBinary = "DRB"; + const char* Sequence = "SEQ"; + const char* Repeat = "RPC"; + const char* StackedRepeat = "RPS"; + const char* Number = "NUM"; + const char* Character = "CHR"; +} // namespace + + +namespace Ingester { +namespace bufr { + + void DataProvider::updateData(int bufrLoc) + { + bufrLoc_ = bufrLoc; + int size = 0; + int *intPtr = nullptr; + double *dataPtr = nullptr; + + int strLen = 0; + char *charPtr = nullptr; + + if (isc_.empty()) + { + get_isc_f(&intPtr, &size); + isc_ = gsl::span(intPtr, size); + + get_link_f(&intPtr, &size); + link_ = gsl::span(intPtr, size); + + get_itp_f(&intPtr, &size); + itp_ = gsl::span(intPtr, size); + + get_typ_f(&charPtr, &strLen, &size); + typ_.resize(size); + for (int wordIdx = 0; wordIdx < size; wordIdx++) + { + static const std::unordered_map TypMap = + {{Subset, Typ::Subset}, + {DelayedRep, Typ::DelayedRep}, + {FixedRep, Typ::FixedRep}, + {DelayedRepStacked, Typ::DelayedRepStacked}, + {DelayedBinary, Typ::DelayedBinary}, + {Sequence, Typ::Sequence}, + {Repeat, Typ::Repeat}, + {StackedRepeat, Typ::StackedRepeat}, + {Number, Typ::Number}, + {Character, Typ::Character}}; + + auto typ = std::string(&charPtr[wordIdx * strLen], strLen); + typ_[wordIdx] = TypMap.at(typ); + } + + get_tag_f(&charPtr, &strLen, &size); + tag_.resize(size); + for (int wordIdx = 0; wordIdx < size; wordIdx++) + { + auto tag = std::string(&charPtr[wordIdx * strLen], strLen); + tag_[wordIdx] = tag.substr(0, tag.find_first_of(' ')); + } + + get_jmpb_f(&intPtr, &size); + jmpb_ = gsl::span(intPtr, size); + } + + get_inode_f(bufrLoc, &inode_); + get_nval_f(bufrLoc, &nval_); + + get_val_f(bufrLoc, &dataPtr, &size); + val_ = gsl::span(dataPtr, size); + + get_inv_f(bufrLoc, &intPtr, &size); + inv_ = gsl::span(intPtr, size); + + subset_ = getTag(getInode()); + } + + void DataProvider::deleteData() + { + delete_table_data_f(); + } + + TypeInfo DataProvider::getTypeInfo(FortranIdx idx) const + { + static const unsigned int UNIT_STR_LEN = 24; + static const unsigned int DESC_STR_LEN = 55; + + char unitCStr[UNIT_STR_LEN]; + char descCStr[DESC_STR_LEN]; + + int retVal; + TypeInfo info; + + nemdefs_f(fileUnit_, + getTag(idx).c_str(), + unitCStr, + UNIT_STR_LEN, + descCStr, + DESC_STR_LEN, + &retVal); + + if (retVal == 0) + { + // trim the unit string + auto unitStr = std::string(unitCStr); + size_t end = unitStr.find_last_not_of(" \n\r\t\f\v"); + unitStr = (end == std::string::npos) ? "" : unitStr.substr(0, end + 1); + info.unit = unitStr; + + // trim the unit string + auto descStr = std::string(descCStr); + end = descStr.find_last_not_of(" \n\r\t\f\v"); + descStr = (end == std::string::npos) ? "" : descStr.substr(0, end + 1); + info.description = descStr; + + int descriptor; + int table_idx; + char table_type; + + nemtab_f(bufrLoc_, + getTag(idx).c_str(), + &descriptor, + &table_type, + &table_idx); + + nemtbb_f(bufrLoc_, + table_idx, + unitCStr, + UNIT_STR_LEN, + &info.scale, + &info.reference, + &info.bits); + } + + return info; + } +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/DataProvider.h b/src/bufr/BufrParser/Query/DataProvider.h new file mode 100644 index 000000000..7635d8190 --- /dev/null +++ b/src/bufr/BufrParser/Query/DataProvider.h @@ -0,0 +1,125 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Ingester{ +namespace bufr { + typedef unsigned int FortranIdx; + + enum class Typ + { + Subset, + DelayedRep, + FixedRep, + DelayedRepStacked, + DelayedBinary, + Sequence, + Repeat, + StackedRepeat, + Number, + Character + }; + + struct TypeInfo + { + int scale = 0; + int reference = 0; + int bits = 0; + std::string unit; + std::string description; + + bool isString() const { return unit == "CCITT IA5"; } + bool isSigned() const + { + // To better support Fortran clients for the generated ObsGroups we will assume all + // fields are signed. Otherwise this code would be reference < 0. + return true; + } + bool isInteger() const { return scale <= 0; } + bool is64Bit() const + { + bool is64Bit; + if (isInteger() && !isSigned()) + { + is64Bit = (log2((pow(2, bits) - 1) / pow(10, scale) + reference) > 32); + } + else if (isInteger() && isSigned()) + { + is64Bit = (log2(fmax(-1 * reference, + (pow(2, bits - 1) - 1) / pow(10, scale) + reference) * 2) + 1 > 32); + } + else + { + is64Bit = false; + } + + return is64Bit; + } + }; + + /// \brief Responsible for exposing the data found in a BUFR file in a C friendly way. + class DataProvider + { + public: + explicit DataProvider(int fileUnit) : fileUnit_(fileUnit) {} + ~DataProvider() = default; + + /// \brief Read the data from the BUFR interface for the current subset and reset the + /// internal data structures. + ////// \param bufrLoc The Fortran idx for the subset we need to read. + void updateData(int bufrLoc); + + /// \brief Tells the Fortran BUFR interface to delete its temporary data structures that are + /// are needed to support this class instanc. + void deleteData(); + + /// \brief Get the subset string for the currently active message subset. + std::string getSubset() const { return subset_; } + + // Getters to get the raw data by idx. Since fortran indices are 1-based, + // we need to subtract 1 to get the correct c style index. + inline FortranIdx getIsc(FortranIdx idx) const { return isc_[idx - 1]; } + inline FortranIdx getLink(FortranIdx idx) const { return link_[idx - 1]; } + inline FortranIdx getItp(FortranIdx idx) const { return itp_[idx - 1]; } + inline FortranIdx getJmpb(FortranIdx idx) const { return jmpb_[idx - 1]; } + inline Typ getTyp(FortranIdx idx) const { return typ_[idx - 1]; } + inline std::string getTag(FortranIdx idx) const { return tag_[idx - 1]; } + + inline FortranIdx getInode() const { return inode_; } + inline FortranIdx getNVal() const { return nval_; } + inline FortranIdx getInv(FortranIdx idx) const { return inv_[idx - 1]; } + inline double getVal(FortranIdx idx) const { return val_[idx - 1]; } + TypeInfo getTypeInfo(FortranIdx idx) const; + + private: + int fileUnit_; + std::string subset_; + + // Table data; + gsl::span isc_; + gsl::span link_; + gsl::span itp_; + gsl::span jmpb_; + std::vector typ_; + std::vector tag_; + + // Subset data + int inode_; + int nval_; + int bufrLoc_; + gsl::span val_; + gsl::span inv_; + }; +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/File.cpp b/src/bufr/BufrParser/Query/File.cpp new file mode 100644 index 000000000..d05aa582d --- /dev/null +++ b/src/bufr/BufrParser/Query/File.cpp @@ -0,0 +1,108 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "File.h" + +#include + +#include "bufr_interface.h" + +#include "QueryRunner.h" +#include "QuerySet.h" +#include "DataProvider.h" + + +namespace Ingester { +namespace bufr { + File::File(const std::string &filename, bool isWmoFormat, const std::string &wmoTablePath) : + filename_(filename), + fileUnit_(nextFileUnit()), + fileUnitTable1_(nextFileUnit()), + fileUnitTable2_(nextFileUnit()), + isWmoFormat_(isWmoFormat), + wmoTablePath_(wmoTablePath) + { + open(); + } + + void File::open() + { + open_f(fileUnit_, filename_.c_str()); + + if (!isWmoFormat_) + { + openbf_f(fileUnit_, "IN", fileUnit_); + } + else + { + openbf_f(fileUnit_, "SEC3", fileUnit_); + + if (!wmoTablePath_.empty()) + { + mtinfo_f(wmoTablePath_.c_str(), fileUnitTable1_, fileUnitTable2_); + } + } + } + + void File::close() + { + closbf_f(fileUnit_); + close_f(fileUnit_); + } + + void File::rewind() { + close(); + open(); + } + + ResultSet File::execute(const QuerySet &querySet, size_t next) + { + static int SubsetLen = 9; + unsigned int messageNum = 0; + char subsetChars[SubsetLen]; + int iddate; + + int bufrLoc; + int il, im; // throw away + + auto dataProvider = DataProvider(fileUnit_); + + auto resultSet = ResultSet(querySet.names()); + auto queryRunner = QueryRunner(querySet, resultSet, dataProvider); + + while (ireadmg_f(fileUnit_, subsetChars, &iddate, SubsetLen) == 0) + { + auto subset = std::string(subsetChars); + subset.erase(std::remove_if(subset.begin(), subset.end(), isspace), subset.end()); + + if (querySet.includesSubset(subset)) + { + while (ireadsb_f(fileUnit_) == 0) + { + status_f(fileUnit_, &bufrLoc, &il, &im); + dataProvider.updateData(bufrLoc); + queryRunner.accumulate(); + } + + if (next > 0 && ++messageNum >= next) break; + } + } + + resultSet.setTargets(queryRunner.getTargets()); + + dataProvider.deleteData(); + + return resultSet; + } + + int File::nextFileUnit() + { + static int lastFileUnit = 11; // Numbers 12 and above are valid. + return ++lastFileUnit; + } +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/File.h b/src/bufr/BufrParser/Query/File.h new file mode 100644 index 000000000..d14c58ec4 --- /dev/null +++ b/src/bufr/BufrParser/Query/File.h @@ -0,0 +1,57 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include + +#include "QuerySet.h" +#include "ResultSet.h" + +namespace Ingester { +namespace bufr { + + /// \breif Manages an open BUFR file. + class File + { + public: + File() = delete; + + File(const std::string& filename, + bool isWmoFormat = false, + const std::string& wmoTablePath = ""); + + /// \brief Execute the queries given in the query set over the BUFR file and accumulate the + /// resulting data in the ResultSet. + /// \param query_set The queryset object that contains the collection of desired queries + /// \param next The number of messages worth of data to run. 0 reads all messages in the + /// file. + ResultSet execute(const QuerySet& query_set, size_t next = 0); + + /// \brief Close the currently opened BUFR file. + void close(); + + /// \brief Rewind the currently opened BUFR file to the beginning. + void rewind(); + + private: + const std::string filename_; + const int fileUnit_; + const int fileUnitTable1_; + const int fileUnitTable2_; + const bool isWmoFormat_; + const std::string wmoTablePath_; + + + /// \brief Open the BUFR file whose parameters where given in the constructor. + void open(); + + /// \brief Get the next available Fortran file unit number. + int nextFileUnit(); + }; +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/QueryParser.cpp b/src/bufr/BufrParser/Query/QueryParser.cpp new file mode 100644 index 000000000..28ea941f5 --- /dev/null +++ b/src/bufr/BufrParser/Query/QueryParser.cpp @@ -0,0 +1,147 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "QueryParser.h" + +#include +#include + +#include "eckit/exception/Exceptions.h" + +namespace Ingester { +namespace bufr { + std::vector QueryParser::parse(const std::string& queryStr) + { + std::vector queries; + for (auto& subStr : QueryParser::splitMultiquery(queryStr)) + { + queries.emplace_back(QueryParser::splitQueryStr(subStr)); + } + + return queries; + } + + std::vector QueryParser::splitMultiquery(const std::string &query) + { + std::vector subqueries; + + // Remove whitespace from query and assign to working_str + std::string working_str = query; + working_str.erase( + std::remove(working_str.begin(), working_str.end(), ' '), working_str.end()); + + if (working_str.substr(0, 1) == "[") { + if (working_str.substr(working_str.length() - 1, 1) != "]") { + std::stringstream errMsg; + errMsg << "Query Parser: multi query is lacking closing brackets." << std::endl; + throw eckit::BadParameter(errMsg.str()); + } + + working_str = working_str.substr(1, working_str.length() - 2); + + std::vector comma_positions; + + size_t last_pos = 0; + while (working_str.find(',', last_pos) != std::string::npos) + { + auto comma_idx = working_str.find(',', last_pos); + comma_positions.push_back(comma_idx); + last_pos = comma_idx + 1; + } + + last_pos = 0; + subqueries.reserve(comma_positions.size() + 1); + for (size_t commaIdx = 0; commaIdx < comma_positions.size() + 1; ++commaIdx) + { + if (commaIdx < comma_positions.size()) + { + subqueries.push_back( + working_str.substr(last_pos, comma_positions[commaIdx] - last_pos)); + last_pos = comma_positions[commaIdx] + 1; + } + else + { + subqueries.push_back( + working_str.substr(last_pos, working_str.length() - last_pos)); + } + } + } + else + { + subqueries.push_back(working_str); + } + + return subqueries; + } + + Query QueryParser::splitQueryStr(const std::string& query) + { + // Find positions of slashes + std::vector slashPositions; + size_t slashIdx = 0; + size_t charIdx = 0; + for (charIdx = 0; charIdx < query.length(); ++charIdx) { + if (charIdx > 0 && query[charIdx] == '/') { + slashPositions.push_back(charIdx); + slashIdx++; + } + } + + if (slashPositions.size() < 1) { + std::stringstream errMsg; + errMsg << "Query Parser: Not a valid query string." << std::endl; + throw eckit::BadParameter(errMsg.str()); + } + + // Capture the subset string + auto subset = query.substr(0, slashPositions[0]); + + std::vector mnemonicStrings(slashPositions.size()); + + // Capture the sequence mnemonic strings + for (size_t mnemonicIdx = 0; mnemonicIdx < mnemonicStrings.size() - 1; ++mnemonicIdx) + { + mnemonicStrings[mnemonicIdx] = + query.substr(slashPositions[mnemonicIdx] + 1, + slashPositions[mnemonicIdx + 1] - slashPositions[mnemonicIdx] - 1); + } + + // Get the last element + std::string lastElement = query.substr(slashPositions[slashPositions.size() - 1] + 1); + + // Parse last element + int index = -1; + size_t startSubscript = lastElement.find_first_of("["); + size_t endSubscript = lastElement.find_first_of("]"); + if (startSubscript != std::string::npos && endSubscript != std::string::npos) + { + index = std::stoi(lastElement.substr(startSubscript + 1, + endSubscript - startSubscript - 1)); + mnemonicStrings[mnemonicStrings.size() - 1] = lastElement.substr(0, startSubscript); + } + else + { + if (endSubscript != std::string::npos || startSubscript != std::string::npos) + { + std::stringstream errMsg; + errMsg << "Query Parser: Not a valid query string. Extra brackets." << std::endl; + throw eckit::BadParameter(errMsg.str()); + } + + mnemonicStrings.back() = lastElement; + } + + auto queryObj = Query(); + queryObj.queryStr = query; + queryObj.subset = subset; + queryObj.mnemonics = mnemonicStrings; + queryObj.index = index; + + return queryObj; + } +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/QueryParser.h b/src/bufr/BufrParser/Query/QueryParser.h new file mode 100644 index 000000000..dd360e78e --- /dev/null +++ b/src/bufr/BufrParser/Query/QueryParser.h @@ -0,0 +1,45 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include + +namespace Ingester { +namespace bufr { + + struct Query + { + std::string queryStr; + std::string subset; + std::vector mnemonics; + int index; + }; + + /// \brief Parses a user supplied query string into its component parts. + /// \note Will be refactored to properly tokenize the query string. + class QueryParser + { + public: + static std::vector parse(const std::string& queryStr); + + private: + /// \brief Split a multi query (ex: ["*/CLONH", "*/CLON"]) into a vector of single queries. + /// \param query The query to split. + static std::vector splitMultiquery(const std::string& query); + + /// \brief Split a single query (ex: "*/ROSEQ1/ROSEQ2/PCCF[2]") into its component parts. + /// \param query The query to split. + static Query splitQueryStr(const std::string& query); + + private: + /// \brief Private constructor. + QueryParser(); + }; +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/QueryRunner.cpp b/src/bufr/BufrParser/Query/QueryRunner.cpp new file mode 100644 index 000000000..fae955f90 --- /dev/null +++ b/src/bufr/BufrParser/Query/QueryRunner.cpp @@ -0,0 +1,417 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ +#include "QueryRunner.h" + +#include "eckit/exception/Exceptions.h" +#include "oops/util/Logger.h" + +#include +#include + +#include "Constants.h" + +namespace Ingester { +namespace bufr { + + struct NodeData { + std::vector values; + std::vector counts; + }; + + QueryRunner::QueryRunner(const QuerySet &querySet, + ResultSet &resultSet, + const DataProvider &dataProvider) : + querySet_(querySet), + resultSet_(resultSet), + dataProvider_(dataProvider) + { + } + + void QueryRunner::accumulate() + { + Targets targets; + std::shared_ptr<__details::ProcessingMasks> masks; + + findTargets(targets, masks); + collectData(targets, masks, resultSet_); + } + + void QueryRunner::findTargets(Targets &targets, + std::shared_ptr<__details::ProcessingMasks> &masks) + { + // Check if the target list for this subset is cached + if (targetCache_.find(dataProvider_.getSubset()) != targetCache_.end()) + { + targets = targetCache_.at(dataProvider_.getSubset()); + masks = maskCache_.at(dataProvider_.getSubset()); + return; + } + + masks = std::make_shared<__details::ProcessingMasks>(); + + size_t numNodes = dataProvider_.getIsc(dataProvider_.getInode()); + + masks->valueNodeMask.resize(numNodes, false); + masks->pathNodeMask.resize(numNodes, false); + + for (size_t targetIdx = 0; targetIdx < querySet_.size(); ++targetIdx) + { + auto queryName = querySet_.names()[targetIdx]; + auto subQueries = querySet_.queriesFor(queryName); + + bool foundTarget = false; + std::shared_ptr target; + for (size_t subQueryIdx = 0; subQueryIdx < subQueries.size(); ++subQueryIdx) + { + const Query& subQuery = subQueries[subQueryIdx]; + + target = findTarget(queryName, subQuery); + + if (target->nodeIds.size() > 0) + { + // Collect mask data + masks->valueNodeMask[target->nodeIds[0]] = true; + for (size_t pathIdx = 0; pathIdx < target->seqPath.size(); ++pathIdx) { + masks->pathNodeMask[target->seqPath[pathIdx]] = true; + } + + targets.push_back(target); + foundTarget = true; + break; + } + } + + if (!foundTarget) + { + // Add the last missing target to the list + targets.push_back(target); + oops::Log::warning() << "Warning: Query String "; + + auto queries = querySet_.queriesFor(queryName); + + if (queries.size() == 1) + { + oops::Log::warning() << queries[0].queryStr; + } + else + { + oops::Log::warning() << "["; + for (auto subQuery = queries.cbegin(); + subQuery < queries.cend(); + ++subQuery) + { + if (subQuery != queries.cbegin()) oops::Log::warning() << ", "; + oops::Log::warning() << subQuery->queryStr; + } + oops::Log::warning() << "]"; + } + + oops::Log::warning() << " didn't apply to subset "; + oops::Log::warning() << dataProvider_.getSubset(); + oops::Log::warning() << std::endl; + } + } + + targetCache_.insert({dataProvider_.getSubset(), targets}); + maskCache_.insert({dataProvider_.getSubset(), masks}); + } + + std::shared_ptr QueryRunner::findTarget(const std::string &targetName, + const Query& query) const + { + std::vector branches; + std::vector targetNodes; + std::vector seqPath; + std::vector dimPaths; + std::vector dimIdxs; + + bool targetMissing = !(query.subset == "*" || query.subset == dataProvider_.getSubset()); + if (!targetMissing) { + branches.resize(query.mnemonics.size() - 1); + + seqPath.push_back(dataProvider_.getInode()); + + int tableCursor = -1; + int mnemonicCursor = -1; + + for (auto nodeIdx = dataProvider_.getInode(); + nodeIdx <= dataProvider_.getIsc(dataProvider_.getInode()); + nodeIdx++) { + if (dataProvider_.getTyp(nodeIdx) == Typ::Sequence || + dataProvider_.getTyp(nodeIdx) == Typ::Repeat || + dataProvider_.getTyp(nodeIdx) == Typ::StackedRepeat) { + if (isQueryNode(nodeIdx - 1)) { + if (dataProvider_.getTag(nodeIdx) == query.mnemonics[mnemonicCursor + 1] && + tableCursor == mnemonicCursor) { + mnemonicCursor++; + branches[mnemonicCursor] = nodeIdx - 1; + } + tableCursor++; + } + seqPath.push_back(nodeIdx); + } else if (mnemonicCursor == static_cast(query.mnemonics.size()) - 2 && + tableCursor == mnemonicCursor && + dataProvider_.getTag(nodeIdx) == query.mnemonics.back()) { + // We found a target + targetNodes.push_back(nodeIdx); + getDimInfo(branches, mnemonicCursor, dimPaths, dimIdxs); + } + + // Step back up the tree (unfortunately this is finicky) + if (seqPath.size() > 1) { + // Skip pure sequences not inside any kind of repeated sequence + auto jumpBackNode = dataProvider_.getInode(); + if (nodeIdx < dataProvider_.getIsc(dataProvider_.getInode())) + { + jumpBackNode = dataProvider_.getJmpb(nodeIdx + 1); + if (jumpBackNode == 0) jumpBackNode = dataProvider_.getInode(); + while (dataProvider_.getTyp(jumpBackNode) == Typ::Sequence && + dataProvider_.getTyp(jumpBackNode - 1) != Typ::DelayedRep && + dataProvider_.getTyp(jumpBackNode - 1) != Typ::FixedRep && + dataProvider_.getTyp(jumpBackNode - 1) != Typ::DelayedRepStacked && + dataProvider_.getTyp(jumpBackNode - 1) != Typ::DelayedBinary) + { + auto newJumpBackNode = dataProvider_.getJmpb(jumpBackNode); + if (newJumpBackNode != jumpBackNode) + { + jumpBackNode = newJumpBackNode; + } + else + { + break; + } + } + } + + + // Peak ahead to see if the next node is inside one of the containing sequences + // then go back up the approptiate number of sequences. You may have to exit + // several sequences in a row if the current sequence is the last element in the + // containing sequence. + for (int pathIdx = seqPath.size() - 2; pathIdx >= 0; pathIdx--) { + if (seqPath[pathIdx] == jumpBackNode) { + for (int rewindIdx = seqPath.size() - 1; + rewindIdx > pathIdx; + rewindIdx--) { + // Exit the sequence + if (isQueryNode(seqPath[rewindIdx] - 1)) { + if (mnemonicCursor > -1 && tableCursor == mnemonicCursor) { + mnemonicCursor--; + } + + tableCursor--; + } + // Pop out of the current sequence + seqPath.pop_back(); + } + break; + } + } + } + } + + if (query.index > 0 && query.index <= gsl::narrow(targetNodes.size())) { + targetNodes = {targetNodes[query.index - 1]}; + } + + if (targetNodes.size() > 1) { + std::ostringstream errMsg; + errMsg << "Query string must return 1 target. Are you missing an index? "; + errMsg << query.queryStr << "."; + throw eckit::BadParameter(errMsg.str()); + } + } + + auto target = std::make_shared(); + target->name = targetName; + target->queryStr = query.queryStr; + target->seqPath = branches; + target->nodeIds = targetNodes; + + if (targetNodes.size() > 0) { + target->dimPaths = dimPaths; + target->exportDimIdxs = dimIdxs; + target->typeInfo = dataProvider_.getTypeInfo(targetNodes[0]); + } else { + target->dimPaths = {"*"}; + target->exportDimIdxs = {0}; + target->typeInfo = TypeInfo(); + } + + return target; + } + + bool QueryRunner::isQueryNode(int nodeIdx) const { + return (dataProvider_.getTyp(nodeIdx) == Typ::DelayedRep || + dataProvider_.getTyp(nodeIdx) == Typ::FixedRep || + dataProvider_.getTyp(nodeIdx) == Typ::DelayedRepStacked || + dataProvider_.getTyp(nodeIdx) == Typ::DelayedBinary); + } + + void QueryRunner::getDimInfo(const std::vector &branches, + int mnemonicCursor, + std::vector &dimPaths, + std::vector &dimIdxs) const { + std::string currentDimPath; + std::string mnemonicStr; + + // Initialize out parameters + dimPaths = std::vector(); + dimIdxs = std::vector(); + + // Allocate enough memory to hold all the dim paths + dimPaths.reserve(branches.size() + 1); + dimIdxs.reserve(branches.size() + 1); + + currentDimPath = "*"; + dimPaths.push_back(currentDimPath); + dimIdxs.push_back(0); + + // Split the branches into node idxs for each additional dimension + if (mnemonicCursor >= 0) { + int dimIdx = 1; + for (int branchIdx = 0; branchIdx <= mnemonicCursor; branchIdx++) { + int nodeIdx = branches[branchIdx]; + mnemonicStr = dataProvider_.getTag(nodeIdx); + + std::ostringstream path; + path << currentDimPath << "/" << mnemonicStr.substr(1, mnemonicStr.size() - 2); + currentDimPath = path.str(); + + if (dataProvider_.getTyp(nodeIdx) == Typ::DelayedRep || + dataProvider_.getTyp(nodeIdx) == Typ::FixedRep || + dataProvider_.getTyp(nodeIdx) == Typ::DelayedRepStacked) { + dimIdx = dimIdx + 1; + dimIdxs.push_back(branchIdx + 1); // +1 to account for the root dimension + dimPaths.push_back(currentDimPath); + } + } + } + } + + void QueryRunner::collectData(Targets& targets, + std::shared_ptr<__details::ProcessingMasks> masks, + ResultSet &resultSet) const { + std::vector currentPath; + std::vector currentPathReturns; + + currentPath.reserve(10); + currentPathReturns.reserve(10); + + auto &dataFrame = resultSet.nextDataFrame(); + int returnNodeIdx = -1; + int lastNonZeroReturnIdx = -1; + + // Reorganize the data into a NodeValueTable to make lookups faster (avoid looping over all + // the data a bunch of times) + auto dataTable = __details::OffsetArray( + dataProvider_.getInode(), + dataProvider_.getIsc(dataProvider_.getInode())); + + for (size_t dataCursor = 1; dataCursor <= dataProvider_.getNVal(); ++dataCursor) { + int nodeIdx = dataProvider_.getInv(dataCursor); + + if (masks->valueNodeMask[nodeIdx]) { + auto &values = dataTable[nodeIdx].values; + values.push_back(dataProvider_.getVal(dataCursor)); + } + + // Unfortuantely the fixed replicated sequences do not store their counts as values for + // the Fixed Replication nodes. It's therefore necessary to discover this information by + // manually tracing the nested sequences and counting everything manually. Since we have + // to do it for fixed reps anyways, its easier just to do it for all the squences. + if (dataProvider_.getJmpb(nodeIdx) > 0 && + masks->pathNodeMask[dataProvider_.getJmpb(nodeIdx)]) { + const auto typ = dataProvider_.getTyp(nodeIdx); + const auto jmpbTyp = dataProvider_.getTyp(dataProvider_.getJmpb(nodeIdx)); + if ((typ == Typ::Sequence && (jmpbTyp == Typ::Sequence || + jmpbTyp == Typ::DelayedBinary || + jmpbTyp == Typ::FixedRep)) || + typ == Typ::Repeat || + typ == Typ::StackedRepeat) { + dataTable[nodeIdx].counts.back()++; + } + } + + if (currentPath.size() >= 1) { + if (nodeIdx == returnNodeIdx || + dataCursor == dataProvider_.getNVal() || + (currentPath.size() > 1 && nodeIdx == *(currentPath.end() - 1) + 1)) { + // Look for the first path return idx that is not 0 and check if its this node + // idx. Exit the sequence if its appropriate. A return idx of 0 indicates a + // sequence that occurs as the last element of another sequence. + for (int pathIdx = currentPathReturns.size() - 1; + pathIdx >= lastNonZeroReturnIdx; + --pathIdx) { + currentPathReturns.pop_back(); + auto seqNodeIdx = currentPath.back(); + currentPath.pop_back(); + + const auto typSeqNode = dataProvider_.getTyp(seqNodeIdx); + if (typSeqNode == Typ::DelayedRep || typSeqNode == Typ::DelayedRepStacked) { + dataTable[seqNodeIdx + 1].counts.back()--; + } + } + + lastNonZeroReturnIdx = currentPathReturns.size() - 1; + returnNodeIdx = currentPathReturns[lastNonZeroReturnIdx]; + } + } + + if (masks->pathNodeMask[nodeIdx] && isQueryNode(nodeIdx)) { + if (dataProvider_.getTyp(nodeIdx) == Typ::DelayedBinary && + dataProvider_.getVal(dataCursor) == 0) { + // Ignore the node if it is a delayed binary and the value is 0 + } else { + currentPath.push_back(nodeIdx); + const auto tmpReturnNodeIdx = dataProvider_.getLink(nodeIdx); + currentPathReturns.push_back(tmpReturnNodeIdx); + + if (tmpReturnNodeIdx != 0) { + lastNonZeroReturnIdx = currentPathReturns.size() - 1; + returnNodeIdx = tmpReturnNodeIdx; + } else { + lastNonZeroReturnIdx = 0; + returnNodeIdx = 0; + + if (dataCursor != dataProvider_.getNVal()) { + for (int pathIdx = currentPath.size() - 1; pathIdx >= 0; --pathIdx) { + returnNodeIdx = dataProvider_.getLink( + dataProvider_.getJmpb(currentPath[pathIdx])); + lastNonZeroReturnIdx = currentPathReturns.size() - pathIdx; + + if (returnNodeIdx != 0) break; + } + } + } + } + + dataTable[nodeIdx + 1].counts.push_back(0); + } + } + + for (size_t targetIdx = 0; targetIdx < targets.size(); targetIdx++) { + const auto &targ = targets.at(targetIdx); + auto &dataField = dataFrame.fieldAtIdx(targetIdx); + dataField.target = targ; + + if (targ->nodeIds.size() == 0) { + dataField.data = {MissingValue}; + dataField.seqCounts = {{1}}; + } else { + dataField.seqCounts.resize(targ->seqPath.size() + 1); + dataField.seqCounts[0] = {1}; + for (size_t pathIdx = 0; pathIdx < targ->seqPath.size(); pathIdx++) { + dataField.seqCounts[pathIdx + 1] = dataTable[targ->seqPath[pathIdx] + 1].counts; + } + + dataField.data = dataTable[targ->nodeIds[0]].values; + } + } + } +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/QueryRunner.h b/src/bufr/BufrParser/Query/QueryRunner.h new file mode 100644 index 000000000..8e9d0d731 --- /dev/null +++ b/src/bufr/BufrParser/Query/QueryRunner.h @@ -0,0 +1,131 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include +#include + +#include "QuerySet.h" +#include "ResultSet.h" +#include "DataProvider.h" +#include "Target.h" + +namespace Ingester { +namespace bufr { + namespace __details + { + /// \brief BUFR messages are indexed according to start and stop values that are dependant + /// on the message itself (the indexing is a property of the message). This object allows + /// lets you make an array where the indexing is offset with respect to the actual position + /// of the object in the array. + template + class OffsetArray + { + public: + OffsetArray(size_t startIdx, size_t endIdx) + : offset_(startIdx) + { + data_.resize(endIdx - startIdx + 1); + } + + T& operator[](size_t idx) { return data_[idx - offset_]; } + + private: + std::vector data_; + size_t offset_; + }; + + /// \brief Masks used to make the processing of BUFR data more efficient. The aim is to skip + /// branches without data we care about. + struct ProcessingMasks { + std::vector valueNodeMask; + std::vector pathNodeMask; + }; + } // namespace __details + + /// \brief Manages the execution of queries against on a BUFR file. + class QueryRunner + { + public: + /// \brief Constructor. + /// \param[in] querySet The set of queries to execute against the BUFR file. + /// \param[in, out] resultSet The object used to store the accumulated collected data. + /// \param[in] dataProvider The BUFR data provider to use. + QueryRunner(const QuerySet& querySet, + ResultSet& resultSet, + const DataProvider& dataProvider); + void accumulate(); + + Targets getTargets() + { + Targets targets; + for (auto& subset : targetCache_) + { + for (auto& target : subset.second) + { + targets.push_back(target); + } + } + return targets; + } + + private: + const QuerySet querySet_; + ResultSet& resultSet_; + const DataProvider& dataProvider_; + + std::unordered_map targetCache_; + std::unordered_map> maskCache_; + std::unordered_map> unitCache_; + + + /// \brief Look for the list of targets for the currently active BUFR message subset that + /// apply to the QuerySet and cache them. Processing mask information is also collected in + /// order to make the data collection more efficient. + /// \param[in, out] targets The list of targets to populate. + /// \param[in, out] masks The processing masks to populate. + void findTargets(Targets& targets, + std::shared_ptr<__details::ProcessingMasks>& masks); + + + /// \brief Find the target associated with a specific user provided query string. + /// \param[in] targetName The name specified for the target. + /// \param[in] query The query string to use. + std::shared_ptr findTarget(const std::string &targetName, + const Query& query) const; + + + /// \brief Does the node idx correspond to an element you'd find in a query string (repeat + /// or binary sequence)? + /// \param[in] nodeIdx The node index to check. + bool isQueryNode(int nodeIdx) const; + + + /// \brief Get the dimensional information for the query with the given branches. + /// \param[in] branches The branches to use. + /// \param[in] mnemonicCursor The current position in the subset tree. + /// \param[in, out] dimPaths The list of dimensioning query sub-paths + /// \param[in, out] dimIdxs The idxs of the dimensioning elements in the query. + void getDimInfo(const std::vector& branches, + int mnemonicCursor, + std::vector& dimPaths, + std::vector& dimIdxs) const; + + + /// \brief Accumulate the data for the currently open BUFR message subset. + /// \param[in] targets The list of targets to collect for this subset. + /// \param[in] masks The processing masks to use. + /// \param[in, out] resultSet The object used to store the accumulated collected data. + void collectData(Targets& targets, + std::shared_ptr<__details::ProcessingMasks> masks, + ResultSet& resultSet) const; + }; +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/QuerySet.cpp b/src/bufr/BufrParser/Query/QuerySet.cpp new file mode 100644 index 000000000..5587136de --- /dev/null +++ b/src/bufr/BufrParser/Query/QuerySet.cpp @@ -0,0 +1,88 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "QuerySet.h" + +#include +#include + +namespace Ingester { +namespace bufr { + + QuerySet::QuerySet(const std::vector& subsets) : + includesAllSubsets_(false), + limitSubsets_(std::set(subsets.begin(), + subsets.end())), + presentSubsets_({}) + { + } + + void QuerySet::add(const std::string& name, const std::string& queryStr) + { + std::vector queries; + for (const auto &query : QueryParser::parse(queryStr)) + { + if (limitSubsets_.empty()) + { + if (query.subset == "*") + { + includesAllSubsets_ = true; + } + + presentSubsets_.insert(query.subset); + } + else + { + if (query.subset == "*") + { + presentSubsets_ = limitSubsets_; + } + else + { + presentSubsets_.insert(query.subset); + + std::vector newSubsets; + std::set_intersection(limitSubsets_.begin(), + limitSubsets_.end(), + presentSubsets_.begin(), + presentSubsets_.end(), + std::back_inserter(newSubsets)); + + presentSubsets_ = std::set(newSubsets.begin(), + newSubsets.end()); + } + } + + queries.emplace_back(query); + } + + queryMap_[name] = queries; + } + + bool QuerySet::includesSubset(const std::string& subset) const + { + bool includesSubset = true; + if (!includesAllSubsets_) + { + includesSubset = (presentSubsets_.find(subset) != presentSubsets_.end()); + } + + return includesSubset; + } + + std::vector QuerySet::names() const + { + std::vector names; + for (auto const& query : queryMap_) + { + names.push_back(query.first); + } + + return names; + } +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/QuerySet.h b/src/bufr/BufrParser/Query/QuerySet.h new file mode 100644 index 000000000..54864418f --- /dev/null +++ b/src/bufr/BufrParser/Query/QuerySet.h @@ -0,0 +1,54 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include +#include + +#include "QueryParser.h" + +namespace Ingester { +namespace bufr +{ + typedef std::set Subsets; + + /// \brief Manages a collection of queries. + class QuerySet + { + public: + explicit QuerySet(const std::vector& subsets); + ~QuerySet() = default; + + /// \brief Add a new query to the collection. + /// \param[in] name The name of the query. + /// \param[in] query The query string. + void add(const std::string& name, const std::string& query); + + /// \brief Returns the size of the collection. + size_t size() const { return queryMap_.size(); } + + /// \brief Returns the names of all the queries. + /// \return A vector of the names of all the queries. + std::vector names() const; + + /// \brief Returns a list of subsets. + /// \return A vector of the names of all the queries. + bool includesSubset(const std::string& subset) const; + + std::vector queriesFor(const std::string& name) const { return queryMap_.at(name); } + + private: + std::unordered_map> queryMap_; + bool includesAllSubsets_; + Subsets limitSubsets_; + Subsets presentSubsets_; + }; +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/ResultSet.cpp b/src/bufr/BufrParser/Query/ResultSet.cpp new file mode 100644 index 000000000..3b975c10c --- /dev/null +++ b/src/bufr/BufrParser/Query/ResultSet.cpp @@ -0,0 +1,511 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "ResultSet.h" + +#include "eckit/exception/Exceptions.h" + +#include +#include +#include + +#include "Constants.h" +#include "VectorMath.h" + + +namespace Ingester { +namespace bufr { + ResultSet::ResultSet(const std::vector& names) : + names_(names) + { + fieldWidths.resize(names.size()); + } + + ResultSet::~ResultSet() + { + } + + std::shared_ptr + ResultSet::get(const std::string& fieldName, + const std::string& groupByFieldName, + const std::string& overrideType) const + { + std::vector data; + std::vector dims; + std::vector dimPaths; + TypeInfo info; + + getRawValues(fieldName, + groupByFieldName, + data, + dims, + dimPaths, + info); + + // Add dim path strings + const char* ws = " \t\n\r\f\v"; + std::vector paths(dims.size()); + for (size_t dimIdx = 0; dimIdx < dims.size(); dimIdx++) + { + auto path_str = dimPaths[dimIdx]; + + // Trim extra chars from the path str + path_str.erase(path_str.find_last_not_of(ws) + 1); + paths[dimIdx] = path_str; + } + + std::shared_ptr object = makeDataObject(fieldName, + groupByFieldName, + info, + overrideType, + data, + dims, + paths); + + return object; + } + + + DataFrame& ResultSet::nextDataFrame() + { + dataFrames_.push_back(DataFrame(names_.size())); + return dataFrames_.back(); + } + + void ResultSet::getRawValues(const std::string& fieldName, + const std::string& groupByField, + std::vector& data, + std::vector& dims, + std::vector& dimPaths, + TypeInfo& info) const + { + // Find the dims based on the largest sequence counts in the fields + + // Compute Dims + std::vector dimsList; + std::vector exportDims; + int groupbyIdx = 0; + int totalGroupbyElements = 0; + + int targetFieldIdx = 0; + int groupByFieldIdx = 0; + if (dataFrames_.size() > 0) + { + targetFieldIdx = dataFrames_[0].fieldIndexForNodeNamed(fieldName); + + if (groupByField != "") + { + groupByFieldIdx = dataFrames_[0].fieldIndexForNodeNamed(groupByField); + } + + auto& targetField = dataFrames_[0].fieldAtIdx(targetFieldIdx); + dimPaths = targetField.target->dimPaths; + + exportDims = targetField.target->exportDimIdxs; + } + + for (auto& dataFrame : dataFrames_) + { + auto& targetField = dataFrame.fieldAtIdx(targetFieldIdx); + if (!targetField.target->dimPaths.empty() && + dimPaths.size() < targetField.target->dimPaths.size()) + { + dimPaths = targetField.target->dimPaths; + exportDims = targetField.target->exportDimIdxs; + } + + size_t dimsLen = targetField.seqCounts.size(); + if (dimsList.size() < dimsLen) + { + dimsList.resize(dimsLen, 0); + } + + for (size_t cntIdx = 0; cntIdx < targetField.seqCounts.size(); ++cntIdx) + { + if (!targetField.seqCounts[cntIdx].empty()) + { + dimsList[cntIdx] = std::max(dimsList[cntIdx], + max(targetField.seqCounts[cntIdx])); + } + } + + info.reference = std::min(info.reference, targetField.target->typeInfo.reference); + info.bits = std::max(info.bits, targetField.target->typeInfo.bits); + + if (std::abs(targetField.target->typeInfo.scale) > info.scale) + { + info.scale = targetField.target->typeInfo.scale; + } + + if (info.unit.empty()) info.unit = targetField.target->typeInfo.unit; + + if (groupByField != "") + { + auto& groupByField = dataFrame.fieldAtIdx(groupByFieldIdx); + groupbyIdx = std::max(groupbyIdx, static_cast(groupByField.seqCounts.size())); + + if (groupbyIdx > static_cast(dimsList.size())) + { + dimPaths = {groupByField.target->dimPaths.back()}; + + int groupbyElementsForFrame = 1; + for (auto &seqCount : groupByField.seqCounts) + { + if (!seqCount.empty()) + { + groupbyElementsForFrame *= max(seqCount); + } + } + + totalGroupbyElements = std::max(totalGroupbyElements, groupbyElementsForFrame); + } + else + { + dimPaths = {}; + for (size_t targetIdx = groupByField.target->exportDimIdxs.size() - 1; + targetIdx < targetField.target->dimPaths.size(); + ++targetIdx) + { + dimPaths.push_back(targetField.target->dimPaths[targetIdx]); + } + } + } + } + + auto allDims = dimsList; + + // If there is absolutely no data for a field you will have the problem were the + // size of some dimensions are zero. We need to have at least 1 element in each + // dimension to make room for the missing value. This if statement makes sure there + // is at least 1 element in each dimension. + for (size_t dimIdx = 0; dimIdx < allDims.size(); ++dimIdx) + { + if (allDims[dimIdx] == 0) + { + allDims[dimIdx] = 1; + } + } + + if (groupbyIdx > 0) + { + // The groupby field occurs at the same or greater repetition level as the target field. + if (groupbyIdx > static_cast(dimsList.size())) + { + dims.resize(1, totalGroupbyElements); + exportDims = {0}; + allDims = dims; + } + // The groupby field occurs at a lower repetition level than the target field. + else + { + dims.resize(dimsList.size() - groupbyIdx + 1, 1); + for (auto dimIdx = 0; dimIdx < groupbyIdx; ++dimIdx) + { + dims[0] *= allDims[dimIdx]; + } + + for (size_t dimIdx = groupbyIdx; dimIdx < allDims.size(); ++dimIdx) + { + dims[dimIdx - groupbyIdx + 1] = allDims[dimIdx]; + } + + exportDims = exportDims - (groupbyIdx - 1); + + // Filter out exportDims that are 0 + std::vector filteredExportDims; + for (size_t dimIdx = 0; dimIdx < exportDims.size(); ++dimIdx) + { + if (exportDims[dimIdx] >= 0) + { + filteredExportDims.push_back(exportDims[dimIdx]); + } + } + + if (filteredExportDims.empty() || filteredExportDims[0] != 0) + { + filteredExportDims.insert(filteredExportDims.begin(), 0); + } + + exportDims = filteredExportDims; + } + } + else + { + dims = allDims; + } + + size_t totalRows = dims[0] * dataFrames_.size(); + + // Make data set + int rowLength = 1; + for (size_t dimIdx = 1; dimIdx < dims.size(); ++dimIdx) + { + rowLength *= dims[dimIdx]; + } + + data.resize(totalRows * rowLength, MissingValue); + for (size_t frameIdx = 0; frameIdx < dataFrames_.size(); ++frameIdx) + { + auto& dataFrame = dataFrames_[frameIdx]; + std::vector> frameData; + auto& targetField = dataFrame.fieldAtIdx(targetFieldIdx); + + if (!targetField.data.size() == 0) { + getRowsForField(targetField, + frameData, + allDims, + groupbyIdx); + + auto dataRowIdx = dims[0] * frameIdx; + for (size_t rowIdx = 0; rowIdx < frameData.size(); ++rowIdx) + { + auto &row = frameData[rowIdx]; + for (size_t colIdx = 0; colIdx < row.size(); ++colIdx) + { + data[dataRowIdx*rowLength + rowIdx * row.size() + colIdx] = row[colIdx]; + } + } + } + } + + // Convert dims per data frame to dims for all the collected data. + dims[0] = totalRows; + if (dataFrames_.size() > 1) + { + dims = slice(dims, exportDims); + } + } + +// subroutine result_set__get_rows_for_field(self, target_field, data_rows, dims, groupby_idx) + + void ResultSet::getRowsForField(const DataField& targetField, + std::vector>& dataRows, + const std::vector& dims, + int groupbyIdx) const + { + size_t maxCounts = 0; + std::vector idxs(targetField.data.size()); + for (size_t i = 0; i < idxs.size(); ++i) + { + idxs[i] = i; + } + + // Compute max counts + for (size_t i = 0; i < targetField.seqCounts.size(); ++i) + { + if (maxCounts < targetField.seqCounts[i].size()) + { + maxCounts = targetField.seqCounts[i].size(); + } + } + + // Compute insert array + std::vector> inserts(dims.size(), {0}); + for (size_t repIdx = 0; + repIdx < std::min(dims.size(), targetField.seqCounts.size()); + ++repIdx) + { + inserts[repIdx] = product(dims.begin() + repIdx, dims.end()) - + targetField.seqCounts[repIdx] * + product(dims.begin() + repIdx + 1, dims.end()); + } + + // Inflate the data, compute the idxs for each data element in the result array + for (int dim_idx = dims.size() - 1; dim_idx >= 0; --dim_idx) + { + for (size_t insert_idx = 0; insert_idx < inserts[dim_idx].size(); ++insert_idx) + { + size_t num_inserts = inserts[dim_idx][insert_idx]; + if (num_inserts > 0) + { + int data_idx = product(dims.begin() + dim_idx, dims.end()) * + insert_idx + product(dims.begin() + dim_idx, dims.end()) + - num_inserts - 1; + + for (size_t i = 0; i < idxs.size(); ++i) + { + if (static_cast(idxs[i]) > data_idx) + { + idxs[i] += num_inserts; + } + } + } + } + } + + auto output = std::vector(product(dims), MissingValue); + for (size_t i = 0; i < idxs.size(); ++i) + { + output[idxs[i]] = targetField.data[i]; + } + + // Apply groupBy and make output + if (groupbyIdx > 0) + { + if (groupbyIdx > static_cast(targetField.seqCounts.size())) + { + size_t numRows = product(dims); + dataRows.resize(numRows * maxCounts, {MissingValue}); + for (size_t i = 0; i < numRows; ++i) + { + if (output.size()) + { + dataRows[i][0] = output[0]; + } + } + } + else + { + size_t numRows = product(dims.begin(), dims.begin() + groupbyIdx); + std::vector rowDims; + rowDims.assign(dims.begin() + groupbyIdx, dims.end()); + + size_t numsPerRow = static_cast(product(rowDims)); + dataRows.resize(numRows, std::vector(numsPerRow, MissingValue)); + for (size_t i = 0; i < numRows; ++i) + { + for (size_t j = 0; j < numsPerRow; ++j) + { + dataRows[i][j] = output[i * numsPerRow + j]; + } + } + } + } + else + { + dataRows.resize(1); + dataRows[0] = output; + } + } + + std::string ResultSet::unit(const std::string& fieldName) const + { + auto fieldIdx = dataFrames_.front().fieldIndexForNodeNamed(fieldName); + return dataFrames_.front().fieldAtIdx(fieldIdx).target->unit; + } + + std::shared_ptr ResultSet::makeDataObject( + const std::string& fieldName, + const std::string& groupByFieldName, + TypeInfo& info, + const std::string& overrideType, + const std::vector data, + const std::vector dims, + const std::vector dimPaths) const + { + std::shared_ptr object; + if (overrideType.empty()) + { + object = objectByTypeInfo(info); + } + else + { + object = objectByType(overrideType); + + if ((overrideType == "string" && !info.isString()) || + (overrideType != "string" && info.isString())) + { + std::ostringstream errMsg; + errMsg << "Conversions between numbers and strings are not currently supported. "; + errMsg << "See the export definition for \"" << fieldName << "\"."; + throw eckit::BadParameter(errMsg.str()); + } + } + + object->setData(data, 10e10); + object->setDims(dims); + object->setFieldName(fieldName); + object->setGroupByFieldName(groupByFieldName); + object->setDimPaths(dimPaths); + + return object; + } + + std::shared_ptr ResultSet::objectByTypeInfo(TypeInfo &info) const + { + std::shared_ptr object; + + if (info.isString()) + { + object = std::make_shared>(); + } + else if (info.isInteger()) + { + if (info.isSigned()) + { + if (info.is64Bit()) + { + object = std::make_shared>(); + } + else + { + object = std::make_shared>(); + } + } + else + { + if (info.is64Bit()) + { + object = std::make_shared>(); + } + else + { + object = std::make_shared>(); + } + } + } + else + { + if (info.is64Bit()) + { + object = std::make_shared>(); + } + else + { + object = std::make_shared>(); + } + } + + return object; + } + + std::shared_ptr ResultSet::objectByType(const std::string& overrideType) const + { + std::shared_ptr object; + + if (overrideType == "int" || overrideType == "int32") + { + object = std::make_shared>(); + } + else if (overrideType == "float") + { + object = std::make_shared>(); + } + else if (overrideType == "double") + { + object = std::make_shared>(); + } + else if (overrideType == "string") + { + object = std::make_shared>(); + } + else if (overrideType == "int64") + { + object = std::make_shared>(); + } + else + { + std::ostringstream errMsg; + errMsg << "Unknown or unsupported type " << overrideType << "."; + throw eckit::BadParameter(errMsg.str()); + } + + return object; + } + +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/ResultSet.h b/src/bufr/BufrParser/Query/ResultSet.h new file mode 100644 index 000000000..2ce61192f --- /dev/null +++ b/src/bufr/BufrParser/Query/ResultSet.h @@ -0,0 +1,179 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "DataProvider.h" +#include "DataObject.h" +#include "Target.h" + + +namespace Ingester { +namespace bufr { + /// \brief Represents a single BUFR data element (a element from one message subset). It + /// contains both the data value(s) and the associated metadata that is used to construct the + /// results data. + struct DataField + { + std::shared_ptr target; + std::vector data; + std::vector> seqCounts; + }; + + /// \brief Container for a "row" of data (all the collected data for a message subset)., with a + /// DataField for each data element + class DataFrame + { + public: + explicit DataFrame(int fieldCnt) + { + fields_.resize(fieldCnt); + } + + /// \brief Get a reference for the const DataField at the given index. + /// \param idx The index of the data field to get. + inline const DataField& fieldAtIdx(size_t idx) const { return fields_[idx]; } + + /// \brief Get a reference for the DataField at the given index. + /// \param idx The index of the data field to get. + inline DataField& fieldAtIdx(size_t idx) { return fields_[idx]; } + + /// \brief Get the index for the field with the given name. This field idx is valid for all + /// data frames in the result set. + /// \param name The name of the field to get the index for. + int fieldIndexForNodeNamed(const std::string& name) const + { + auto result = -1; + for (size_t fieldIdx = 0; fieldIdx < fields_.size(); fieldIdx++) + { + if (fields_[fieldIdx].target->name == name) + { + result = fieldIdx; + break; + } + } + + return result; + } + + private: + std::vector fields_; + }; + + /// \brief This class acts as the container for all the data that is collected during the + /// the BUFR querying process. Internally it arranges the data as DataFrames for each message + /// subset observation. Each DataFrame contains a list of DataFields, one for each named element + /// that was collected. Because of the way the collection process works, each DataFrame is + /// organized the same way (indexes of DataFields line up). + /// + /// \par The getter functions for the data construct the final output based on the data and + /// metadata in these DataFields. There are many complications. For one the data may be jagged + /// (DataFields instances don't necessarily all have the same number of elements [repeated data + /// could have a different number of repeats per instance]). Another is the application group_by + /// fields which affect the dimensionality of the data. In order to make the data into + /// rectangular arrays it may be necessary to strategically fill in missing values so that the + /// data is organized correctly in each dimension. + /// + class ResultSet + { + public: + explicit ResultSet(const std::vector& names); + ~ResultSet(); + + /// \brief Gets the resulting data for a specific field with a given name grouped by the + /// optional groupByFieldName. + /// \param fieldName The name of the field to get the data for. + /// \param groupByFieldName The name of the field to group the data by. + /// \param overrideType The name of the override type to convert the data to. Possible + /// values are int, uint, int32, uint32, int64, uint64, float, double + /// \return A Result object containing the data. + std::shared_ptr + get(const std::string& fieldName, + const std::string& groupByFieldName = "", + const std::string& overrideType = "") const; + + /// \brief Adds a new DataFrame to the ResultSet and returns a reference to it. + /// \return A reference to the new DataFrame. + DataFrame& nextDataFrame(); + + void setTargets(Targets targets) { targets_ = targets; } + + private: + Targets targets_; + std::vector dataFrames_; + std::vector names_; + std::vector fieldWidths; + + /// \brief Computes the data for a specific field with a given name grouped by the + /// groupByField. It determines the required dimensions for the field and then uses + /// getRowsForField to get the relevant data from each DataFrame. + /// \param fieldName The name of the field to get the data for. + /// \param groupByFieldName The name of the field to group the data by. + /// \param dims The size of the dimensions of the result data (any number of dimensions). + /// \param dimPaths The dimensioning sub-query path strings. + /// \param info The meta data for the element. + void getRawValues(const std::string& fieldName, + const std::string& groupByField, + std::vector& data, + std::vector& dims, + std::vector& dimPaths, + TypeInfo& info) const; + + /// \brief Retrieves the data for the specified target field, one row per message subset. + /// The dims are used to determine the filling pattern so that that the resulting data can + /// be reshaped to the dimensions specified. + /// \param[in] targetField The target field to retrieve. + /// \param[out] dataRows The data. + /// \param[in] dims Vector of dimension sizes. + /// \param[in] groupbyIdx Idx of the group by field (which query component). + void getRowsForField(const DataField& targetField, + std::vector>& dataRows, + const std::vector& dims, + int groupbyIdx) const; + + /// \brief Is the field a string field? + /// \param fieldName The name of the field. + std::string unit(const std::string& fieldName) const; + + /// \brief Make an appropriate DataObject for the data considering all the META data + /// \param fieldName The name of the field to get the data for. + /// \param groupByFieldName The name of the field to group the data by. + /// \param info The meta data for the element. + /// \param overrideType The name of the override type to convert the data to. Possible + /// values are int, uint, int32, uint32, int64, uint64, float, double + /// \param data The data + /// \param dims The dimensioning information + /// \param dimPaths The sub-query path strings for each dimension. + /// \return A Result DataObject containing the data. + std::shared_ptr makeDataObject( + const std::string& fieldName, + const std::string& groupByFieldName, + TypeInfo& info, + const std::string& overrideType, + const std::vector data, + const std::vector dims, + const std::vector dimPaths) const; + + /// \brief Make an appropriate DataObject for data with the TypeInfo + /// \param info The meta data for the element. + /// \return A Result DataObject containing the data. + std::shared_ptr objectByTypeInfo(TypeInfo& info) const; + + /// \brief Make an appropriate DataObject for data with the override type + /// \param overrideType The meta data for the element. + /// \return A Result DataObject containing the data. + std::shared_ptr objectByType(const std::string& overrideType) const; + }; +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/SubsetTable.cpp b/src/bufr/BufrParser/Query/SubsetTable.cpp new file mode 100644 index 000000000..804ffed8f --- /dev/null +++ b/src/bufr/BufrParser/Query/SubsetTable.cpp @@ -0,0 +1,249 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "SubsetTable.h" + +#include +#include +#include +#include + + +namespace Ingester { +namespace bufr { + + SubsetTable::SubsetTable(const DataProvider& dataProvider) : + dataProvider_(dataProvider) + { + initialize(); + } + + std::vector SubsetTable::allQueryData() + { + std::vector queryData; + queryData.reserve(queryMap_.size()); + + for (auto queryKey : queryMapKeys_) + { + queryData.push_back(queryMap_[queryKey]); + } + + return queryData; + } + + + QueryData SubsetTable::dataForQuery(const std::vector& queryComponents) const + { + QueryData queryData; + auto key = mapKey(queryComponents); + if (queryMap_.find(key) != queryMap_.end()) + { + queryData = queryMap_.at(key); + } + else + { + queryData.isMissing = true; + queryData.nodeId = 0; + queryData.pathComponents = {}; + queryData.isString = false; + queryData.seqPath = {}; + queryData.dimIdxs = {}; + } + + return queryData; + } + + + void SubsetTable::initialize() + { + std::vector seqPath; + std::vector> currentPathElements = {{}}; + + std::vector> allQueries; + std::unordered_map> foundQueryMap; + + seqPath.push_back(dataProvider_.getInode()); + for (auto nodeIdx = dataProvider_.getInode(); + nodeIdx <= dataProvider_.getIsc(dataProvider_.getInode()); + nodeIdx++) + { + if (dataProvider_.getTyp(nodeIdx) == Typ::Repeat || + dataProvider_.getTyp(nodeIdx) == Typ::StackedRepeat) + { + seqPath.push_back(nodeIdx); + currentPathElements.push_back({}); + } + else if (dataProvider_.getTyp(nodeIdx) == Typ::DelayedBinary || + dataProvider_.getTyp(nodeIdx) == Typ::FixedRep) + { + seqPath.push_back(nodeIdx + 1); // push the node idx for the embedded sequence + currentPathElements.push_back({}); + } + else if (dataProvider_.getTyp(nodeIdx) == Typ::Number || + dataProvider_.getTyp(nodeIdx) == Typ::Character) + { + auto elementTag = dataProvider_.getTag(nodeIdx); + currentPathElements.back().push_back(elementTag); + + auto numElements = std::count(currentPathElements.back().begin(), + currentPathElements.back().end(), + elementTag); + + auto pathComponents = makePathComponents(seqPath, nodeIdx); + + if (numElements == 2) + { + // Update the matching quiry to require idxs. + auto& otherQuery = foundQueryMap[mapKey(pathComponents, 1)]; + otherQuery->requiresIdx = true; + } + + auto query = std::make_shared(); + query->nodeId = nodeIdx; + query->isMissing = false; + query->seqPath = seqPath; + query->pathComponents = pathComponents; + query->isString = (dataProvider_.getItp(nodeIdx) == 3); + query->dimIdxs = dimPathIdxs(seqPath); + query->idx = numElements; + query->requiresIdx = (numElements > 1); + query->typeInfo = dataProvider_.getTypeInfo(nodeIdx); + + allQueries.push_back(query); + foundQueryMap[mapKey(query->pathComponents, numElements)] = allQueries.back(); + } + + if (seqPath.size() > 1) + { + auto jumpBackNode = dataProvider_.getInode(); + if (nodeIdx < dataProvider_.getIsc(dataProvider_.getInode())) + { + // Skip pure sequences not inside any kind of repeated sequence + jumpBackNode = dataProvider_.getJmpb(nodeIdx + 1); + if (jumpBackNode == 0) jumpBackNode = dataProvider_.getInode(); + + while (dataProvider_.getTyp(jumpBackNode) == Typ::Sequence && + dataProvider_.getTyp(jumpBackNode - 1) != Typ::DelayedRep && + dataProvider_.getTyp(jumpBackNode - 1) != Typ::FixedRep && + dataProvider_.getTyp(jumpBackNode - 1) != Typ::DelayedRepStacked && + dataProvider_.getTyp(jumpBackNode - 1) != Typ::DelayedBinary) + { + auto newJumpBackNode = dataProvider_.getJmpb(jumpBackNode); + if (newJumpBackNode != jumpBackNode) + { + jumpBackNode = newJumpBackNode; + } + else + { + break; + } + } + } + + // Peak ahead to see if the next node is inside one of the containing sequences. + for (int pathIdx = seqPath.size() - 2; pathIdx >= 0; pathIdx--) + { + // Check if the node idx is the next node for the current path + // or if the parent node of the next node is the previous path index + + if (seqPath[pathIdx] == jumpBackNode) + { + auto numToRewind = seqPath.size() - pathIdx - 1; + for (size_t rewindIdx = 0; rewindIdx < numToRewind; rewindIdx++) + { + seqPath.pop_back(); + currentPathElements.pop_back(); + } + } + } + } + } + + for (const auto& query : allQueries) + { + auto queryStr = mapKey(query); + queryMapKeys_.push_back(queryStr); + queryMap_[queryStr] = *query; + } + } + + + std::vector SubsetTable::dimPathIdxs(std::vector seqPath) const + { + std::vector dimPathIdxs; + dimPathIdxs.push_back(0); + for (size_t idx = 1; idx < seqPath.size(); idx++) + { + if (dataProvider_.getTyp(seqPath[idx] - 1) == Typ::DelayedRep || + dataProvider_.getTyp(seqPath[idx] - 1) == Typ::FixedRep || + dataProvider_.getTyp(seqPath[idx] - 1) == Typ::DelayedRepStacked) + { + dimPathIdxs.push_back(idx); + } + } + + return dimPathIdxs; + } + + + std::vector SubsetTable::makePathComponents(std::vector seqPath, + int nodeIdx) + { + std::vector pathComps; + + pathComps.push_back(dataProvider_.getTag(seqPath[0])); + for (size_t idx = 1; idx < seqPath.size(); idx++) + { + if (dataProvider_.getTyp(seqPath[idx] - 1) == Typ::DelayedRep || + dataProvider_.getTyp(seqPath[idx] - 1) == Typ::FixedRep || + dataProvider_.getTyp(seqPath[idx] - 1) == Typ::DelayedRepStacked || + dataProvider_.getTyp(seqPath[idx] - 1) == Typ::DelayedBinary) + { + pathComps.push_back(dataProvider_.getTag(seqPath[idx])); + } + } + + pathComps.push_back(dataProvider_.getTag(nodeIdx)); + return pathComps; + } + + + std::string SubsetTable::mapKey(const std::vector& pathComponents, + size_t idx) const + { + std::ostringstream ostr; + for (size_t pathIdx = 1; pathIdx < pathComponents.size(); pathIdx++) + { + ostr << "/" << pathComponents[pathIdx]; + } + + if (idx > 0) + { + ostr << "[" << idx << "]"; + } + + return ostr.str(); + } + + std::string SubsetTable::mapKey(const std::shared_ptr query) const + { + std::string key; + if (query->requiresIdx) + { + key = mapKey(query->pathComponents, query->idx); + } + else + { + key = mapKey(query->pathComponents); + } + + return key; + } +} // namespace bufr +} // namespace Ingester + + diff --git a/src/bufr/BufrParser/Query/SubsetTable.h b/src/bufr/BufrParser/Query/SubsetTable.h new file mode 100644 index 000000000..0f56b22a2 --- /dev/null +++ b/src/bufr/BufrParser/Query/SubsetTable.h @@ -0,0 +1,63 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include +#include + +#include "DataProvider.h" + +namespace Ingester { +namespace bufr { + /// \brief Meta data for a query according to the BUFR subset table. + struct QueryData + { + bool isMissing; + int nodeId; + std::vector pathComponents; + bool isString; + std::vector seqPath; + std::vector dimIdxs; + size_t idx; + bool requiresIdx; + TypeInfo typeInfo; + }; + + /// \brief Parses the BUFR message subset Meta data tables. + class SubsetTable + { + public: + SubsetTable() = delete; + explicit SubsetTable(const DataProvider& dataProvider); + ~SubsetTable() = default; + + /// \brief Returns all the query data elements. + /// \returns BUFR table meta data for all the queries + std::vector allQueryData(); + + /// \brief Get meta data for the query with the given components. + QueryData dataForQuery(const std::vector& queryComponents) const; + + private: + const DataProvider& dataProvider_; + std::unordered_map queryMap_; + std::vector queryMapKeys_; + + void initialize(); + std::vector dimPathIdxs(std::vector seqPath) const; + std::vector makePathComponents(std::vector seqPath, int nodeIdx); + std::string mapKey(const std::vector& pathComponents, size_t idx = 0) const; + std::string mapKey(const std::shared_ptr query) const; + +// void parseSequence(std::shared_ptr parentNode, size_t& nodeIdx, size_t endIdx); + }; +} // namespace bufr +} // namespace Ingester + diff --git a/src/bufr/BufrParser/Query/Target.h b/src/bufr/BufrParser/Query/Target.h new file mode 100644 index 000000000..d2649ed65 --- /dev/null +++ b/src/bufr/BufrParser/Query/Target.h @@ -0,0 +1,34 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include + +#include "DataProvider.h" + +namespace Ingester { +namespace bufr { + /// \brief The information or Meta data for a BUFR field whose data we wish to capture when + /// we execute a query. + struct Target + { + std::string name; + std::string queryStr; + std::string unit; + std::vector seqPath; + std::vector nodeIds; + std::vector dimPaths; + std::vector exportDimIdxs; + TypeInfo typeInfo; + }; + + typedef std::vector> Targets; +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/BufrParser/Query/VectorMath.h b/src/bufr/BufrParser/Query/VectorMath.h new file mode 100644 index 000000000..922f0adb4 --- /dev/null +++ b/src/bufr/BufrParser/Query/VectorMath.h @@ -0,0 +1,141 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + + +#pragma once + +#include +#include + +namespace Ingester { +namespace bufr { + + /// \brief Multiply all the values in a vector together. + /// \return Scalar value. + template + T product(const std::vector &vec) + { + return std::accumulate(vec.begin(), vec.end(), 1, std::multiplies()); + } + + /// \brief Multiply a range of values in a vector together. + /// \param begin The beginning of the range. + /// \param end The end of the range. + /// \return Scalar value. + template + T product(typename std::vector::const_iterator begin, + typename std::vector::const_iterator end) + { + T result = 1; + for (auto i = begin; i < end; ++i) + { + result *= *i; + } + + return result; + } + + /// \brief Slice a vector into a range of values. + /// \param begin The beginning of the range. + /// \param end The end of the range. + /// \return Sliced vector. + template + std::vector slice(typename std::vector::const_iterator begin, + typename std::vector::const_iterator end) + { + std::vector result(end - begin); + for (auto i = begin; i < end; ++i) + { + result[i] = *i; + } + + return result; + } + + /// \brief Slice a vector according to a list of indices. + /// \param vec The vector to slice. + /// \param indices The indices to slice the vector. + /// \return Sliced vector. + template + std::vector slice(const std::vector& vec, const std::vector& indices) + { + std::vector result(indices.size()); + for (size_t i = 0; i < indices.size(); ++i) + { + result[i] = vec[indices[i]]; + } + + return result; + } + + /// \brief Multiple a vector by a scalar. + /// \param vec The vector to multiply. + /// \param scalar The scalar to multiply by. + /// \return Multiplied vector. + template + std::vector operator*(const std::vector& vec, const U& scalar) + { + std::vector result(vec.size(), 1); + for (size_t i = 0; i < vec.size(); i++) + { + result[i] = vec[i] * static_cast(scalar); + } + + return result; + } + + /// \brief Subtract a vector from a scalar. + /// \param scalar The scalar to subtract from. + /// \param vec The vector to subtract. + /// \return Result vector. + template + std::vector operator-(const U& scalar, const std::vector& vec) + { + std::vector result(vec.size()); + for (size_t i = 0; i < vec.size(); i++) + { + result[i] = scalar - vec[i]; + } + + return result; + } + + /// \brief Subtract a scalar from a vector. + /// \param vec The vector to subtract from. + /// \param scalar The scalar to subtract. + /// \return Result vector. + template + std::vector operator-(const std::vector& vec, const U& scalar) + { + std::vector result(vec.size()); + for (size_t i = 0; i < vec.size(); i++) + { + result[i] = vec[i] - static_cast(scalar); + } + + return result; + } + + /// \brief Max value in a vector. + /// \param vec The vector to find the max value in. + /// \return Max value. + template + T max(const std::vector& vec) + { + T result = vec[0]; + for (size_t i = 1; i < vec.size(); i++) + { + if (vec[i] > result) + { + result = vec[i]; + } + } + + return result; + } +} // namespace bufr +} // namespace Ingester diff --git a/src/bufr/CMakeLists.txt b/src/bufr/CMakeLists.txt index 8fdeec6a0..6af627aaf 100644 --- a/src/bufr/CMakeLists.txt +++ b/src/bufr/CMakeLists.txt @@ -9,44 +9,28 @@ list(APPEND _ingester_srcs DataContainer.cpp Parser.h ParserFactory.h - DataObject/DataObject.h - DataObject/StrVecDataObject.h - DataObject/StrVecDataObject.cpp - DataObject/ArrayDataObject.h - DataObject/ArrayDataObject.cpp - DataObject/Int64VecDataObject.h - DataObject/Int64VecDataObject.cpp + DataObject.h + DataObject.cpp BufrParser/BufrParser.h BufrParser/BufrParser.cpp BufrParser/BufrDescription.h BufrParser/BufrDescription.cpp - BufrParser/BufrMnemonicSet.h - BufrParser/BufrMnemonicSet.cpp - BufrParser/BufrTypes.h - BufrParser/BufrCollectors/BufrAccumulator.cpp - BufrParser/BufrCollectors/BufrAccumulator.h - BufrParser/BufrCollectors/BufrCollector.cpp - BufrParser/BufrCollectors/BufrCollector.h - BufrParser/BufrCollectors/BufrIntCollector.cpp - BufrParser/BufrCollectors/BufrIntCollector.h - BufrParser/BufrCollectors/BufrRepCollector.cpp - BufrParser/BufrCollectors/BufrRepCollector.h - BufrParser/BufrCollectors/BufrCollectors.cpp - BufrParser/BufrCollectors/BufrCollectors.h BufrParser/Exports/Export.h BufrParser/Exports/Export.cpp - BufrParser/Exports/RowSlice.h BufrParser/Exports/Filters/Filter.h BufrParser/Exports/Filters/BoundingFilter.h BufrParser/Exports/Filters/BoundingFilter.cpp BufrParser/Exports/Splits/Split.h + BufrParser/Exports/Splits/Split.cpp BufrParser/Exports/Splits/CategorySplit.h BufrParser/Exports/Splits/CategorySplit.cpp BufrParser/Exports/Variables/Variable.h BufrParser/Exports/Variables/DatetimeVariable.h BufrParser/Exports/Variables/DatetimeVariable.cpp - BufrParser/Exports/Variables/MnemonicVariable.h - BufrParser/Exports/Variables/MnemonicVariable.cpp + BufrParser/Exports/Variables/TimeoffsetVariable.h + BufrParser/Exports/Variables/TimeoffsetVariable.cpp + BufrParser/Exports/Variables/QueryVariable.h + BufrParser/Exports/Variables/QueryVariable.cpp BufrParser/Exports/Variables/Transforms/Transform.h BufrParser/Exports/Variables/Transforms/OffsetTransform.h BufrParser/Exports/Variables/Transforms/OffsetTransform.cpp @@ -54,6 +38,20 @@ list(APPEND _ingester_srcs BufrParser/Exports/Variables/Transforms/ScalingTransform.cpp BufrParser/Exports/Variables/Transforms/TransformBuilder.h BufrParser/Exports/Variables/Transforms/TransformBuilder.cpp + BufrParser/Query/DataProvider.h + BufrParser/Query/DataProvider.cpp + BufrParser/Query/File.h + BufrParser/Query/File.cpp + BufrParser/Query/VectorMath.h + BufrParser/Query/QuerySet.h + BufrParser/Query/QuerySet.cpp + BufrParser/Query/QueryRunner.h + BufrParser/Query/QueryRunner.cpp + BufrParser/Query/QueryParser.h + BufrParser/Query/QueryParser.cpp + BufrParser/Query/ResultSet.h + BufrParser/Query/ResultSet.cpp + BufrParser/Query/Target.h IodaEncoder/IodaEncoder.cpp IodaEncoder/IodaEncoder.h IodaEncoder/IodaDescription.cpp @@ -65,7 +63,7 @@ list(APPEND _ingester_deps eckit ${oops_LIBRARIES} ioda_engines - bufr::bufr_4 + bufr::bufr_d ) ecbuild_add_library( TARGET ingester @@ -82,7 +80,7 @@ target_include_directories(ingester PUBLIC $ # /bufr ) -add_library( ${PROJECT_NAME}::ingester ALIAS ingester ) +add_library( ${PROJECT_NAME}::ingester ALIAS ingester) ecbuild_add_executable( TARGET bufr2ioda.x SOURCES bufr2ioda.cpp diff --git a/src/bufr/DataContainer.cpp b/src/bufr/DataContainer.cpp index e385c41b1..35b19972d 100644 --- a/src/bufr/DataContainer.cpp +++ b/src/bufr/DataContainer.cpp @@ -6,11 +6,9 @@ */ -#include #include #include -#include "Eigen/Dense" #include "eckit/exception/Exceptions.h" #include "DataContainer.h" @@ -31,7 +29,7 @@ namespace Ingester } void DataContainer::add(const std::string& fieldName, - const std::shared_ptr data, + const std::shared_ptr data, const SubCategory& categoryId) { if (hasKey(fieldName, categoryId)) @@ -45,8 +43,8 @@ namespace Ingester dataSets_.at(categoryId).insert({fieldName, data}); } - std::shared_ptr DataContainer::get(const std::string& fieldName, - const SubCategory& categoryId) const + std::shared_ptr DataContainer::get(const std::string& fieldName, + const SubCategory& categoryId) const { if (!hasKey(fieldName, categoryId)) { @@ -61,6 +59,39 @@ namespace Ingester return dataSets_.at(categoryId).at(fieldName); } + std::shared_ptr DataContainer::getGroupByObject( + const std::string& fieldName, + const SubCategory& categoryId) const + { + if (!hasKey(fieldName, categoryId)) + { + std::ostringstream errStr; + errStr << "ERROR: Either field called " << fieldName; + errStr << " or category " << makeSubCategoryStr(categoryId); + errStr << " does not exist."; + + throw eckit::BadParameter(errStr.str()); + } + + auto& dataObject = dataSets_.at(categoryId).at(fieldName); + const auto& groupByFieldName = dataObject->getGroupByFieldName(); + + std::shared_ptr groupByObject = dataObject; + if (!groupByFieldName.empty()) + { + for (const auto &obj : dataSets_.at(categoryId)) + { + if (obj.second->getFieldName() == groupByFieldName) + { + groupByObject = obj.second; + break; + } + } + } + + return groupByObject; + } + bool DataContainer::hasKey(const std::string& fieldName, const SubCategory& categoryId) const { @@ -85,7 +116,7 @@ namespace Ingester throw eckit::BadParameter(errStr.str()); } - return dataSets_.at(categoryId).begin()->second->nrows(); + return dataSets_.at(categoryId).begin()->second->getDims().at(0); } std::vector DataContainer::allSubCategories() const diff --git a/src/bufr/DataContainer.h b/src/bufr/DataContainer.h index 9b2447c37..7150c8106 100644 --- a/src/bufr/DataContainer.h +++ b/src/bufr/DataContainer.h @@ -7,14 +7,12 @@ #pragma once - #include #include #include #include -#include "Eigen/Dense" -#include "DataObject/DataObject.h" +#include "DataObject.h" #include "IngesterTypes.h" @@ -27,7 +25,7 @@ namespace Ingester typedef std::map CategoryMap; /// Map string paths (ex: variable/radiance) to DataObject - typedef std::map> DataSetMap; + typedef std::map> DataSetMap; /// Map category combo (ex: SatId/sat_1, GeoBox/lat_25_30__lon_23_26) to the relevant DataSetMap typedef std::map, DataSetMap> DataSets; @@ -53,14 +51,20 @@ namespace Ingester /// \param data The DataObject to store /// \param categoryId The vector for the subcategory void add(const std::string& fieldName, - std::shared_ptr data, + std::shared_ptr data, const SubCategory& categoryId = {}); /// \brief Get a DataObject from the collection /// \param fieldName The name of the data object ot get /// \param categoryId The vector for the subcategory - std::shared_ptr get(const std::string& fieldName, - const SubCategory& categoryId = {}) const; + std::shared_ptr get(const std::string& fieldName, + const SubCategory& categoryId = {}) const; + + /// \brief Get a DataObject for the group by field + /// \param fieldName The name of the data object ot get + /// \param categoryId The vector for the subcategory + std::shared_ptr getGroupByObject(const std::string& fieldName, + const SubCategory& categoryId = {}) const; /// \brief Check if DataObject with name is available /// \param fieldName The name of the object diff --git a/src/bufr/DataObject.cpp b/src/bufr/DataObject.cpp new file mode 100644 index 000000000..9c34ba1fb --- /dev/null +++ b/src/bufr/DataObject.cpp @@ -0,0 +1,17 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "DataObject.h" + +#include + +#include "eckit/exception/Exceptions.h" + + +namespace Ingester +{ +} // namespace Ingester diff --git a/src/bufr/DataObject.h b/src/bufr/DataObject.h new file mode 100644 index 000000000..6c575bb75 --- /dev/null +++ b/src/bufr/DataObject.h @@ -0,0 +1,670 @@ +/* + * (C) Copyright 2022 NOAA/NWS/NCEP/EMC + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "eckit/exception/Exceptions.h" +#include "ioda/ObsGroup.h" +#include "ioda/defs.h" + +#include "BufrParser/Query/Constants.h" + +namespace Ingester +{ + typedef std::vector Dimensions; + typedef Dimensions Location; + + struct DimensionDataBase + { + std::shared_ptr dimScale; + + virtual void write(ioda::Variable& var) = 0; + }; + + template + struct DimensionData : public DimensionDataBase + { + std::vector data; + + DimensionData() = delete; + + explicit DimensionData(size_t size) : + data(std::vector(size, _default())) + { + } + + void write(ioda::Variable& var) + { + var.write(data); + } + + private: + template + T _default(typename std::enable_if::value, U>::type* = nullptr) + { + return static_cast(0); + } + + template + T _default(typename std::enable_if::value, U>::type* = nullptr) + { + return std::string(""); + } + }; + + /// \brief Abstract base class for intermediate data object that bridges the Parsers with the + /// IodaEncoder. + class DataObjectBase + { + public: + explicit DataObjectBase(const std::string& fieldName, + const std::string& groupByFieldName, + const Dimensions& dims, + const std::string& query, + const std::vector& dimPaths) : + + fieldName_(fieldName), + groupByFieldName_(groupByFieldName), + dims_(dims), + query_(query), + dimPaths_(dimPaths) + {}; + + DataObjectBase() = default; + virtual ~DataObjectBase() = default; + + // Setters + void setFieldName(const std::string& fieldName) { fieldName_ = fieldName; } + void setGroupByFieldName(const std::string& fieldName) { groupByFieldName_ = fieldName; } + void setDims(const std::vector dims) { dims_ = dims; } + void setQuery(const std::string& query) { query_ = query; } + void setDimPaths(const std::vector& dimPaths) { dimPaths_ = dimPaths; } + virtual void setData(const std::vector& data, double dataMissingValue) = 0; + + // Getters + std::string getFieldName() const { return fieldName_; } + std::string getGroupByFieldName() const { return groupByFieldName_; } + Dimensions getDims() const { return dims_; } + std::string getPath() const { return query_; } + std::vector getDimPaths() const { return dimPaths_; } + + /// \brief Print the data object to a output stream. + virtual void print(std::ostream &out) const = 0; + + /// \brief Get the data at the location as an integer. + /// \return Integer data. + virtual int getAsInt(const Location& loc) const = 0; + + /// \brief Get the data at the location as an float. + /// \return Float data. + virtual float getAsFloat(const Location& loc) const = 0; + + /// \brief Get the data at the index as an int. + /// \return Int data. + virtual int getAsInt(size_t idx) const = 0; + + /// \brief Get the data at the index as an float. + /// \return Float data. + virtual float getAsFloat(size_t idx) const = 0; + + /// \brief Is the element at the index the missing value. + /// \return bool data. + virtual bool isMissing(size_t idx) const = 0; + + /// \brief Get the data at the Location as an string. + /// \return String data. + virtual std::string getAsString(const Location& loc) const = 0; + + /// \brief Get the size of the data + /// \return Data size. + virtual size_t size() const = 0; + + /// \brief Makes an ioda::Variable and adds it to the given ioda::ObsGroup + /// \param obsGroup Obsgroup where to add the variable + /// \param name The name to associate with the variable (ex "latitude@MetaData") + /// \param dimensions List of Variables to use as the dimensions for this new variable + /// \param chunks List of integers specifying the chunking dimensions + /// \param compressionLevel The GZip compression level to use, must be 0-9 + virtual ioda::Variable createVariable( + ioda::ObsGroup& obsGroup, + const std::string& name, + const std::vector& dimensions, + const std::vector& chunks, + int compressionLevel) const = 0; + + /// \brief Makes a new dimension scale using this data object as the source + /// \param name The name of the dimension variable. + /// \param dimIdx The idx of the data dimension to use. + virtual std::shared_ptr createDimensionFromData( + const std::string& name, + std::size_t dimIdx) const = 0; + + /// \brief Makes a new blank dimension scale with default type. + /// \param name The name of the dimension variable. + /// \param dimIdx The idx of the data dimension to use. + virtual std::shared_ptr createEmptyDimension( + const std::string& name, + std::size_t dimIdx) const = 0; + + /// \brief Slice the data object given a vector of row indices. + /// \param slice The indices to slice. + /// \return Slice of the data object. + virtual std::shared_ptr + slice(const std::vector& rows) const = 0; + + /// \brief Multiply the stored values in this data object by a scalar. + /// \param val Scalar to multiply to the data.. + virtual void multiplyBy(double val) = 0; + + /// \brief Add a scalar to the stored values in this data object. + /// \param val Scalar to add to the data.. + virtual void offsetBy(double val) = 0; + + protected: + std::string fieldName_; + std::string groupByFieldName_; + Dimensions dims_; + std::string query_; + std::vector dimPaths_; + }; + + + template + class DataObject : public DataObjectBase + { + public: + typedef T value_type; + static constexpr T missingValue() { return std::numeric_limits::max(); } + + /// \brief Constructor. + /// \param dimensions The dimensions of the data object. + DataObject() = default; + + DataObject(const std::vector& data, + const std::string& field_name, + const std::string& group_by_field_name, + const Dimensions& dimensions, + const std::string& query, + const std::vector& dimPaths) : + DataObjectBase(field_name, group_by_field_name, dimensions, query, dimPaths), + data_(data) + {}; + + ~DataObject() = default; + + /// \brief Set the data for this object + /// \param data The data vector + void setData(const std::vector& data) { data_ = data; } + + /// \brief Set the data for this object + /// \param data The data vector + /// \param dataMissingValue The missing value used in the raw data + void setData(const std::vector& data, double dataMissingValue) final + { + _setData(data, dataMissingValue); + } + + /// \brief Makes an ioda::Variable and adds it to the given ioda::ObsGroup + /// \param obsGroup Obsgroup were to add the variable + /// \param name The name to associate with the variable (ex "latitude@MetaData") + /// \param dimensions List of Variables to use as the dimensions for this new variable + /// \param chunks List of integers specifying the chunking dimensions + /// \param compressionLevel The GZip compression level to use, must be 0-9 + ioda::Variable createVariable(ioda::ObsGroup& obsGroup, + const std::string& name, + const std::vector& dimensions, + const std::vector& chunks, + int compressionLevel) const final + { + auto params = makeCreationParams(chunks, compressionLevel); + auto var = obsGroup.vars.createWithScales(name, dimensions, params); + var.write(data_); + return var; + }; + + /// \brief Makes a new dimension scale using this data object as the source + /// \param name The name of the dimension variable. + /// \param dimIdx The idx of the data dimension to use. + std::shared_ptr createDimensionFromData(const std::string& name, + std::size_t dimIdx) const final + { + auto dimData = std::make_shared>(getDims()[dimIdx]); + dimData->dimScale = ioda::NewDimensionScale(name, getDims()[dimIdx]); + + std::copy(data_.begin(), + data_.begin() + dimData->data.size(), + dimData->data.begin()); + + // Validate this data object is a valid (has values that repeat for each frame + for (size_t idx = 0; idx < data_.size(); idx += dimData->data.size()) + { + if (!std::equal(data_.begin(), + data_.begin() + dimData->data.size(), + data_.begin() + idx, + data_.begin() + idx + dimData->data.size())) + { + std::stringstream errStr; + errStr << "Dimension " << name << " has an invalid source field. "; + errStr << "The values dont repeat in each sequence."; + throw eckit::BadParameter(errStr.str()); + } + } + + return dimData; + } + + /// \brief Makes a new blank dimension scale with default type. + /// \param name The name of the dimension variable. + /// \param dimIdx The idx of the data dimension to use. + std::shared_ptr createEmptyDimension(const std::string& name, + std::size_t dimIdx) const final + { + auto dimData = std::make_shared>(getDims()[dimIdx]); + dimData->dimScale = ioda::NewDimensionScale(name, getDims()[dimIdx]); + return dimData; + } + + /// \brief Print the data object to a output stream. + void print(std::ostream &out) const final + { + out << "DataObject " << fieldName_ << ":"; + for (auto val = data_.cbegin(); val != data_.cend(); ++val) + { + if (val != data_.cbegin()) out << ", "; + out << *val; + } + + out << std::endl; + }; + + /// \brief Get the raw data. + std::vector getRawData() const { return data_; } + + /// \brief Set the raw data. + void setRawData(std::vector data) { data_ = data; } + + /// \brief Get data associated with a given location. + /// \param location The location to get data for. + /// \return The data at the given location. + T get(const Location& loc) const + { + size_t dim_prod = 1; + for (int dim_idx = dims_.size(); dim_idx > static_cast(loc.size()); --dim_idx) + { + dim_prod *= dims_[dim_idx]; + } + + // Compute the index into the data array + size_t index = 0; + for (int dim_idx = loc.size() - 1; dim_idx >= 0; --dim_idx) + { + index += dim_prod*loc[dim_idx]; + dim_prod *= dims_[dim_idx]; + } + + return data_[index]; + }; + + /// \brief Get the size of the data. + /// \return The size of the data. + size_t size() const { return data_.size(); } + + /// \brief Get the data at the location as an integer. + /// \param loc The coordinate for the data point (ex: if data 2d then loc {2,4} gets data + /// at that coordinate). + /// \return Int data. + int getAsInt(const Location& loc) const final { return _getAsInt(loc); } + + /// \brief Get the data at the location as a float. + /// \param loc The coordinate for the data point (ex: if data 2d then loc {2,4} gets data + /// at that coordinate). + /// \return Float data. + float getAsFloat(const Location& loc) const final { return _getAsFloat(loc); } + + /// \brief Get the data at the location as a string. + /// \param loc The coordinate for the data point (ex: if data 2d then loc {2,4} gets data + /// at that coordinate). + /// \return String data. + std::string getAsString(const Location& loc) const final { return _getAsString(loc); } + + + /// \brief Get the data at the index into the internal 1d array as a int. This function + /// gives you direct access to the internal data and doesn't account for dimensional + /// information (its up to the user). + /// \param idx The idx into the internal 1d array. + /// \return Int data. + int getAsInt(size_t idx) const final { return _getAsInt(idx); } + + + /// \brief idx Get the data at the index into the internal 1d array as a float. This + /// function gives you direct access to the internal data and doesn't account for + /// dimensional information (its up to the user). + /// \param idx The idx into the internal 1d array. + /// \return Float data. + float getAsFloat(const size_t idx) const final { return _getAsFloat(idx); } + + + /// \brief idx See if the data at the index into the internal 1d array is missing. This + /// function gives you direct access to the internal data and doesn't account for + /// dimensional information (its up to the user). + /// \param idx The idx into the internal 1d array. + /// \return bool data. + bool isMissing(const size_t idx) const final + { + return data_[idx] == missingValue(); + } + + + /// \brief Slice the dta object according to a list of indices. + /// \param rows The indices to slice the data object by. + /// \return Sliced DataObject. + std::shared_ptr slice(const std::vector& rows) const final + { + // Compute product of extra dimensions) + std::size_t extraDims = 1; + for (std::size_t i = 1; i < dims_.size(); ++i) + { + extraDims *= dims_[i]; + } + + // Make new DataObject with the rows we want + std::vector newData; + newData.reserve(rows.size() * extraDims); + for (std::size_t i = 0; i < rows.size(); ++i) + { + newData.insert(newData.end(), + data_.begin() + rows[i] * extraDims, + data_.begin() + (rows[i] + 1) * extraDims); + } + + auto sliceDims = dims_; + sliceDims[0] = rows.size(); + + return std::make_shared>(newData, + fieldName_, + groupByFieldName_, + sliceDims, + query_, + dimPaths_); + } + + private: + std::vector data_; + + /// \brief Make the variable creation parameters. + /// \param chunks The chunk sizes + /// \param compressionLevel The compression level + /// \return The variable creation patterns. + ioda::VariableCreationParameters makeCreationParams( + const std::vector& chunks, + int compressionLevel) const + { + return _makeCreationParams(chunks, compressionLevel); + } + + + /// \brief Make the variable creation parameters for numeric data. + /// \param chunks The chunk sizes + /// \param compressionLevel The compression level + /// \return The variable creation patterns. + template + ioda::VariableCreationParameters _makeCreationParams( + const std::vector& chunks, + int compressionLevel, + typename std::enable_if::value, U>::type* = nullptr) const + { + ioda::VariableCreationParameters params; + params.chunk = true; + params.chunks = chunks; + params.compressWithGZIP(compressionLevel); + params.setFillValue(static_cast(missingValue())); + + return params; + } + + /// \brief Make the variable creation parameters for string data. + /// \param chunks The chunk sizes + /// \param compressionLevel The compression level + /// \return The variable creation patterns. + template + ioda::VariableCreationParameters _makeCreationParams( + const std::vector& chunks, + int compressionLevel, + typename std::enable_if::value, U>::type* = nullptr) const + { + ioda::VariableCreationParameters params; + params.chunk = true; + params.chunks = chunks; + params.compressWithGZIP(compressionLevel); + params.setFillValue(static_cast(std::string(""))); + + return params; + } + + /// \brief Get the data at the location as a float for numeric data. + /// \return Float data. + template + float _getAsFloat(const Location& loc, + typename std::enable_if::value, U>::type* = nullptr) const + { + return static_cast (get(loc)); + } + + /// \brief Get the data at the location as a float for string data. + /// \return Float data. + template + float _getAsFloat(const Location& loc, + typename std::enable_if::value, U>::type* = nullptr) const + { + return std::stof(get(loc)); + } + + /// \brief Get the data at the location as int for numeric data. + /// \return Int data. + template + int _getAsInt(const Location& loc, + typename std::enable_if::value, U>::type* = nullptr) const + { + return static_cast (get(loc)); + } + + /// \brief Get the data at the location as int for string data. + /// \return Int data. + template + int _getAsInt(const Location& loc, + typename std::enable_if::value, U>::type* = nullptr) const + { + return std::stoi(get(loc)); + } + + /// \brief Get the data at the location as string for numeric data. + /// \return string data. + template + std::string _getAsString(const Location& loc, + typename std::enable_if::value, U>::type* = nullptr) const + { + return std::to_string(get(loc)); + } + + /// \brief Get the data at the location as string for string data. + /// \return string data. + template + std::string _getAsString(const Location& loc, + typename std::enable_if::value, U>::type* = nullptr) const + { + return get(loc); + } + + /// \brief Get the data at the index as a int for numeric data. + /// \return Int data. + template + int _getAsInt(size_t idx, + typename std::enable_if::value, U>::type* = nullptr) const + { + return static_cast(data_[idx]); + } + + /// \brief Get the data at the index as a int for non-numeric data. + /// \return Int data. + template + int _getAsInt(size_t idx, + typename std::enable_if::value, U>::type* = nullptr) const + { + throw std::runtime_error("The stored value is not a number"); + } + + /// \brief Get the data at the index as a float for numeric data. + /// \return Float data. + template + float _getAsFloat(size_t idx, + typename std::enable_if::value, U>::type* = nullptr) const + { + return static_cast(data_[idx]); + } + + /// \brief Get the data at the index as a float for non-numeric data. + /// \return Float data. + template + float _getAsFloat(size_t idx, + typename std::enable_if::value, U>::type* = nullptr) const + { + throw std::runtime_error("The stored value was is not a number"); + return 0.0f; + } + + /// \brief Set the data associated with this data object (numeric DataObject). + /// \param data - double vector of raw data + /// \param dataMissingValue - The number that represents missing values within the raw data + template + void _setData(const std::vector& data, + double dataMissingValue, + typename std::enable_if::value, U>::type* = nullptr) + { + data_ = std::vector(data.begin(), data.end()); + std::replace(data_.begin(), + data_.end(), + static_cast(dataMissingValue), + missingValue()); + } + + /// \brief Set the data associated with this data object (string DataObject). + /// \param data - double vector of raw data + /// \param dataMissingValue - The number that represents missing values within the raw data + template + void _setData( + const std::vector& data, + double dataMissingValue, + typename std::enable_if::value, U>::type* = nullptr) + { + data_ = std::vector(); + auto charPtr = reinterpret_cast(data.data()); + for (size_t row_idx = 0; row_idx < data.size(); row_idx++) + { + if (data[row_idx] != dataMissingValue) + { + std::string str = std::string( + charPtr + row_idx * sizeof(double), sizeof(double)); + + // trim trailing whitespace from str + str.erase(std::find_if(str.rbegin(), str.rend(), + [](char c){ return !std::isspace(c); }).base(), + str.end()); + + data_.push_back(str); + } + else + { + data_.push_back(""); + } + } + } + + /// \brief Multiply the stored values in this data object by a scalar. + /// \param val Scalar to multiply to the data.. + void multiplyBy(double val) final + { + _multiplyBy(val); + } + + /// \brief Multiply the stored values in this data object by a scalar (numeric version). + /// \param val Scalar to multiply to the data. + template + void _multiplyBy(double val, + typename std::enable_if::value, U>::type* = nullptr) + { + if (typeid(T) == typeid(float) || // NOLINT + typeid(T) == typeid(double) || // NOLINT + trunc(val) == val) + { + for (size_t i = 0; i < data_.size(); i++) + { + if (data_[i] != missingValue()) + { + data_[i] = static_cast(static_cast(data_[i]) * val); + } + } + } + else + { + std::ostringstream str; + str << "Multiplying integer field \"" << fieldName_ << "\" with a non-integer is "; + str << "illegal. Please convert it to a float or double."; + throw std::runtime_error(str.str()); + } + } + + /// \brief Multiply the stored values in this data object by a scalar (string version). + /// \param val Scalar to multiply to the data. + template + void _multiplyBy( + double val, + typename std::enable_if::value, U>::type* = nullptr) + { + throw std::runtime_error("Trying to multiply a string by a number"); + } + + /// \brief Add a scalar to the stored values in this data object. + /// \param val Scalar to add to the data. + void offsetBy(double val) final + { + _offsetBy(val); + } + + + /// \brief Add a scalar to the stored values in this data object (numeric version). + /// \param val Scalar to add to the data. + template + void _offsetBy(double val, + typename std::enable_if::value, U>::type* = nullptr) + { + for (size_t i = 0; i < data_.size(); i++) + { + if (data_[i] != missingValue()) + { + data_[i] = data_[i] + static_cast(val); + } + } + } + + /// \brief Add a scalar to the stored values in this data object (string version). + /// \param val Scalar to add to the data. + template + void _offsetBy( + double val, + typename std::enable_if::value, U>::type* = nullptr) + { + throw std::runtime_error("Trying to offset a string by a number"); + } + }; +} // namespace Ingester diff --git a/src/bufr/DataObject/ArrayDataObject.cpp b/src/bufr/DataObject/ArrayDataObject.cpp deleted file mode 100644 index 7dda15d83..000000000 --- a/src/bufr/DataObject/ArrayDataObject.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include - -#include "ArrayDataObject.h" - -namespace Ingester -{ - ArrayDataObject::ArrayDataObject(const IngesterArray& eigArray) : - eigArray_(eigArray) - { - } - - ioda::Variable ArrayDataObject::createVariable(ioda::ObsGroup& obsGroup, - const std::string& name, - const std::vector& dimensions, - const std::vector& chunks, - int compressionLevel) - { - auto params = makeCreationParams(chunks, compressionLevel); - auto var = obsGroup.vars.createWithScales(name, dimensions, params); - var.writeWithEigenRegular(eigArray_); - return var; - } - - void ArrayDataObject::print() const - { - std::cout << eigArray_ << std::endl; - } - - size_t ArrayDataObject::nrows() const - { - return eigArray_.rows(); - } - - size_t ArrayDataObject::ncols() const - { - return eigArray_.cols(); - } - - ioda::VariableCreationParameters ArrayDataObject::makeCreationParams( - const std::vector& chunks, - int compressionLevel) - { - ioda::VariableCreationParameters params; - params.chunk = true; - params.chunks = chunks; - params.compressWithGZIP(compressionLevel); - params.setFillValue(1.0e+11); - - return params; - } - -} // namespace Ingester diff --git a/src/bufr/DataObject/ArrayDataObject.h b/src/bufr/DataObject/ArrayDataObject.h deleted file mode 100644 index e0ee91cba..000000000 --- a/src/bufr/DataObject/ArrayDataObject.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include "DataObject.h" -#include "IngesterTypes.h" - - -namespace Ingester -{ - /// \brief Container for Parser data that is expressed as a Eigen Array of doubles. - class ArrayDataObject : public DataObject - { - public: - explicit ArrayDataObject(const IngesterArray& eigArray); - ~ArrayDataObject() = default; - - /// \brief Makes an ioda::Variable and ads it to the given ioda::ObsGroup - /// \param obsGroup Obsgroup were to add the variable - /// \param name The name to associate with the variable (ex "latitude@MetaData") - /// \param dimensions List of Variables to use as the dimensions for this new variable - /// \param chunks List of integers specifying the chunking dimensions - /// \param compressionLevel The GZip compression level to use, must be 0-9 - ioda::Variable createVariable(ioda::ObsGroup& obsGroup, - const std::string& name, - const std::vector& dimensions, - const std::vector& chunks, - int compressionLevel) final; - - /// \brief Print data to stdout for debug purposes. - void print() const final; - - /// \brief Get number of rows represented in the data - size_t nrows() const final; - - /// \brief Get number of columns represented in the data - size_t ncols() const final; - - // Getters - inline IngesterArray get() const { return eigArray_; } - - private: - /// \brief Eigen Array that holds the data - const IngesterArray eigArray_; - - /// \brief Create an ioda::VariableCreationParameters for the data. - /// \param chunks List of integers specifying the chunking dimensions - /// \param compressionLevel The GZip compression level to use, must be 0-9 - static ioda::VariableCreationParameters makeCreationParams( - const std::vector& chunks, - int compressionLevel); - }; -} // namespace Ingester - - diff --git a/src/bufr/DataObject/DataObject.h b/src/bufr/DataObject/DataObject.h deleted file mode 100644 index 6b21d5f34..000000000 --- a/src/bufr/DataObject/DataObject.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include - -#include "ioda/ObsGroup.h" -#include "ioda/defs.h" - -namespace Ingester -{ - /// \brief Abstract base class for intermediate data object that bridges the Parsers with the - /// IodaEncoder. - class DataObject - { - public: - /// \brief Makes an ioda::Variable and ads it to the given ioda::ObsGroup - /// \param obsGroup Obsgroup were to add the variable - /// \param name The name to associate with the variable (ex "latitude@MetaData") - /// \param dimensions List of Variables to use as the dimensions for this new variable - /// \param chunks List of integers specifying the chunking dimensions - /// \param compressionLevel The GZip compression level to use, must be 0-9 - virtual ioda::Variable createVariable( - ioda::ObsGroup& obsGroup, - const std::string& name, - const std::vector& dimensions, - const std::vector& chunks, - int compressionLevel) = 0; - - /// \brief Print data to stdout for debug purposes. - virtual void print() const = 0; - - /// \brief Get number of columns represented in the data. - virtual size_t ncols() const = 0; - - /// \brief Get number of rows represented in the data. - virtual size_t nrows() const = 0; - }; -} // namespace Ingester - - diff --git a/src/bufr/DataObject/Int64VecDataObject.cpp b/src/bufr/DataObject/Int64VecDataObject.cpp deleted file mode 100644 index e6c16e7b4..000000000 --- a/src/bufr/DataObject/Int64VecDataObject.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * (C) Copyright 2022 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include -#include - -#include "Int64VecDataObject.h" - - -namespace Ingester -{ - Int64VecDataObject::Int64VecDataObject(const std::vector& int64Vector) : - int64Vector_(int64Vector) - { - } - - ioda::Variable Int64VecDataObject::createVariable(ioda::ObsGroup& obsGroup, - const std::string& name, - const std::vector& dimensions, - const std::vector& chunks, - int compressionLevel) - { - auto params = makeCreationParams(chunks, compressionLevel); - auto var = obsGroup.vars.createWithScales(name, dimensions, params); - var.write(int64Vector_); - return var; - } - - void Int64VecDataObject::print() const - { - for (const auto& val : int64Vector_) - { - std::cout << "Value: " << val << std::endl; - } - } - - size_t Int64VecDataObject::nrows() const - { - return int64Vector_.size(); - } - - size_t Int64VecDataObject::ncols() const - { - return 1; - } - - ioda::VariableCreationParameters Int64VecDataObject::makeCreationParams( - const std::vector& chunks, - int compressionLevel) - { - ioda::VariableCreationParameters params; - params.chunk = true; - params.chunks = chunks; - params.compressWithGZIP(compressionLevel); - params.setFillValue(INT_MIN); - - return params; - } -} // namespace Ingester diff --git a/src/bufr/DataObject/Int64VecDataObject.h b/src/bufr/DataObject/Int64VecDataObject.h deleted file mode 100644 index 78df2c6e6..000000000 --- a/src/bufr/DataObject/Int64VecDataObject.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * (C) Copyright 2022 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include "DataObject.h" - -#include - - -namespace Ingester -{ - typedef int64_t Int64Type; - - /// \brief Container for data that can be expressed as lists of strings - class Int64VecDataObject : public DataObject - { - public: - explicit Int64VecDataObject(const std::vector& int64Vector); - ~Int64VecDataObject() = default; - - /// \brief Makes an ioda::Variable and ads it to the given ioda::ObsGroup - /// \param obsGroup Obsgroup were to add the variable - /// \param name The name to associate with the variable (ex "latitude@MetaData") - /// \param dimensions List of Variables to use as the dimensions for this new variable - /// \param chunks List of integers specifying the chunking dimensions - /// \param compressionLevel The GZip compression level to use, must be 0-9 - ioda::Variable createVariable(ioda::ObsGroup& obsGroup, - const std::string& name, - const std::vector& dimensions, - const std::vector& chunks, - int compressionLevel) final; - - /// \brief Print data to stdout for debug purposes. - void print() const final; - - /// \brief Get the number of rows represented in the data. - size_t nrows() const final; - - /// \brief Get the number of columns represented in the data. - size_t ncols() const final; - - // Getters - inline std::vector get() const { return int64Vector_; } - - private: - /// \brief The data - const std::vector int64Vector_; - - /// \brief Create an ioda::VariableCreationParameters for the data - /// \param chunks List of integers specifying the chunking dimensions - /// \param compressionLevel The GZip compression level to use, must be 0-9 - static ioda::VariableCreationParameters makeCreationParams( - const std::vector& chunks, - int compressionLevel); - }; -} // namespace Ingester - - diff --git a/src/bufr/DataObject/StrVecDataObject.cpp b/src/bufr/DataObject/StrVecDataObject.cpp deleted file mode 100644 index ed2e3a1c6..000000000 --- a/src/bufr/DataObject/StrVecDataObject.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include - -#include "StrVecDataObject.h" - - -namespace Ingester -{ - StrVecDataObject::StrVecDataObject(const std::vector& strVector) : - strVector_(strVector) - { - } - - ioda::Variable StrVecDataObject::createVariable(ioda::ObsGroup& obsGroup, - const std::string& name, - const std::vector& dimensions, - const std::vector& chunks, - int compressionLevel) - { - auto params = makeCreationParams(chunks, compressionLevel); - auto var = obsGroup.vars.createWithScales(name, dimensions, params); - var.write(strVector_); - return var; - } - - void StrVecDataObject::print() const - { - for (const auto& str : strVector_) - { - std::cout << str << std::endl; - } - } - - size_t StrVecDataObject::nrows() const - { - return strVector_.size(); - } - - size_t StrVecDataObject::ncols() const - { - return 1; - } - - ioda::VariableCreationParameters StrVecDataObject::makeCreationParams( - const std::vector& chunks, - int compressionLevel) - { - ioda::VariableCreationParameters params; - params.chunk = true; - params.chunks = chunks; - params.compressWithGZIP(compressionLevel); - - return params; - } -} // namespace Ingester diff --git a/src/bufr/DataObject/StrVecDataObject.h b/src/bufr/DataObject/StrVecDataObject.h deleted file mode 100644 index 630342795..000000000 --- a/src/bufr/DataObject/StrVecDataObject.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * (C) Copyright 2020 NOAA/NWS/NCEP/EMC - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include "DataObject.h" - -#include -#include - - -namespace Ingester -{ - /// \brief Container for data that can be expressed as lists of strings - class StrVecDataObject : public DataObject - { - public: - explicit StrVecDataObject(const std::vector& strVector); - ~StrVecDataObject() = default; - - /// \brief Makes an ioda::Variable and ads it to the given ioda::ObsGroup - /// \param obsGroup Obsgroup were to add the variable - /// \param name The name to associate with the variable (ex "latitude@MetaData") - /// \param dimensions List of Variables to use as the dimensions for this new variable - /// \param chunks List of integers specifying the chunking dimensions - /// \param compressionLevel The GZip compression level to use, must be 0-9 - ioda::Variable createVariable(ioda::ObsGroup& obsGroup, - const std::string& name, - const std::vector& dimensions, - const std::vector& chunks, - int compressionLevel) final; - - /// \brief Print data to stdout for debug purposes. - void print() const final; - - /// \brief Get the number of rows represented in the data. - size_t nrows() const final; - - /// \brief Get the number of columns represented in the data. - size_t ncols() const final; - - // Getters - inline std::vector get() const { return strVector_; } - - private: - /// \brief The data - const std::vector strVector_; - - /// \brief Create an ioda::VariableCreationParameters for the data - /// \param chunks List of integers specifying the chunking dimensions - /// \param compressionLevel The GZip compression level to use, must be 0-9 - static ioda::VariableCreationParameters makeCreationParams( - const std::vector& chunks, - int compressionLevel); - }; -} // namespace Ingester - - diff --git a/src/bufr/IngesterTypes.h b/src/bufr/IngesterTypes.h index 31d308555..0aed46840 100644 --- a/src/bufr/IngesterTypes.h +++ b/src/bufr/IngesterTypes.h @@ -7,16 +7,16 @@ #pragma once -#include +#include #include #include +#include +#include "DataObject.h" #include "Eigen/Dense" namespace Ingester { - typedef float FloatType; - typedef Eigen::Array IngesterArray; - typedef std::map IngesterArrayMap; - typedef std::vector IngesterStrVector; + typedef Eigen::Array RawData; + typedef std::unordered_map > BufrDataMap; } diff --git a/src/bufr/IodaEncoder/IodaDescription.cpp b/src/bufr/IodaEncoder/IodaDescription.cpp index 1c788c773..8493913ae 100644 --- a/src/bufr/IodaEncoder/IodaDescription.cpp +++ b/src/bufr/IodaEncoder/IodaDescription.cpp @@ -24,14 +24,15 @@ namespace namespace Dimension { const char* Name = "name"; - const char* Size = "size"; + const char* Path = "path"; + const char* Paths = "paths"; + const char* Source = "source"; } // Dimension namespace Variable { const char* Name = "name"; const char* Source = "source"; - const char* Dimensions = "dimensions"; const char* LongName = "longName"; const char* Units = "units"; const char* Range = "range"; @@ -83,21 +84,39 @@ namespace Ingester } } - auto dimConfs = conf.getSubConfigurations(ConfKeys::Dimensions); - if (dimConfs.size() == 0) + if (conf.has(ConfKeys::Dimensions)) { - std::stringstream errStr; - errStr << "ioda::dimensions must contain a list of dimensions!"; - throw eckit::BadParameter(errStr.str()); - } + auto dimConfs = conf.getSubConfigurations(ConfKeys::Dimensions); + if (dimConfs.size() == 0) { + std::stringstream errStr; + errStr << "ioda::dimensions must contain a list of dimensions!"; + throw eckit::BadParameter(errStr.str()); + } - for (const auto& dimConf : dimConfs) - { - DimensionDescription scale; - scale.name = dimConf.getString(ConfKeys::Dimension::Name); - scale.size = dimConf.getString(ConfKeys::Dimension::Size); + for (const auto &dimConf : dimConfs) { + DimensionDescription dim; + dim.name = dimConf.getString(ConfKeys::Dimension::Name); + + if (dimConf.has(ConfKeys::Dimension::Paths)) + { + dim.paths = dimConf.getStringVector(ConfKeys::Dimension::Paths); + } + else if (dimConf.has(ConfKeys::Dimension::Path)) + { + dim.paths = {dimConf.getString(ConfKeys::Dimension::Path)}; + } + else + { + throw eckit::BadParameter(R"(ioda::dimensions section must have either "path" or "paths".)"); + } - addDimension(scale); + if (dimConf.has(ConfKeys::Dimension::Source)) + { + dim.source = {dimConf.getString(ConfKeys::Dimension::Source)}; + } + + addDimension(dim); + } } auto varConfs = conf.getSubConfigurations(ConfKeys::Variables); @@ -113,9 +132,13 @@ namespace Ingester VariableDescription variable; variable.name = varConf.getString(ConfKeys::Variable::Name); variable.source = varConf.getString(ConfKeys::Variable::Source); - variable.dimensions = varConf.getStringVector(ConfKeys::Variable::Dimensions); variable.longName = varConf.getString(ConfKeys::Variable::LongName); - variable.units = varConf.getString(ConfKeys::Variable::Units); + + variable.units = ""; + if (varConf.has(ConfKeys::Variable::Units)) + { + variable.units = varConf.getString(ConfKeys::Variable::Units); + } if (varConf.has(ConfKeys::Variable::Coords)) { @@ -165,61 +188,61 @@ namespace Ingester } if (conf.has(ConfKeys::Globals)) - { - auto globalConfs = conf.getSubConfigurations(ConfKeys::Globals); - for (const auto& globalConf : globalConfs) - { - if (globalConf.getString(ConfKeys::Global::Type) == \ - ConfKeys::Global::StringType) - { - auto global = std::make_shared>(); - global->name = globalConf.getString(ConfKeys::Global::Name); - global->value = globalConf.getString(ConfKeys::Global::Value); - addGlobal(global); - } - else if (globalConf.getString(ConfKeys::Global::Type) == \ - ConfKeys::Global::FloatType) - { - auto global = std::make_shared>(); - global->name = globalConf.getString(ConfKeys::Global::Name); - global->name = globalConf.getFloat(ConfKeys::Global::Name); - addGlobal(global); - } - else if (globalConf.getString(ConfKeys::Global::Type) == \ - ConfKeys::Global::FloatVectorType) - { - auto global = std::make_shared>>(); - global->name = globalConf.getString(ConfKeys::Global::Name); - global->value = globalConf.getFloatVector(ConfKeys::Global::Value); - addGlobal(global); - } - else if (globalConf.getString(ConfKeys::Global::Type) == \ - ConfKeys::Global::IntType) - { - auto global = std::make_shared>(); - global->name = globalConf.getString(ConfKeys::Global::Name); - global->value = globalConf.getInt(ConfKeys::Global::Value); - addGlobal(global); - } - else if (globalConf.getString(ConfKeys::Global::Type) == \ - ConfKeys::Global::IntVectorType) - { - auto global = std::make_shared>>(); - global->name = globalConf.getString(ConfKeys::Global::Name); - global->value = globalConf.getIntVector(ConfKeys::Global::Value); - addGlobal(global); - } - else - { - throw eckit::BadParameter("Unsupported global attribute type"); - } - } - } + { + auto globalConfs = conf.getSubConfigurations(ConfKeys::Globals); + for (const auto& globalConf : globalConfs) + { + if (globalConf.getString(ConfKeys::Global::Type) == \ + ConfKeys::Global::StringType) + { + auto global = std::make_shared>(); + global->name = globalConf.getString(ConfKeys::Global::Name); + global->value = globalConf.getString(ConfKeys::Global::Value); + addGlobal(global); + } + else if (globalConf.getString(ConfKeys::Global::Type) == \ + ConfKeys::Global::FloatType) + { + auto global = std::make_shared>(); + global->name = globalConf.getString(ConfKeys::Global::Name); + global->name = globalConf.getFloat(ConfKeys::Global::Name); + addGlobal(global); + } + else if (globalConf.getString(ConfKeys::Global::Type) == \ + ConfKeys::Global::FloatVectorType) + { + auto global = std::make_shared>>(); + global->name = globalConf.getString(ConfKeys::Global::Name); + global->value = globalConf.getFloatVector(ConfKeys::Global::Value); + addGlobal(global); + } + else if (globalConf.getString(ConfKeys::Global::Type) == \ + ConfKeys::Global::IntType) + { + auto global = std::make_shared>(); + global->name = globalConf.getString(ConfKeys::Global::Name); + global->value = globalConf.getInt(ConfKeys::Global::Value); + addGlobal(global); + } + else if (globalConf.getString(ConfKeys::Global::Type) == \ + ConfKeys::Global::IntVectorType) + { + auto global = std::make_shared>>(); + global->name = globalConf.getString(ConfKeys::Global::Name); + global->value = globalConf.getIntVector(ConfKeys::Global::Value); + addGlobal(global); + } + else + { + throw eckit::BadParameter("Unsupported global attribute type"); + } + } + } } - void IodaDescription::addDimension(const DimensionDescription& scale) + void IodaDescription::addDimension(const DimensionDescription& dim) { - dimensions_.push_back(scale); + dimensions_.push_back(dim); } void IodaDescription::addVariable(const VariableDescription& variable) diff --git a/src/bufr/IodaEncoder/IodaDescription.h b/src/bufr/IodaEncoder/IodaDescription.h index dda94a864..d5813b79f 100644 --- a/src/bufr/IodaEncoder/IodaDescription.h +++ b/src/bufr/IodaEncoder/IodaDescription.h @@ -7,13 +7,12 @@ #pragma once -#include #include #include #include #include "eckit/config/LocalConfiguration.h" -#include "ioda/Engines/Factory.h" +#include "ioda/Engines/EngineUtils.h" #include "ioda/Group.h" namespace Ingester @@ -27,7 +26,8 @@ namespace Ingester struct DimensionDescription { std::string name; - std::string size; + std::vector paths; + std::string source; }; struct VariableDescription @@ -47,6 +47,7 @@ namespace Ingester { std::string name; virtual void addTo(ioda::Group& group) = 0; + virtual ~GlobalDescriptionBase() = default; }; template @@ -98,7 +99,7 @@ namespace Ingester explicit IodaDescription(const eckit::Configuration& conf); /// \brief Add Dimension defenition - void addDimension(const DimensionDescription& scale); + void addDimension(const DimensionDescription& dim); /// \brief Add Variable defenition void addVariable(const VariableDescription& variable); diff --git a/src/bufr/IodaEncoder/IodaEncoder.cpp b/src/bufr/IodaEncoder/IodaEncoder.cpp index d341fcc92..206779bdf 100644 --- a/src/bufr/IodaEncoder/IodaEncoder.cpp +++ b/src/bufr/IodaEncoder/IodaEncoder.cpp @@ -8,23 +8,24 @@ #include "IodaEncoder.h" #include -#include +#include +#include +#include #include "eckit/exception/Exceptions.h" -#include "ioda/Layout.h" +#include "oops/util/Logger.h" -#include +#include "ioda/Layout.h" +#include "ioda/Misc/DimensionScales.h" namespace Ingester { - IodaEncoder::IodaEncoder(const eckit::Configuration& conf) : - description_(IodaDescription(conf)) - { - } + static const char* LocationName = "Location"; + static const char* DefualtDimName = "dim"; - IodaEncoder::IodaEncoder(const IodaDescription& description) : - description_(description) + IodaEncoder::IodaEncoder(const eckit::Configuration& conf): + description_(IodaDescription(conf)) { } @@ -32,11 +33,146 @@ namespace Ingester IodaEncoder::encode(const std::shared_ptr& dataContainer, bool append) { auto backendParams = ioda::Engines::BackendCreationParameters(); - std::map obsGroups; + // Get the named dimensions + NamedPathDims namedLocDims; + NamedPathDims namedExtraDims; + + // Get a list of all the named dimensions + { + std::set dimNames; + std::set dimPaths; + for (const auto& dim : description_.getDims()) + { + if (dimNames.find(dim.name) != dimNames.end()) + { + throw eckit::UserError("ioda::dimensions: Duplicate dimension name: " + + dim.name); + } + + dimNames.insert(dim.name); + + for (auto path : dim.paths) + { + if (dimPaths.find(path) != dimPaths.end()) + { + throw eckit::BadParameter("ioda::dimensions: Declared duplicate dim. path: " + + path); + } + + if (path.substr(0, 1) != "*") + { + std::ostringstream errStr; + errStr << "ioda::dimensions: "; + errStr << "Path " << path << " must start with *. "; + errStr << "Subset specific named dimensions are not supported."; + + throw eckit::BadParameter(errStr.str()); + } + + dimPaths.insert(path); + } + + namedExtraDims.insert({dim.paths, dim}); + } + } + + // Got through each unique category for (const auto& categories : dataContainer->allSubCategories()) { + // Create the dimensions variables + std::map> dimMap; + + auto dataObjectGroupBy = dataContainer->getGroupByObject( + description_.getVariables()[0].source, categories); + + // When we find that the primary index is zero we need to skip this category + if (dataObjectGroupBy->getDims()[0] == 0) + { + for (auto category : categories) + { + oops::Log::warning() << " Skipped category " << category << std::endl; + } + + continue; + } + + // Create the root Location dimension for this category + auto rootDim = std::make_shared>(dataObjectGroupBy->getDims()[0]); + rootDim->dimScale = + ioda::NewDimensionScale(LocationName, dataObjectGroupBy->getDims()[0]); + dimMap[LocationName] = rootDim; + + // Add the root Location dimension as a named dimension + auto rootLocation = DimensionDescription(); + rootLocation.name = LocationName; + rootLocation.source = ""; + namedLocDims[{dataObjectGroupBy->getDimPaths()[0]}] = rootLocation; + + // Create the dimension data for dimensions which include source data + for (const auto& dimDesc : description_.getDims()) + { + if (!dimDesc.source.empty()) + { + auto dataObject = dataContainer->get(dimDesc.source, categories); + + // Validate the path for the source field makes sense for the dimension + if (std::find(dimDesc.paths.begin(), + dimDesc.paths.end(), + dataObject->getDimPaths().back()) == dimDesc.paths.end()) + { + std::stringstream errStr; + errStr << "ioda::dimensions: Source field " << dimDesc.source << " in "; + errStr << dimDesc.name << " is not in the correct path."; + throw eckit::BadParameter(errStr.str()); + } + + // Create the dimension data + dimMap[dimDesc.name] = dataObject->createDimensionFromData( + dimDesc.name, + dataObject->getDimPaths().size() - 1); + } + } + + // Discover and create the dimension data for dimensions with no source field. If + // dim is un-named (not listed) then call it dim_ + int autoGenDimNumber = 2; + for (const auto& varDesc : description_.getVariables()) + { + auto dataObject = dataContainer->get(varDesc.source, categories); + + for (std::size_t dimIdx = 1; dimIdx < dataObject->getDimPaths().size(); dimIdx++) + { + auto dimPath = dataObject->getDimPaths()[dimIdx]; + std::string dimName = ""; + + if (existsInNamedPath(dimPath, namedExtraDims)) + { + dimName = dimForDimPath(dimPath, namedExtraDims).name; + } + else + { + auto newDimStr = std::ostringstream(); + newDimStr << DefualtDimName << "_" << autoGenDimNumber; + + dimName = newDimStr.str(); + + auto dimDesc = DimensionDescription(); + dimDesc.name = dimName; + dimDesc.source = ""; + + namedExtraDims[{dimPath}] = dimDesc; + autoGenDimNumber++; + } + + if (dimMap.find(dimName) == dimMap.end()) + { + dimMap[dimName] = dataObject->createEmptyDimension(dimName, dimIdx); + } + } + } + // Make the filename string if (description_.getBackend() == ioda::Engines::BackendNames::Hdf5File) { @@ -63,60 +199,15 @@ namespace Ingester auto rootGroup = ioda::Engines::constructBackend(description_.getBackend(), backendParams); - bool foundInvalidDim = false; - ioda::NewDimensionScales_t newDims; - for (const auto& scale : description_.getDims()) + ioda::NewDimensionScales_t allDims; + for (auto dimPair : dimMap) { - std::size_t size = 0; - if (isInteger(scale.size)) - { - size = std::stoi(scale.size); - } - else - { - std::string token = scale.size.substr(scale.size.find('.') + 1, - scale.size.size()); - - std::string varName = scale.size.substr(0, - scale.size.find('.')); - - if (token == "ncols") - { - size = dataContainer->get(varName, categories)->ncols(); - } - else if (token == "nrows") - { - size = dataContainer->get(varName, categories)->nrows(); - } - else - { - std::ostringstream errStr; - errStr << "Tried to get unknown parameter " << token; - errStr << " from " << varName; - throw eckit::BadParameter(errStr.str()); - } - } - - auto newDim = ioda::NewDimensionScale(scale.name, size); - newDims.push_back(newDim); - - if (size <= 0) { foundInvalidDim = true; } - } - - if (foundInvalidDim) - { - continue; + allDims.push_back(dimPair.second->dimScale); } auto policy = ioda::detail::DataLayoutPolicy::Policies::ObsGroup; auto layoutPolicy = ioda::detail::DataLayoutPolicy::generate(policy); - auto obsGroup = ioda::ObsGroup::generate(rootGroup, newDims, layoutPolicy); - - auto scaleMap = std::map(); - for (const auto& scale : description_.getDims()) - { - scaleMap.insert({scale.name, obsGroup.vars[scale.name]}); - } + auto obsGroup = ioda::ObsGroup::generate(rootGroup, allDims, layoutPolicy); // Create Globals for (auto& global : description_.getGlobals()) @@ -124,14 +215,54 @@ namespace Ingester global->addTo(rootGroup); } - // Create Variables + // Write the Dimension Variables + for (const auto& dimDesc : description_.getDims()) + { + if (!dimDesc.source.empty()) + { + auto dataObject = dataContainer->get(dimDesc.source, categories); + for (size_t dimIdx = 0; dimIdx < dataObject->getDims().size(); dimIdx++) + { + auto dimPath = dataObject->getDimPaths()[dimIdx]; + + NamedPathDims namedPathDims; + if (dimIdx == 0) + { + namedPathDims = namedLocDims; + } + else + { + namedPathDims = namedExtraDims; + } + + auto dimName = dimForDimPath(dimPath, namedPathDims).name; + auto dimVar = obsGroup.vars[dimName]; + dimMap[dimName]->write(dimVar); + } + } + } + + // Write all the other Variables for (const auto& varDesc : description_.getVariables()) { std::vector chunks; auto dimensions = std::vector(); - for (size_t dimIdx = 0; dimIdx < varDesc.dimensions.size(); dimIdx++) + auto dataObject = dataContainer->get(varDesc.source, categories); + for (size_t dimIdx = 0; dimIdx < dataObject->getDims().size(); dimIdx++) { - auto dimVar = scaleMap.at(varDesc.dimensions[dimIdx]); + auto dimPath = dataObject->getDimPaths()[dimIdx]; + + NamedPathDims namedPathDims; + if (dimIdx == 0) + { + namedPathDims = namedLocDims; + } + else + { + namedPathDims = namedExtraDims; + } + + auto dimVar = obsGroup.vars[dimForDimPath(dimPath, namedPathDims).name]; dimensions.push_back(dimVar); if (dimIdx < varDesc.chunks.size()) @@ -145,18 +276,15 @@ namespace Ingester } } - auto data = dataContainer->get(varDesc.source, categories); - auto var = data->createVariable(obsGroup, - varDesc.name, - dimensions, - chunks, - varDesc.compressionLevel); - - + auto var = dataObject->createVariable(obsGroup, + varDesc.name, + dimensions, + chunks, + varDesc.compressionLevel); var.atts.add("long_name", { varDesc.longName }, {1}); - if (!(varDesc.units == "none" || varDesc.units == "")) + if (!varDesc.units.empty()) { var.atts.add("units", { varDesc.units }, {1}); } @@ -168,7 +296,7 @@ namespace Ingester if (varDesc.range) { - var.atts.add("valid_range", + var.atts.add("valid_range", {varDesc.range->start, varDesc.range->end}, {2}); } @@ -240,25 +368,33 @@ namespace Ingester return result; } - bool IodaEncoder::isInteger(const std::string& str) const - { - bool isInt = true; - if (str.empty()) - { - isInt = false; - } - else - { - for (auto it = str.begin(); it != str.end(); it++) - { - if (!std::isdigit(*it)) - { - isInt = false; + bool IodaEncoder::existsInNamedPath(const std::string& path, const NamedPathDims& pathMap) const + { + for (auto& paths : pathMap) + { + if (std::find(paths.first.begin(), paths.first.end(), path) != paths.first.end()) + { + return true; + } + } + + return false; + } + + DimensionDescription IodaEncoder::dimForDimPath(const std::string& path, + const NamedPathDims& pathMap) const + { + DimensionDescription dimDesc; + + for (auto paths : pathMap) + { + if (std::find(paths.first.begin(), paths.first.end(), path) != paths.first.end()) + { + dimDesc = paths.second; break; - } - } - } + } + } - return isInt; - } + return dimDesc; + } } // namespace Ingester diff --git a/src/bufr/IodaEncoder/IodaEncoder.h b/src/bufr/IodaEncoder/IodaEncoder.h index 11148b516..0153eb620 100644 --- a/src/bufr/IodaEncoder/IodaEncoder.h +++ b/src/bufr/IodaEncoder/IodaEncoder.h @@ -11,7 +11,7 @@ #include "eckit/config/LocalConfiguration.h" #include "ioda/Group.h" -#include "ioda/Engines/Factory.h" +#include "ioda/Engines/EngineUtils.h" #include "ioda/ObsGroup.h" #include "DataContainer.h" @@ -25,7 +25,6 @@ namespace Ingester { public: explicit IodaEncoder(const eckit::Configuration& conf); - explicit IodaEncoder(const IodaDescription& description); /// \brief Encode the data into an ioda::ObsGroup object /// \param data The data container to use @@ -34,6 +33,8 @@ namespace Ingester bool append = false); private: + typedef std::map, DimensionDescription> NamedPathDims; + /// \brief The description const IodaDescription description_; @@ -43,18 +44,22 @@ namespace Ingester std::string makeStrWithSubstitions(const std::string& prototype, const std::map& subMap); - /// \brief Used to find indecies of { and } by the makeStrWithSubstitions method. + /// \brief Used to find indicies of { and } by the makeStrWithSubstitions method. /// \param str Template string to search. std::vector>> findSubIdxs(const std::string& str); - bool isInteger(const std::string& str) const; - - // Todo: Delete with USE_OLD_LAYOUT - std::string fixCoordinatesStr(const std::string& coordStr, - std::map varMap); + /// \brief Check if the subquery string is a named dimension. + /// \param path The subquery string to check. + /// \param pathMap The map of named dimensions. + /// \return True if the subquery string is a named dimension. + bool existsInNamedPath(const std::string& path, const NamedPathDims& pathMap) const; - // Todo: Delete with USE_OLD_LAYOUT - std::pair splitVar(const std::string& varNameStr); + /// \brief Get the description associated with the named dimension. + /// \param path The subquery string for the dimension. + /// \param pathMap The map of named dimensions. + /// \return The dimension description associated with the named dimension. + DimensionDescription dimForDimPath(const std::string& path, + const NamedPathDims& pathMap) const; }; } // namespace Ingester diff --git a/src/bufr/Parser.h b/src/bufr/Parser.h index 43a2dc747..8251c021d 100644 --- a/src/bufr/Parser.h +++ b/src/bufr/Parser.h @@ -19,7 +19,6 @@ namespace Ingester public: Parser() = default; explicit Parser(const eckit::Configuration& conf); - virtual ~Parser() = default; /// \brief Parse the input. diff --git a/src/bufr/README.md b/src/bufr/README.md index 556befc98..b5d47ddf0 100644 --- a/src/bufr/README.md +++ b/src/bufr/README.md @@ -36,12 +36,6 @@ The obs space describes how to read data from the BUFR file and then how to expo obsdatain: "./testinput/gdas.t18z.1bmhs.tm00.bufr_d" isWmoFormat: true # Optional tablepath: "./testinput/bufr_tables" # Optional - - mnemonicSets: - - mnemonics: [SAID, FOVN, YEAR, MNTH, DAYS, HOUR, MINU, SECO, CLAT, CLON, CLATH, CLONH, HOLS] - - mnemonics: [SAZA, SOZA, BEARAZ, SOLAZI] - - mnemonics: [TMBR] - channels : 1-5 ``` Defines how to read data from the input BUFR file. Its sections are as follows: @@ -55,64 +49,87 @@ Defines how to read data from the input BUFR file. Its sections are as follows: standard WMO formated files. Only applies if `isWmoFormat` is `true`. If this field is missing and`isWmoFormat` is `true` then NCEPLib-bufr will look for the table data in its default directory. -* `mnemonicSets` Defines the list of mnemonic sets to read from the BUFR file. - * `mnemonics` Defines a group of mnemonics to parse from a BUFR subset. - * _(optional)_ `channels` specifies channels to capture. This could be a disjoint set such as - “1, 3, 8-15” or just "1-5" as in the example. - * Internally _NCEPLib BUFR_ **ufbint** used for single value mnemonics, and **ufbrep** used for - multi channels mnemonics. #### Exports ```yaml exports: + group_by_variable: longitude # Optional + subsets: + - NC004001 + - NC004002 + - NC004003 + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" # default assumed zero if skipped or found as missing + hoursFromUtc: 0 # Optional + + # Or, sometimes BUFR data use an offset time related to model analysis/cycle. + timestamp: + timeoffset: + timeOffset: "*/PRSLEVEL/DRFTINFO/HRDR" + transforms: + - scale: 3600 + referenceTime: "2020-11-01T12:00:00Z" + + satellite_id: + query: "*/SAID" + type: int64 + longitude: + query: "*/CLON" + transforms: + - offset: -180 + latitude: + query: "*/CLAT" + channels: + query: "[*/BRITCSTC/CHNM, */BRIT/CHNM]" + radiance: + query: "[*/BRITCSTC/TMBR, */BRIT/TMBR]" + splits: satId: - category: - mnemonic: SAID - map: + category: + variable: satellite_id + map: _3: sat_1 # can't use integers as keys _5: sat_2 _8: sat_3 - + filters: - bounding: - mnemonic: CLON + variable: longitude upperBound: -68 # optional lowerBound: -86.3 # optional - - variables: - timestamp: - datetime: - year: YEAR - month: MNTH - day: DAYS - hour: HOUR - minute: MINU - second: SECO # default assumed zero if skipped or found as missing. - hoursFromUtc: 0 # optional - longitude: - mnemonic: CLON - transforms: - - offset: -180 - latitude: - mnemonic: CLAT - radiance: - mnemonic: TMBR ``` Exports is a dictionary of key value pairs which define a name to the data element to expose the ioda encoder. It has the following sections: - +* `group_by_variable` _(optional)_ String value that defines the name of the variable to group + observations by. If this field is missing then observations will not be grouped. +* `subsets` _(optional)_ List of subsets that you want to process. If the field is not present then + all subsets will be processed in accordance with the query definitions. * `variables` * **keys** are arbitrary strings (anything you want). They can be referenced in the ioda section. * **values** (One of these types): - * `mnemonic` Associate **key** with data for mnemonic listed in mnemonic set. _(optional)_ Can - apply a list of `tranforms` to the data. Possible transforms are `offset` and `scale`. - * `dateTime` Associate **key** with data for mnemonics for `year`, `month`, `day`, `hour`, + * `query` Query string which is used to get the data from the BUFR file. _(optional)_ Can + apply a list of `tranforms` to the numeric (not string) data. Possible transforms are + `offset` and `scale`. You can also manually override the type by specifying the `type` as + **int**, **int64**, **float**, or **double**. + * `datetime` Associate **key** with data for mnemonics for `year`, `month`, `day`, `hour`, `minute`, _(optional)_ `second`, and _(optional)_ `hoursFromUtc` (must be an **integer**). Internally, the value stored is number of seconds elapsed since a reference epoch, currently set to 1970-01-01T00:00:00Z. + * `timeoffset` Associate **key** with data for mnemonic for `timeOffset`, that should result + in seconds relative to an ISO-8601 string of date and time (e.g., `2020-11-01T11:42:56Z`). + If the timeOffset mnemonic is a floating-point value in hours, then simply use **transforms** + and scale by 3600 seconds. Internally, the value stored is number of seconds elapsed since + a reference epoch, currently set to 1970-01-01T00:00:00Z. * _(optional)_ `splits` List of key value pair (splits) that define how to split the data into @@ -123,7 +140,7 @@ ioda encoder. It has the following sections: * **keys** are arbitrary strings (anything you want). They can be referenced in the ioda section. * **values** Type of split to apply (currently supports `category`) * `category` Splits data based on values assocatied with a BUFR mnemonic. Constists of: - * `mnemonic` The mnemonic to use. + * `variable` The variable from the `variables` section to split on. * _(optional)_ `map` Associates integer values in BUFR mnemonic data to a string. Please not that integer keys must be prepended with an `_` (ex: `_2`). Rows where where the mnemonic value is not defined in the map will be rejected (won't appear in output). @@ -132,7 +149,7 @@ ioda encoder. It has the following sections: * _(optional)_ `filters`List of filters to apply to the data before exporting. Filters exclude data which does not meet their requirements. The following filters are supported: * `bounding` - * `mnemonic` The mnemonic to use. + * `variable` The variable from the `variables` section to filter on. * _(optional)_ `upperBound` The highest possible value to accept * _(optional)_ `lowerBound` The lowest possible value to accept @@ -147,28 +164,26 @@ ioda encoder. It has the following sections: obsdataout: "./testrun/gdas.t00z.1bhrs4.tm00.{splits/satId}.nc" dimensions: - - name: "nlocs" - size: "variables/radiance.nrows" - - name: "nchans" - size: "variables/radiance.ncols" + - name: nchans + paths: + - "*/BRIT" + - "*/BRITCSTC" + source: variables/channels variables: - name: "MetaData/dateTime" source: "variables/timestamp" - dimensions: [ "nlocs" ] longName: "dateTime" units: "seconds since 1970-01-01T00:00:00Z" - name: "MetaData/latitude" source: "variables/latitude" - dimensions: ["nlocs"] longName: "Latitude" units: "degrees_north" range: [-90, 90] - name: "MetaData/longitude" source: "variables/longitude" - dimensions: ["nlocs"] longName: "Longitude" units: "degrees_east" range: [-180, 180] @@ -176,7 +191,6 @@ ioda encoder. It has the following sections: - name: "ObsValue/radiance" coordinates: "longitude latitude nchans" source: "variables/radiance" - dimensions: ["nlocs", "nchans"] longName: "Radiance" units: "K" range: [120, 500] @@ -192,15 +206,19 @@ The `ioda` section defines the ObsGroup objects that will be created. replaced with the relevant split category ID for that file to form a unique name for every file. * `dimensions` used to define dimension information in variables * `name` arbitrary name for the dimension - * `size` can be either a integer or a reference to exported data ex: - **variables/radiance.nrows** + * `paths` list of subqueries for that dimension (different paths for different BUFR subsets + only) **or** `path` Single subquery for that dimension ex: **\*/BRITCSTC** + * `source` (optional) The exported data that acts as the source field for this dimension. + The data dimension values (labels) will reflect this field. The source is validated + to make sure it makes sense for the dimension and that it is made up of repeated + values for each occurrence of the sequence. The source field must be inside the + dimension and be 1:1 with it. * `variables` List of output variable objects to create. * `name` standardized pathname **group**/**var_name**. * **var_name** name for the variable * **group** group name to which this variable belongs. * `source` reference to exported data ex: **variables/radiance** * `coordinates` (optional): - * `dimensions` list of ids defined in dimensions section * `longName`any arbitrary string. * `units` string representing units (arbitrary but following udunits). * _(optional)_ `range` Possible range of values (list of 2 ints). diff --git a/src/bufr/bufr2ioda.cpp b/src/bufr/bufr2ioda.cpp index 30fd62cbe..6d1f37b89 100644 --- a/src/bufr/bufr2ioda.cpp +++ b/src/bufr/bufr2ioda.cpp @@ -15,7 +15,6 @@ #include "eckit/filesystem/PathName.h" #include "BufrParser/BufrDescription.h" -#include "BufrParser/BufrMnemonicSet.h" #include "BufrParser/BufrParser.h" #include "IodaEncoder/IodaDescription.h" #include "IodaEncoder/IodaEncoder.h" @@ -26,7 +25,7 @@ namespace Ingester { - void parse(std::string yamlPath) + void parse(const std::string& yamlPath, std::size_t numMsgs = 0) { std::unique_ptr yaml(new eckit::YAMLConfiguration(eckit::PathName(yamlPath))); @@ -43,7 +42,7 @@ namespace Ingester } auto parser = ParserFactory::create(obsConf.getSubConfiguration("obs space")); - auto data = parser->parse(); + auto data = parser->parse(numMsgs); auto encoder = IodaEncoder(obsConf.getSubConfiguration("ioda")); encoder.encode(data); @@ -62,15 +61,65 @@ namespace Ingester } // namespace Ingester +static void showHelp() +{ + std::cerr << "Usage: bufr2ioda.x [-n NUM_MESSAGES] YAML_PATH" + << "Options:\n" + << " -h, Show this help message\n" + << " -n NUM_MESSAGES, Number of BUFR messages to parse." + << std::endl; +} + + int main(int argc, char **argv) { if (argc < 2) { - eckit::BadParameter("Missing argument. Must include YAML file path."); + showHelp(); + return 0; + } + + std::string yamlPath; + std::size_t numMsgs = 0; + + std::size_t argIdx = 1; + while (argIdx < static_cast (argc)) + { + if (strcmp(argv[argIdx], "-n") == 0) + { + if (static_cast (argc) > argIdx + 1) + { + numMsgs = atoi(argv[argIdx + 1]); + } + else + { + showHelp(); + return 0; + } + + argIdx += 2; + } + else if (strcmp(argv[argIdx], "-h") == 0) + { + showHelp(); + return 0; + } + else + { + yamlPath = std::string(argv[argIdx]); + argIdx++; + } } - Ingester::registerParsers(); - Ingester::parse(std::string(argv[1])); + try + { + Ingester::registerParsers(); + Ingester::parse(yamlPath, numMsgs); + } + catch (const std::exception &e) + { + throw; + } return 0; } diff --git a/src/compo/modis_aod2ioda.py b/src/compo/modis_aod2ioda.py old mode 100644 new mode 100755 diff --git a/src/compo/mopitt_co_nc2ioda.py b/src/compo/mopitt_co_nc2ioda.py index 9268a317e..a88b7aed8 100755 --- a/src/compo/mopitt_co_nc2ioda.py +++ b/src/compo/mopitt_co_nc2ioda.py @@ -54,8 +54,9 @@ class mopitt(object): - def __init__(self, filenames): + def __init__(self, filenames, time_range): self.filenames = filenames + self.time_range = time_range self.varDict = defaultdict(lambda: defaultdict(dict)) self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) self.varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) @@ -151,18 +152,29 @@ def _read(self): ap_tc = ap_tc.astype('float32') # set flag: rule out all anomalous data - flg = qa == 0 + qaf = qa == 0 + + # date range to fit DA window + date_start = datetime.strptime(self.time_range[0], "%Y%m%d%H") + date_end = datetime.strptime(self.time_range[1], "%Y%m%d%H") + date_list = [datetime.strptime(date, "%Y-%m-%dT%H:%M:%SZ") for date in times] + tsf = [(date_i >= date_start) & (date_i < date_end) for date_i in date_list] + + flg = np.logical_and(qaf, tsf) if first: # add metadata variables self.outdata[('datetime', 'MetaData')] = times[flg] self.outdata[('latitude', 'MetaData')] = lats[flg] self.outdata[('longitude', 'MetaData')] = lons[flg] - self.outdata[('apriori_term', 'MetaData')] = ap_tc[flg] + self.outdata[('apriori_term', 'RtrvlAncData')] = ap_tc[flg] for k in range(nlevs): - varname_ak = ('averaging_kernel_level_'+str(k+1), 'MetaData') + varname_ak = ('averaging_kernel_level_'+str(k+1), 'RtrvlAncData') self.outdata[varname_ak] = ak_tc_dimless[:, k][flg] - varname_pr = ('pressure_level_'+str(k+1), 'MetaData') + # add top vertice in IODA file, here it is 0hPa but can be different + # for other obs stream + for k in range(nlevs+1): + varname_pr = ('pressure_level_'+str(k+1), 'RtrvlAncData') self.outdata[varname_pr] = hPa2Pa * pr_gd[:, k][flg] self.outdata[self.varDict[iodavar]['valKey']] = xr_tc[flg] @@ -176,13 +188,16 @@ def _read(self): self.outdata[('latitude', 'MetaData')], lats[flg])) self.outdata[('longitude', 'MetaData')] = np.concatenate(( self.outdata[('longitude', 'MetaData')], lons[flg])) - self.outdata[('apriori_term', 'MetaData')] = np.concatenate(( - self.outdata[('apriori_term', 'MetaData')], ap_tc[flg])) + self.outdata[('apriori_term', 'RtrvlAncData')] = np.concatenate(( + self.outdata[('apriori_term', 'RtrvlAncData')], ap_tc[flg])) for k in range(nlevs): - varname_ak = ('averaging_kernel_level_'+str(k+1), 'MetaData') + varname_ak = ('averaging_kernel_level_'+str(k+1), 'RtrvlAncData') self.outdata[varname_ak] = np.concatenate( (self.outdata[varname_ak], ak_tc_dimless[:, k][flg])) - varname_pr = ('pressure_level_'+str(k+1), 'MetaData') + # add top vertice in IODA file, here it is 0hPa but can be different + # for other obs stream + for k in range(nlevs+1): + varname_pr = ('pressure_level_'+str(k+1), 'RtrvlAncData') self.outdata[varname_pr] = np.concatenate( (self.outdata[varname_pr], hPa2Pa * pr_gd[:, k][flg])) @@ -199,7 +214,7 @@ def _read(self): for k in range(nlevs): varname = 'averaging_kernel_level_'+str(k+1) - vkey = (varname, 'MetaData') + vkey = (varname, 'RtrvlAncData') self.varAttrs[vkey]['coordinates'] = 'longitude latitude' self.varAttrs[vkey]['units'] = '' @@ -224,10 +239,18 @@ def main(): help="path of IODA output file", type=str, required=True) + optional = parser.add_argument_group(title='optional arguments') + optional.add_argument( + '-r', '--time_range', + help="extract a date range to fit the data assimilation window" + "format -r YYYYMMDDHH YYYYMMDDHH", + type=str, metavar=('begindate', 'enddate'), nargs=2, + default=('1970010100', '2170010100')) + args = parser.parse_args() # Read in the MOPITT CO data - co = mopitt(args.input) + co = mopitt(args.input, args.time_range) # setup the IODA writer writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) diff --git a/src/compo/omi_o3_nc2ioda.py b/src/compo/omi_o3_nc2ioda.py index a9f0f8adf..778001ce3 100755 --- a/src/compo/omi_o3_nc2ioda.py +++ b/src/compo/omi_o3_nc2ioda.py @@ -328,11 +328,15 @@ def main(): rawFiles.sort() # only read files in window rawFilesOut = [] - for f in rawFiles: + for fi, f in enumerate(rawFiles): vv = f.split('_') # v003-2020m1214t060743.he5 2020m1214t0003-o87313 startDateFile = datetime.strptime(vv[-2][0:-7], "%Ym%m%dt%H%M") - endDateFile = datetime.strptime(vv[-1][5:-6], "%Ym%m%dt%H%M") + endDateFile = startDateFile + # Check the the start time for the next file to get the end time of current file. + if(fi != len(rawFiles)-1): + vv = rawFiles[fi+1].split('_') + endDateFile = datetime.strptime(vv[-2][0:-7], "%Ym%m%dt%H%M") if(startDateWindow <= startDateFile <= endDateWindow or startDateWindow <= endDateFile <= endDateWindow): rawFilesOut.append(f) rawFiles = rawFilesOut diff --git a/src/compo/ompsnm_o3_nc2ioda.py b/src/compo/ompsnm_o3_nc2ioda.py index c76cbe50d..ca72c0785 100755 --- a/src/compo/ompsnm_o3_nc2ioda.py +++ b/src/compo/ompsnm_o3_nc2ioda.py @@ -248,11 +248,15 @@ def main(): # only read files in the window. rawFilesOut = [] - for f in rawFiles: + for fi, f in enumerate(rawFiles): vv = f.split('_') # 2020m1216t011958.h5 2020m1215t222840 startDateFile = datetime.strptime(vv[-3][0:-2], "%Ym%m%dt%H%M") - endDateFile = datetime.strptime(vv[-1][0:-5], "%Ym%m%dt%H%M") + endDateFile = startDateFile + # Check the the start time for the next file to get the end time of current file. + if(fi != len(rawFiles)-1): + vv = rawFiles[fi+1].split('_') + endDateFile = datetime.strptime(vv[-2][0:-7], "%Ym%m%dt%H%M") if(startDateWindow <= startDateFile <= endDateWindow or startDateWindow <= endDateFile <= endDateWindow): rawFilesOut.append(f) rawFiles = rawFilesOut diff --git a/src/compo/tropomi_no2_nc2ioda.py b/src/compo/tropomi_no2_nc2ioda.py index 32f67f008..99dea084f 100755 --- a/src/compo/tropomi_no2_nc2ioda.py +++ b/src/compo/tropomi_no2_nc2ioda.py @@ -25,33 +25,31 @@ from orddicts import DefaultOrderedDict locationKeyList = [ - ("latitude", "float"), - ("longitude", "float"), - ("datetime", "string"), + ("latitude", "float", "degrees_north"), + ("longitude", "float", "degrees_east"), + ("dateTime", "long", "seconds since 1970-01-01T00:00:00Z"), ] - -obsvars = { - 'nitrogendioxide_tropospheric_column': 'nitrogen_dioxide_in_tropospheric_column', - 'nitrogendioxide_total_column': 'nitrogen_dioxide_in_total_column', -} +meta_keys = [m_item[0] for m_item in locationKeyList] AttrData = { 'converter': os.path.basename(__file__), - 'nvars': np.int32(len(obsvars)), + 'nvars': np.int32(1), } DimDict = { } -VarDims = { - 'nitrogen_dioxide_in_tropospheric_column': ['nlocs'], - 'nitrogen_dioxide_in_total_column': ['nlocs'], -} +iso8601_string = locationKeyList[meta_keys.index('dateTime')][2] +epoch = datetime.fromisoformat(iso8601_string[14:-1]) class tropomi(object): - def __init__(self, filenames): + def __init__(self, filenames, columnType, qa_flg, thin, obsVar): self.filenames = filenames + self.columnType = columnType + self.qa_flg = qa_flg + self.thin = thin + self.obsVar = obsVar self.varDict = defaultdict(lambda: defaultdict(dict)) self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) self.varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) @@ -60,125 +58,142 @@ def __init__(self, filenames): # Open input file and read relevant info def _read(self): # set up variable names for IODA - for iodavar in ['nitrogen_dioxide_in_tropospheric_column', 'nitrogen_dioxide_in_total_column']: - self.varDict[iodavar]['valKey'] = iodavar, iconv.OvalName() - self.varDict[iodavar]['errKey'] = iodavar, iconv.OerrName() - self.varDict[iodavar]['qcKey'] = iodavar, iconv.OqcName() - self.varAttrs[iodavar, iconv.OvalName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OerrName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OqcName()]['coordinates'] = 'longitude latitude' - self.varAttrs[iodavar, iconv.OvalName()]['units'] = 'mol m-2' - self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'mol m-2' + if self.columnType == 'total': + iodavar = self.obsVar['nitrogendioxide_total_column'] + elif self.columnType == 'tropo': + iodavar = self.obsVar['nitrogendioxide_tropospheric_column'] + self.varDict[iodavar]['valKey'] = iodavar, iconv.OvalName() + self.varDict[iodavar]['errKey'] = iodavar, iconv.OerrName() + self.varDict[iodavar]['qcKey'] = iodavar, iconv.OqcName() + self.varAttrs[iodavar, iconv.OvalName()]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, iconv.OerrName()]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, iconv.OqcName()]['coordinates'] = 'longitude latitude' + self.varAttrs[iodavar, iconv.OvalName()]['units'] = 'mol m-2' + self.varAttrs[iodavar, iconv.OerrName()]['units'] = 'mol m-2' # loop through input filenames first = True for f in self.filenames: ncd = nc.Dataset(f, 'r') + # get global attributes AttrData['date_time_string'] = ncd.getncattr('time_reference')[0:19]+'Z' AttrData['sensor'] = ncd.getncattr('sensor') AttrData['platform'] = ncd.getncattr('platform') + # many variables are time, scanline, ground_pixel # but others are just time, scanline lats = ncd.groups['PRODUCT'].variables['latitude'][:].ravel() lons = ncd.groups['PRODUCT'].variables['longitude'][:].ravel() qa_value = ncd.groups['PRODUCT'].variables['qa_value'][:] # 2D - times = np.empty_like(qa_value, dtype=object) + time_offsets = np.empty_like(qa_value, dtype=np.int64) qa_value = qa_value.ravel() + + # adding ability to pre filter the data using the qa value + # and also perform thinning using random uniform draw + qaf = qa_value > self.qa_flg + thi = np.random.uniform(size=len(lons)) > self.thin + flg = np.logical_and(qaf, thi) qc_flag = ncd.groups['PRODUCT'].groups['SUPPORT_DATA'].groups['DETAILED_RESULTS']\ .variables['processing_quality_flags'][:] qc_flag = qc_flag.ravel().astype('int32') time1 = ncd.groups['PRODUCT'].variables['time_utc'][:] for t in range(len(time1[0])): - times[0, t, :] = time1[0, t][0:19]+'Z' - times = times.ravel() - # need additional variable to use the averaging kernel for DA - kernel_err = ncd.groups['PRODUCT'].\ - variables['nitrogendioxide_tropospheric_column_precision_kernel'][:].ravel() - kernel_err_total = ncd.groups['PRODUCT'].groups['SUPPORT_DATA'].groups['DETAILED_RESULTS'].\ - variables['nitrogendioxide_total_column_precision_kernel'][:].ravel() + this_datetime = datetime.fromisoformat(time1[0, t][0:19]) + time_offset = round((this_datetime - epoch).total_seconds()) + time_offsets[0, t, :] = time_offset + time_offsets = time_offsets.ravel() trop_layer = ncd.groups['PRODUCT'].variables['tm5_tropopause_layer_index'][:].ravel() total_airmass = ncd.groups['PRODUCT'].variables['air_mass_factor_total'][:].ravel() trop_airmass = ncd.groups['PRODUCT'].\ variables['air_mass_factor_troposphere'][:].ravel() + # get info to construct the pressure level array ps = ncd.groups['PRODUCT'].groups['SUPPORT_DATA'].groups['INPUT_DATA'].\ variables['surface_pressure'][:] + # bottom of layer is vertice 0, very top layer is TOA (0hPa) - ak = ncd.groups['PRODUCT'].variables['tm5_constant_a'][:, 0] - bk = ncd.groups['PRODUCT'].variables['tm5_constant_b'][:, 0] + ak = ncd.groups['PRODUCT'].variables['tm5_constant_a'][:, :] + bk = ncd.groups['PRODUCT'].variables['tm5_constant_b'][:, :] + # grab the averaging kernel avg_kernel = ncd.groups['PRODUCT'].variables['averaging_kernel'][:] nlevs = len(avg_kernel[0, 0, 0]) AttrData['averaging_kernel_levels'] = np.int32(nlevs) + # scale the avk using AMF ratio and tropopause level for tropo column + nlocf = len(trop_layer[flg]) + scaleAK = np.ones((nlocf, nlevs), dtype=np.float32) + if self.columnType == 'tropo': + # do not loop over nlocs here this makes the execution very slow + for k in range(nlevs): + scaleAK[..., k][np.full((nlocf), k, dtype=int) > trop_layer[flg]] = 0 + scaleAK[..., k] *= total_airmass[flg] / trop_airmass[flg] + if first: # add metadata variables - self.outdata[('datetime', 'MetaData')] = times - self.outdata[('latitude', 'MetaData')] = lats - self.outdata[('longitude', 'MetaData')] = lons - self.outdata[('quality_assurance_value', 'MetaData')] = qa_value - self.outdata[('troposphere_layer_index', 'MetaData')] = trop_layer - self.outdata[('air_mass_factor_total', 'MetaData')] = total_airmass - self.outdata[('air_mass_factor_troposphere', 'MetaData')] = trop_airmass - self.outdata[('tropospheric_averaging_kernel_precision', 'MetaData')] = kernel_err - self.outdata[('averaging_kernel_precision', 'MetaData')] = kernel_err_total + self.outdata[('dateTime', 'MetaData')] = time_offsets[flg] + self.outdata[('latitude', 'MetaData')] = lats[flg] + self.outdata[('longitude', 'MetaData')] = lons[flg] + self.outdata[('quality_assurance_value', 'MetaData')] = qa_value[flg] for k in range(nlevs): - varname_ak = ('averaging_kernel_level_'+str(k+1), 'MetaData') - self.outdata[varname_ak] = avg_kernel[..., k].ravel() - varname_pr = ('pressure_level_'+str(k+1), 'MetaData') - self.outdata[varname_pr] = ak[k] + bk[k]*ps[...].ravel() + varname_ak = ('averaging_kernel_level_'+str(k+1), 'RtrvlAncData') + self.outdata[varname_ak] = avg_kernel[..., k].ravel()[flg] * scaleAK[..., k] + varname_pr = ('pressure_level_'+str(k+1), 'RtrvlAncData') + self.outdata[varname_pr] = ak[k, 0] + bk[k, 0]*ps[...].ravel()[flg] + # add top vertice in IODA file, here it is 0hPa but can be different + # for other obs stream + varname_pr = ('pressure_level_'+str(nlevs+1), 'RtrvlAncData') + self.outdata[varname_pr] = ak[nlevs-1, 1] + bk[nlevs-1, 1]*ps[...].ravel() + else: - self.outdata[('datetime', 'MetaData')] = np.concatenate(( - self.outdata[('datetime', 'MetaData')], times)) + self.outdata[('dateTime', 'MetaData')] = np.concatenate(( + self.outdata[('dateTime', 'MetaData')], time_offsets[flg])) self.outdata[('latitude', 'MetaData')] = np.concatenate(( - self.outdata[('latitude', 'MetaData')], lats)) + self.outdata[('latitude', 'MetaData')], lats[flg])) self.outdata[('longitude', 'MetaData')] = np.concatenate(( - self.outdata[('longitude', 'MetaData')], lons)) + self.outdata[('longitude', 'MetaData')], lons[flg])) self.outdata[('quality_assurance_value', 'MetaData')] = np.concatenate(( - self.outdata[('quality_assurance_value', 'MetaData')], qa_value)) - self.outdata[('troposphere_layer_index', 'MetaData')] = np.concatenate(( - self.outdata[('troposphere_layer_index', 'MetaData')], trop_layer)) - self.outdata[('air_mass_factor_total', 'MetaData')] = np.concatenate(( - self.outdata[('air_mass_factor_total', 'MetaData')], total_airmass)) - self.outdata[('air_mass_factor_troposphere', 'MetaData')] = np.concatenate(( - self.outdata[('air_mass_factor_troposphere', 'MetaData')], trop_airmass)) - self.outdata[('tropospheric_averaging_kernel_precision', 'MetaData')] = np.concatenate(( - self.outdata[('tropospheric_averaging_kernel_precision', 'MetaData')], kernel_err)) - self.outdata[('averaging_kernel_precision', 'MetaData')] = np.concatenate(( - self.outdata[('averaging_kernel_precision', 'MetaData')], kernel_err_total)) + self.outdata[('quality_assurance_value', 'MetaData')], qa_value[flg])) for k in range(nlevs): - varname_ak = ('averaging_kernel_level_'+str(k+1), 'MetaData') + varname_ak = ('averaging_kernel_level_'+str(k+1), 'RtrvlAncData') self.outdata[varname_ak] = np.concatenate( - (self.outdata[varname_ak], avg_kernel[..., k].ravel())) - varname_pr = ('pressure_level_'+str(k+1), 'MetaData') + (self.outdata[varname_ak], avg_kernel[..., k].ravel()[flg] * scaleAK[..., k])) + varname_pr = ('pressure_level_'+str(k+1), 'RtrvlAncData') self.outdata[varname_pr] = np.concatenate( - (self.outdata[varname_pr], ak[k] + bk[k]*ps[...].ravel())) - for ncvar, iodavar in obsvars.items(): + (self.outdata[varname_pr], ak[k, 0] + bk[k, 0]*ps[...].ravel()[flg])) + varname_pr = ('pressure_level_'+str(nlevs+1), 'RtrvlAncData') + self.outdata[varname_pr] = np.concatenate( + (self.outdata[varname_pr], ak[nlevs-1, 1] + bk[nlevs-1, 1]*ps[...].ravel()[flg])) + + for ncvar, iodavar in self.obsVar.items(): + if ncvar in ['nitrogendioxide_tropospheric_column']: - data = ncd.groups['PRODUCT'].variables[ncvar][:].ravel() - err = ncd.groups['PRODUCT'].variables[ncvar+'_precision'][:].ravel() + data = ncd.groups['PRODUCT'].variables[ncvar][:].ravel()[flg] + err = ncd.groups['PRODUCT'].variables[ncvar+'_precision'][:].ravel()[flg] else: - data = ncd.groups['PRODUCT'].groups['SUPPORT_DATA'].groups['DETAILED_RESULTS'].variables[ncvar][:].ravel() - err = ncd.groups['PRODUCT'].groups['SUPPORT_DATA'].groups['DETAILED_RESULTS'].variables[ncvar+'_precision'][:].ravel() + data = ncd.groups['PRODUCT'].groups['SUPPORT_DATA'].groups['DETAILED_RESULTS'].variables[ncvar][:].ravel()[flg] + err = ncd.groups['PRODUCT'].groups['SUPPORT_DATA'].groups['DETAILED_RESULTS'].variables[ncvar+'_precision'][:].ravel()[flg] + if first: self.outdata[self.varDict[iodavar]['valKey']] = data self.outdata[self.varDict[iodavar]['errKey']] = err - self.outdata[self.varDict[iodavar]['qcKey']] = qc_flag + self.outdata[self.varDict[iodavar]['qcKey']] = qc_flag[flg] else: self.outdata[self.varDict[iodavar]['valKey']] = np.concatenate( (self.outdata[self.varDict[iodavar]['valKey']], data)) self.outdata[self.varDict[iodavar]['errKey']] = np.concatenate( (self.outdata[self.varDict[iodavar]['errKey']], err)) self.outdata[self.varDict[iodavar]['qcKey']] = np.concatenate( - (self.outdata[self.varDict[iodavar]['qcKey']], qc_flag)) + (self.outdata[self.varDict[iodavar]['qcKey']], qc_flag[flg])) + first = False - DimDict['nlocs'] = len(self.outdata[('datetime', 'MetaData')]) - AttrData['nlocs'] = np.int32(DimDict['nlocs']) + + DimDict['Location'] = len(self.outdata[('dateTime', 'MetaData')]) + AttrData['Location'] = np.int32(DimDict['Location']) for k in range(nlevs): varname = 'averaging_kernel_level_'+str(k+1) - vkey = (varname, 'MetaData') + vkey = (varname, 'RtrvlAncData') self.varAttrs[vkey]['coordinates'] = 'longitude latitude' - self.varAttrs[vkey]['units'] = '' def main(): @@ -186,7 +201,7 @@ def main(): # get command line arguments parser = argparse.ArgumentParser( description=( - 'Reads TROPOMI NO2 netCDF files provided by NESDIS' + 'Reads TROPOMI NO2 netCDF files: official Copernicus product' 'and converts into IODA formatted output files. Multiple' 'files are able to be concatenated.') ) @@ -200,17 +215,54 @@ def main(): '-o', '--output', help="path of IODA output file", type=str, required=True) + required.add_argument( + '-c', '--column', + help="type of column: total or tropo", + type=str, required=True) + optional = parser.add_argument_group(title='optional arguments') + optional.add_argument( + '-q', '--qa_value', + help="qa value used to preflag data that goes into file before QC" + "default at 0.75 as suggested in the documentation. See:" + "https://sentinel.esa.int/documents/247904/2474726/" + "Sentinel-5P-Level-2-Product-User-Manual-Nitrogen-Dioxide.pdf section 8.6", + type=float, default=0.75) + optional.add_argument( + '-n', '--thin', + help="percentage of random thinning from 0.0 to 1.0. Zero indicates" + " no thinning is performed. (default: %(default)s)", + type=float, default=0.0) args = parser.parse_args() + if args.column == "tropo": + + obsVar = { + 'nitrogendioxide_tropospheric_column': 'nitrogen_dioxide_in_tropospheric_column' + } + + varDims = { + 'nitrogen_dioxide_in_tropospheric_column': ['Location'] + } + + elif args.column == "total": + + obsVar = { + 'nitrogendioxide_total_column': 'nitrogen_dioxide_in_total_column' + } + + varDims = { + 'nitrogen_dioxide_in_total_column': ['Location'] + } + # Read in the NO2 data - no2 = tropomi(args.input) + no2 = tropomi(args.input, args.column, args.qa_value, args.thin, obsVar) # setup the IODA writer writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) # write everything out - writer.BuildIoda(no2.outdata, VarDims, no2.varAttrs, AttrData) + writer.BuildIoda(no2.outdata, varDims, no2.varAttrs, AttrData) if __name__ == '__main__': diff --git a/src/conventional/CMakeLists.txt b/src/conventional/CMakeLists.txt index 12be1952e..47869d107 100644 --- a/src/conventional/CMakeLists.txt +++ b/src/conventional/CMakeLists.txt @@ -5,6 +5,7 @@ list(APPEND programs buoy_bufr2ioda.py metar_csv2ioda.py sonde_bufr2ioda.py + sonde_tac2ioda.py ship_bufr2ioda.py synop_bufr2ioda.py ) diff --git a/src/conventional/amdar_bufr2ioda.py b/src/conventional/amdar_bufr2ioda.py index a27421087..d49a4118f 100644 --- a/src/conventional/amdar_bufr2ioda.py +++ b/src/conventional/amdar_bufr2ioda.py @@ -71,10 +71,10 @@ obserrlist = [1.2, 0.75E-3, 1.7, 1.7] VarDims = { - 'airTemperature': ['nlocs'], - 'specificHumidity': ['nlocs'], - 'windEastward': ['nlocs'], - 'windNorthward': ['nlocs'], + 'airTemperature': ['Location'], + 'specificHumidity': ['Location'], + 'windEastward': ['Location'], + 'windNorthward': ['Location'], } metaDataName = iconv.MetaDataName() @@ -87,7 +87,7 @@ 'ioda_version': 2, 'description': 'Aircraft observations converted from BUFR', 'source': 'LDM at NCAR-RAL', - 'source_files': '' + 'sourceFiles': '' } DimDict = { @@ -134,12 +134,12 @@ def main(file_names, output_file): for fname in file_names: logging.debug("Reading file: " + fname) - AttrData['source_files'] += ", " + fname + AttrData['sourceFiles'] += ", " + fname data, count, start_pos = read_file(fname, count, start_pos, data) - AttrData['source_files'] = AttrData['source_files'][2:] - logging.debug("All source files: " + AttrData['source_files']) + AttrData['sourceFiles'] = AttrData['sourceFiles'][2:] + logging.debug("All source files: " + AttrData['sourceFiles']) if not data: logging.critical("ABORT: no message data was captured, stopping execution.") @@ -147,8 +147,7 @@ def main(file_names, output_file): logging.info("--- {:9.4f} BUFR read seconds ---".format(time.time() - start_time)) nlocs = len(data['dateTime']) - DimDict = {'nlocs': nlocs} - AttrData['nlocs'] = np.int32(DimDict['nlocs']) + DimDict = {'Location': nlocs} # Set coordinates and units of the ObsValues. for n, iodavar in enumerate(obsvars): diff --git a/src/conventional/buoy_bufr2ioda.py b/src/conventional/buoy_bufr2ioda.py index ae311bbcf..85c777729 100644 --- a/src/conventional/buoy_bufr2ioda.py +++ b/src/conventional/buoy_bufr2ioda.py @@ -71,12 +71,12 @@ obserrlist = [1.2, 0.75E-3, 2.2, 1.7, 1.7, 120.0] VarDims = { - 'airTemperature': ['nlocs'], - 'specificHumidity': ['nlocs'], - 'seaSurfaceTemperature': ['nlocs'], - 'windEastward': ['nlocs'], - 'windNorthward': ['nlocs'], - 'stationPressure': ['nlocs'] + 'airTemperature': ['Location'], + 'specificHumidity': ['Location'], + 'seaSurfaceTemperature': ['Location'], + 'windEastward': ['Location'], + 'windNorthward': ['Location'], + 'stationPressure': ['Location'] } metaDataName = iconv.MetaDataName() @@ -89,7 +89,7 @@ 'ioda_version': 2, 'description': 'Surface (Ship) observations converted from BUFR', 'source': 'LDM at NCAR-RAL', - 'source_files': '' + 'sourceFiles': '' } DimDict = { @@ -136,12 +136,12 @@ def main(file_names, output_file): for fname in file_names: logging.debug("Reading file: " + fname) - AttrData['source_files'] += ", " + fname + AttrData['sourceFiles'] += ", " + fname data, count, start_pos = read_file(fname, count, start_pos, data) - AttrData['source_files'] = AttrData['source_files'][2:] - logging.debug("All source files: " + AttrData['source_files']) + AttrData['sourceFiles'] = AttrData['sourceFiles'][2:] + logging.debug("All source files: " + AttrData['sourceFiles']) if not data: logging.critical("ABORT: no message data was captured, stopping execution.") @@ -149,8 +149,7 @@ def main(file_names, output_file): logging.info("--- {:9.4f} BUFR read seconds ---".format(time.time() - start_time)) nlocs = len(data['dateTime']) - DimDict = {'nlocs': nlocs} - AttrData['nlocs'] = np.int32(DimDict['nlocs']) + DimDict = {'Location': nlocs} # Set coordinates and units of the ObsValues. for n, iodavar in enumerate(obsvars): diff --git a/src/conventional/decode_bufr_LDM_raob.py b/src/conventional/decode_bufr_LDM_raob.py deleted file mode 100644 index 7d84808b9..000000000 --- a/src/conventional/decode_bufr_LDM_raob.py +++ /dev/null @@ -1,797 +0,0 @@ -#!/usr/bin/env python3 - -from datetime import datetime -import dateutil.parser -import os -from pathlib import Path -import sys -import time -import logging - -from itertools import compress -import numpy as np -import netCDF4 as nc -import eccodes as ecc - -# set path to ioda_conv_engines module -IODA_CONV_PATH = Path(__file__).parent/"@SCRIPT_LIB_PATH@" -if not IODA_CONV_PATH.is_dir(): - IODA_CONV_PATH = Path(__file__).parent/'..'/'lib-python' -sys.path.append(str(IODA_CONV_PATH.resolve())) - -# These modules need the path to lib-python modules -import ioda_conv_engines as iconv -import meteo_utils -from orddicts import DefaultOrderedDict -from collections import defaultdict, OrderedDict - -os.environ["TZ"] = "UTC" - -locationKeyList = [ - ("latitude", "float", "degrees_north", "keep"), - ("longitude", "float", "degrees_east", "keep"), - ("station_elevation", "float", "m", "keep"), - ("dateTime", "long", "seconds since 1970-01-01T00:00:00Z", "keep"), - ("LaunchTime", "long", "seconds since 1970-01-01T00:00:00Z", "keep"), - ("air_pressure", "float", "Pa", "keep"), - ("geopotential_height", "float", "m", "keep"), - ("vertSignificance", "integer", "", "toss"), - ("latDisplacement", "float", "degrees", "toss"), - ("lonDisplacement", "float", "degrees", "toss"), - ("timeDisplacement", "float", "s", "toss"), - ("wmoBlockNumber", "integer", "", "toss"), - ("wmoStationNumber", "integer", "", "toss"), - ("station_id", "string", "", "keep"), - ("year", "integer", "", "toss"), - ("month", "integer", "", "toss"), - ("day", "integer", "", "toss"), - ("hour", "integer", "", "toss"), - ("minute", "integer", "", "toss"), - ("second", "integer", "", "toss"), -] -meta_keys = [m_item[0] for m_item in locationKeyList] - -metaDataKeyList = { - 'latitude': ['latitude'], - 'longitude': ['longitude'], - 'station_elevation': ['Constructed', 'heightOfBarometerAboveMeanSeaLevel', - 'heightOfStationGroundAboveMeanSeaLevel', 'heightOfStation', 'height'], - 'dateTime': ['Constructed'], - 'LaunchTime': ['Constructed'], - 'air_pressure': ['pressure', 'nonCoordinatePressure'], - 'geopotential_height': ['nonCoordinateGeopotentialHeight', 'geopotentialHeight'], - 'vertSignificance': ['extendedVerticalSoundingSignificance', 'verticalSoundingSignificance'], - 'latDisplacement': ['latitudeDisplacement'], - 'lonDisplacement': ['longitudeDisplacement'], - 'timeDisplacement': ['timePeriod'], - 'wmoBlockNumber': ['blockNumber'], - 'wmoStationNumber': ['stationNumber'], - 'station_id': ['Constructed'], - # "stationLongName": 'shipOrMobileLandStationIdentifier', - # "instrumentType": 'radiosondeType', - # "instrumentSerialNum": 'radiosondeSerialNumber', - # "instrumentSoftwareVersion": 'softwareVersionNumber', - # "instrumentHumidityCorrectionInfo": 'correctionAlgorithmsForHumidityMeasurements', - # "instrumentRadiationCorrectionInfo": 'solarAndInfraredRadiationCorrection', - 'year': ['year'], - 'month': ['month'], - 'day': ['day'], - 'hour': ['hour'], - 'minute': ['minute'], - 'second': ['second'], -} - -# True incoming BUFR observed variables. -raw_obsvars = ['airTemperature', 'dewpointTemperature', 'windDirection', 'windSpeed'] - -# The outgoing IODA variables (ObsValues), their units, and assigned constant ObsError. -obsvars = ['air_temperature', 'virtual_temperature', 'specific_humidity', 'eastward_wind', 'northward_wind'] -obsvars_units = ['K', 'K', 'kg kg-1', 'm s-1', 'm s-1'] -obserrlist = [1.2, 1.2, 0.75E-3, 1.7, 1.7] - -VarDims = { - 'air_temperature': ['nlocs'], - 'virtual_temperature': ['nlocs'], - 'specific_humidity': ['nlocs'], - 'eastward_wind': ['nlocs'], - 'northward_wind': ['nlocs'] -} - -AttrData = { - 'converter': os.path.basename(__file__), - 'ioda_version': 2, - 'description': 'Radiosonde observations converted from BUFR', - 'source': 'LDM at NCAR-RAL', - 'sourceFiles': '' -} - -DimDict = { -} - -metaDataName = iconv.MetaDataName() -obsValName = iconv.OvalName() -obsErrName = iconv.OerrName() -qcName = iconv.OqcName() - -float_missing_value = nc.default_fillvals['f4'] -int_missing_value = nc.default_fillvals['i4'] -double_missing_value = nc.default_fillvals['f8'] -long_missing_value = nc.default_fillvals['i8'] -string_missing_value = '_' -bufr_missing_value = ecc.CODES_MISSING_LONG - -missing_vals = {'string': string_missing_value, - 'integer': int_missing_value, - 'long': long_missing_value, - 'float': float_missing_value, - 'double': double_missing_value} -dtypes = {'string': object, - 'integer': np.int32, - 'long': np.int64, - 'float': np.float32, - 'double': np.float64} - -iso8601_string = locationKeyList[meta_keys.index('dateTime')][2] -epoch = datetime.fromisoformat(iso8601_string[14:-1]) - - -def main(file_names, output_file, datetimeRef): - - # initialize - count = [1, 0, 0, 0] - start_time = time.time() - varDict = defaultdict(lambda: DefaultOrderedDict(dict)) - varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) - - obs_data = {} # The final outputs. - data = {} # Before assigning the output types into the above. - for key in obsvars: - data[key] = [] - for key in meta_keys: - data[key] = [] - - # Open one input file at a time. - for fname in file_names: - AttrData['sourceFiles'] += ", " + fname - - # Read from a BUFR file. - data, count = read_file(fname, count, data) - - AttrData['datetimeReference'] = datetimeRef - AttrData['sourceFiles'] = AttrData['sourceFiles'][2:] - logging.debug("All source files: " + AttrData['sourceFiles']) - - if not data: - logging.critical("ABORT: no message data was captured, stopping execution.") - sys.exit() - - logging.info("--- {:9.4f} BUFR read seconds ---".format(time.time() - start_time)) - - nlocs = count[1] - DimDict = {'nlocs': nlocs} - - # Set coordinates and units of the ObsValues. - for n, iodavar in enumerate(obsvars): - varDict[iodavar]['valKey'] = iodavar, obsValName - varDict[iodavar]['errKey'] = iodavar, obsErrName - varDict[iodavar]['qcKey'] = iodavar, qcName - varAttrs[iodavar, obsValName]['coordinates'] = 'longitude latitude' - varAttrs[iodavar, obsErrName]['coordinates'] = 'longitude latitude' - varAttrs[iodavar, qcName]['coordinates'] = 'longitude latitude' - varAttrs[iodavar, obsValName]['units'] = obsvars_units[n] - varAttrs[iodavar, obsErrName]['units'] = obsvars_units[n] - - # Set units of the MetaData variables and all _FillValues. - for key in meta_keys: - dtypestr = locationKeyList[meta_keys.index(key)][1] - keep_or_toss = locationKeyList[meta_keys.index(key)][3] - if (keep_or_toss == "keep"): - if locationKeyList[meta_keys.index(key)][2]: - varAttrs[(key, metaDataName)]['units'] = locationKeyList[meta_keys.index(key)][2] - varAttrs[(key, metaDataName)]['_FillValue'] = missing_vals[dtypestr] - obs_data[(key, metaDataName)] = np.array(data[key], dtype=dtypes[dtypestr]) - - # Transfer from the 1-D data vectors and ensure output data (obs_data) types using numpy. - for n, iodavar in enumerate(obsvars): - obs_data[(iodavar, obsValName)] = np.array(data[iodavar], dtype=np.float32) - obs_data[(iodavar, obsErrName)] = np.full(nlocs, obserrlist[n], dtype=np.float32) - obs_data[(iodavar, qcName)] = np.full(nlocs, 2, dtype=np.int32) - - logging.debug("Writing file: " + output_file) - - # setup the IODA writer - writer = iconv.IodaWriter(output_file, locationKeyList, DimDict) - - # write everything out - writer.BuildIoda(obs_data, VarDims, varAttrs, AttrData) - - logging.info("--- {:9.4f} total seconds ---".format(time.time() - start_time)) - - -def read_file(file_name, count, data): - - f = open(file_name, 'rb') - fsize = os.path.getsize(file_name) - logging.info(f"Processing file {file_name} with size: {fsize}") - start_pos = 0 - - while True: - logging.info(f" within file, reading BUFR msg ({count[0]}) starting at: {start_pos}") - # Use eccodes to decode each bufr message in the file - data, count, start_pos = read_bufr_message(f, count, start_pos, data) - count[0] += 1 - - if ((start_pos is None) or (start_pos >= fsize)): - break - - return data, count - - -# Replace BUFR missing value indicators with IODA missings. -def assign_values(data, key): - - if isinstance(data[0], str): - for n, d in enumerate(data): - data[n] = ''.join(c for c in d if c.isalnum()) - if not data[n]: - data[n] = string_missing_value - return np.array(data, dtype=object) - - if (data.dtype == np.float32 or data.dtype == np.float64): - if not (key in meta_keys): - data[np.abs(data) >= np.abs(bufr_missing_value)] = float_missing_value - return np.array(data, dtype=np.float32) - elif (locationKeyList[meta_keys.index(key)][1] == "float"): - data[np.abs(data) >= np.abs(bufr_missing_value)] = float_missing_value - return np.array(data, dtype=np.float32) - elif (locationKeyList[meta_keys.index(key)][1] == "double"): - data[np.abs(data) >= np.abs(bufr_missing_value)] = double_missing_value - return np.array(data, dtype=np.float64) - elif (locationKeyList[meta_keys.index(key)][1] == "integer"): - data[np.abs(data) >= np.abs(bufr_missing_value)] = int_missing_value - return np.array(data, dtype=np.int32) - elif (locationKeyList[meta_keys.index(key)][1] == "long"): - data[np.abs(data) >= np.abs(bufr_missing_value)] = long_missing_value - return np.array(data, dtype=np.int64) - elif (data.dtype == np.int32 or data.dtype == np.int64): - if not (key in meta_keys): - data[np.abs(data) >= np.abs(bufr_missing_value)] = int_missing_value - return np.array(data, dtype=np.int32) - if (locationKeyList[meta_keys.index(key)][1] == "integer"): - data[np.abs(data) >= np.abs(bufr_missing_value)] = int_missing_value - return np.array(data, dtype=np.int32) - elif (locationKeyList[meta_keys.index(key)][1] == "long"): - data[np.abs(data) >= np.abs(bufr_missing_value)] = long_missing_value - return np.array(data, dtype=np.int64) - elif (locationKeyList[meta_keys.index(key)][1] == "float"): - data = data.astype(np.float32) - data[np.abs(data) >= np.abs(bufr_missing_value)] = float_missing_value - return np.array(data, dtype=np.float32) - elif data.dtype.kind in {'U', 'S'}: - return np.array(data, dtype=object) - - logging.critical("ABORT, no matching datatype found for key: " + key) - - -# Assign the correct IODA missing value to MetaData element. -def assign_missing_meta(data, key, num, start): - - dtypestr = locationKeyList[meta_keys.index(key)][1] - if dtypestr is None: - logging.critical("ABORT, no matching datatype found for key: " + key) - else: - if start == 0: - return np.full(num, missing_vals[dtypestr], dtype=dtypes[dtypestr]) - else: - out_data = np.full(num, missing_vals[dtypestr], dtype=dtypes[dtypestr]) - for n in range(0, start): - out_data[n] = data[n] - return out_data - - -# Return True if all elements in the array are missing, otherwise False. -def is_all_missing(data): - - if data.dtype == np.float32: - return all(x == float_missing_value for x in data) - if data.dtype == np.float64: - return all(x == double_missing_value for x in data) - elif data.dtype == np.int32: - return all(x == int_missing_value for x in data) - elif data.dtype == np.int64: - return all(x == long_missing_value for x in data) - elif data.dtype.kind in {'U', 'S'}: - return all(x == string_missing_value for x in data) - - logging.warning("data type not determined, returning False") - return False - - -def get_normalized_bit(value, bit_index): - return (value >> bit_index) & 1 - - -def def_significance_table(): - - significance_table = { - "pressure level originally indicated by height as the veritcal coordinate": 1, - "freezing level": 2, - "level determined by regional decision": 3, - "top of wind sounding": 4, - "end of missing wind data": 5, - "beginning of missing wind data": 6, - "end of missing humidity data": 7, - "beginning of missing humidity data": 8, - "end of missing temperature data": 9, - "beginning of missing temperature data": 10, - "significant wind level": 11, - "significant humidity level": 12, - "significant temperature level": 13, - "maximum wind level": 14, - "tropopause level": 15, - "standard level": 16, - "surface": 17, - } - - return significance_table - - -def specialty_time(tvals, year, month, day, hour, minute, second): - - bad_date = False - result = [] - - # UGH, some BUFR uses 2-digit year, awful. - if (year >= 0 and year <= 50): - year += 2000 - elif (year > 50 and year <= 99): - year += 1900 - - if minute == int_missing_value: - minute = 0 - elif minute < 0 or minute > 59: - minute = 0 - - if second == int_missing_value: - second = 0 - elif second < 0 or second > 59: - second = 0 - - if (year < 1900 or year > 2499 or month < 1 or month > 12 or day < 1 or day > 31 or hour < 0 or hour > 23 or minute < 0 or minute > 59): - bad_date = True - - if bad_date: - year = 1900 - month = 1 - day = 1 - hour = 0 - minute = 0 - - # This should be the launch or release time of sonde. - logging.debug(f"Launch time info {year}-{month}-{day}T{hour}:{minute}:{second}Z") - try: - this_datetime = datetime(year, month, day, hour, minute, second) - except Exception: - logging.critical(f"Bogus launch time info {year}-{month}-{day}T{hour}:{minute}:{second}Z") - year = 1900 - month = 1 - day = 1 - hour = 0 - minute = 0 - this_datetime = datetime(year, month, day, hour, minute, second) - - time_offset = round((this_datetime - epoch).total_seconds()) - - result = np.full(len(tvals), time_offset) - for n, tval in enumerate(tvals): - if (tval < 0 or tval > 6*3600): - result[n] = result[n-1] - else: - result[n] = time_offset + tval - - return result - - -def read_bufr_message(f, count, start_pos, data): - - temp_data = {} # A temporary dictionary to hold things. - meta_data = {} # All the various MetaData go in here. - vals = {} # Floating-point ObsValues variables go in here. - avals = [] # Temporarily hold array of values. - start_pos = f.tell() - - met_utils = meteo_utils.meteo_utils() - significance_table = def_significance_table() - - try: - bufr = ecc.codes_bufr_new_from_file(f) - try: - msg_size = ecc.codes_get_message_size(bufr) - logging.info(f" will attempt to read BUFR msg with size: ({msg_size} bytes)") - except: # noqa - return data, count, None - except ecc.CodesInternalError: - logging.critical(f"Useless BUFR message (codes_bufr_new_from_file)") - start_pos = f.tell() - return data, count, start_pos - - try: - ecc.codes_set(bufr, 'skipExtraKeyAttributes', 1) # Supposedly this is ~25 percent faster - ecc.codes_set(bufr, 'unpack', 1) - except ecc.CodesInternalError: - ecc.codes_release(bufr) - logging.info(f"INCOMPLETE BUFR message, skipping ({msg_size} bytes)") - start_pos += int(0.5*msg_size) - f.seek(start_pos) - return data, count, start_pos - - # Some BUFR messages have subsets (multiple) soundings in a single message (China especially). - # Therefore we have to loop through a vector of the delayedDescriptorReplicationFactor. - try: - nsubsets = ecc.codes_get(bufr, 'numberOfSubsets') - except ecc.KeyValueNotFoundError: - nsubsets = 1 - pass - - # Are the data compressed or uncompressed? If the latter, then when nsubsets>1, we - # have to do things differently. - compressed = ecc.codes_get(bufr, 'compressedData') - - ''' - This will print absolutely every BUFR key in the message. - print(" ") - iterid = ecc.codes_keys_iterator_new(bufr) - while ecc.codes_keys_iterator_next(iterid): - keyname = ecc.codes_keys_iterator_get_name(iterid) - print(f" name: {keyname}") - ''' - - # If multiple soundings repfacs will be vector of length of each sounding. - repfacs = [] - try: - repfacs = ecc.codes_get_array(bufr, 'extendedDelayedDescriptorReplicationFactor').tolist() - except ecc.KeyValueNotFoundError: - try: - repfacs = ecc.codes_get_array(bufr, 'extendedDelayedDescriptorAndDataRepetitionFactor').tolist() - except ecc.KeyValueNotFoundError: - try: - repfacs = ecc.codes_get_array(bufr, 'delayedDescriptorReplicationFactor').tolist() - except ecc.KeyValueNotFoundError: - try: - repfacs = ecc.codes_get_array(bufr, 'delayedDescriptorAndDataRepetitionFactor').tolist() - except ecc.KeyValueNotFoundError: - try: - repfacs = ecc.codes_get_array(bufr, 'shortDelayedDescriptorReplicationFactor').tolist() - except ecc.KeyValueNotFoundError: - pass - - # First, get the MetaData we are interested in (list is in metaDataKeyList) - max_mlen = 0 - for k, v in metaDataKeyList.items(): - temp_data[k] = [] - if (len(v) > 1): - for var in v: - if (var != 'Constructed'): - try: - avals = ecc.codes_get_array(bufr, var) - max_mlen = max(max_mlen, len(avals)) - temp_data[k] = assign_values(avals, k) - if not is_all_missing(temp_data[k]): - break - except ecc.KeyValueNotFoundError: - logging.debug("Caution: unable to find requested BUFR key: " + var) - temp_data[k] = None - else: - temp_data[k] = None - else: - if (v[0] != 'Constructed'): - try: - avals = ecc.codes_get_array(bufr, v[0]) - max_mlen = max(max_mlen, len(avals)) - temp_data[k] = assign_values(avals, k) - except ecc.KeyValueNotFoundError: - logging.debug("Caution, unable to find requested BUFR key: " + v[0]) - temp_data[k] = None - else: - temp_data[k] = None - - # These meta data elements are so critical that we should quit quickly if lacking them: - if (temp_data['year'] is None) and (temp_data['month'] is None) and \ - (temp_data['day'] is None) and (temp_data['hour'] is None): - logging.warning("Useless ob without date info.") - if (temp_data['wmoBlockNumber'] is None) and (temp_data['wmoStationNumber'] is None) and \ - (temp_data['latitude'] is None) and (temp_data['longitude'] is None): - logging.warning("Useless ob without lat,lon or station number info.") - - # Next, get the raw observed weather variables we want. - # TO-DO: currently all ObsValue variables are float type, might need integer/other. - max_dlen = 0 - repfactors = {} - for variable in raw_obsvars: - temp_data[variable] = [] - if not compressed and nsubsets > 1: - repfactors[variable] = [] - for n in range(nsubsets): - var = '/subsetNumber=' + str(n+1) + '/' + variable - try: - avals = ecc.codes_get_array(bufr, var) - repfactors[variable].append(len(avals)) - temp_data[variable] = np.append(temp_data[variable], assign_values(avals, variable)) - except ecc.KeyValueNotFoundError: - logging.debug("Caution, unable to find requested BUFR variable: " + variable) - temp_data[variable] = None - else: - repfactors[variable] = [] - try: - avals = ecc.codes_get_array(bufr, variable) - repfactors[variable].append(len(avals)) - max_dlen = max(max_dlen, len(avals)) - temp_data[variable] = assign_values(avals, variable) - except ecc.KeyValueNotFoundError: - logging.debug("Caution, unable to find requested BUFR variable: " + variable) - temp_data[variable] = None - - if not repfacs: - for variable in raw_obsvars: - if repfactors[variable]: - repfacs = repfactors[variable] - break - - # From repfacs, make begin/end indicies for single or multiple soundings. - nbeg = [] - nend = [] - if repfacs: - if nsubsets > 1: - if nsubsets != len(repfacs): - logging.warning(f"Nonsense: number of subsets, {nsubsets} is not equal to " - f"the length of repfacs vector, {len(repfacs)}") - nend = np.cumsum(repfacs) - nbeg = np.insert(nend[:-1], 0, 0) - else: - nbeg.append(0) - nend.append(repfacs[0]-1) - else: - nbeg.append(0) - nend.append(int(1E6)) - - # Be done with this BUFR message. - ecc.codes_release(bufr) - - # Loop over each pair of beginning and ending indices and transfer data - # In some circumstances, we have no idea how many vertical levels of data in a sonde are - # upcoming, so we use 1E6 as largest possible number and hope to discover the true - # number from some other variable (the multi-IF-block test below). - obnum = 0 - for b, e in tuple(zip(nbeg, nend)): - if b < 0 or e < 0 or e <= b: - logging.warning(f"Skipping nonsense BUFR msg with a negative index [{b},{e}]") - return data, count, start_pos - logging.debug(f"Within BUFR msg, processing ob {obnum+1} with bounds: [{b},{e-1}]") - if e < 999999: - target_number = e - b - else: - logging.debug("Msg did not contain repfacs, trying to determine target number of obs") - if temp_data['vertSignificance'] is not None: - target_number = len(temp_data['vertSignificance']) - elif temp_data['timeDisplacement'] is not None: - target_number = len(temp_data['timeDisplacement']) - elif temp_data['latDisplacement'] is not None: - target_number = len(temp_data['latDisplacement']) - elif temp_data['air_pressure'] is not None: - target_number = len(temp_data['air_pressure']) - elif temp_data['airTemperature'] is not None: - target_number = len(temp_data['airTemperature']) - elif temp_data['windSpeed'] is not None: - target_number = len(temp_data['windSpeed']) - else: - print("HOW on earth is target_number zero? BUFR sucks!") - return data, count, start_pos - e = target_number - - # For any of the MetaData elements that were totally lacking, fill entire vector with missing. - empty = [] - for k, v in metaDataKeyList.items(): - meta_data[k] = assign_missing_meta(empty, k, target_number, 0) - if temp_data[k] is None: - next - elif b == 0 and len(temp_data[k]) == 1: - meta_data[k] = np.full(target_number, temp_data[k][0]) - else: - if len(temp_data[k]) == nsubsets: - meta_data[k] = np.full(target_number, temp_data[k][obnum]) - else: - try: - meta_data[k] = temp_data[k][b:e] - if len(meta_data[k]) < target_number: - meta_data[k] = np.full(target_number, meta_data[k][0]) - except Exception: - logging.warning(f"Failed copying temp_data to meta_data, var: {k}, ({b},{e}):{target_number}, len:{len(temp_data[k])}") - count[2] += target_number - return data, count, start_pos - - # Sondes are special with a launch time and time displacement. - if temp_data['timeDisplacement'] is not None: - meta_data['dateTime'] = specialty_time(temp_data['timeDisplacement'][b:e], - meta_data['year'][0], meta_data['month'][0], meta_data['day'][0], # noqa - meta_data['hour'][0], meta_data['minute'][0], meta_data['second'][0]) # noqa - meta_data['LaunchTime'] = np.full(target_number, meta_data['dateTime'][0]) - else: - meta_data['dateTime'][0] = specialty_time([0], - meta_data['year'][0], meta_data['month'][0], meta_data['day'][0], # noqa - meta_data['hour'][0], meta_data['minute'][0], meta_data['second'][0]) # noqa - meta_data['dateTime'] = np.full(target_number, meta_data['dateTime'][0]) - meta_data['LaunchTime'] = np.full(target_number, meta_data['dateTime'][0]) - - # Sondes also have lat/lon displacement from launch/release location. - if temp_data['latDisplacement'] is not None and temp_data['lonDisplacement'] is not None: - for n, delta_lat in enumerate(temp_data['latDisplacement'][b:e]): - delta_lon = temp_data['lonDisplacement'][n+b] - meta_data['latitude'][n] = meta_data['latitude'][0] + delta_lat - meta_data['longitude'][n] = meta_data['longitude'][0] + delta_lon - else: - meta_data['latitude'] = np.full(target_number, meta_data['latitude'][0]) - meta_data['longitude'] = np.full(target_number, meta_data['longitude'][0]) - - # Force longitude into space of -180 to +180 only. Reset both lat/lon missing if either absent. - mask_lat = np.logical_or(meta_data['latitude'] < -90.0, meta_data['latitude'] > 90.0) - mask_lon = np.logical_or(meta_data['longitude'] < -180.0, meta_data['longitude'] > 360.0) - meta_data['latitude'][mask_lat] = float_missing_value - meta_data['longitude'][mask_lon] = float_missing_value - meta_data['latitude'][mask_lon] = float_missing_value - meta_data['longitude'][mask_lat] = float_missing_value - for n, longitude in enumerate(meta_data['longitude']): - if (meta_data['longitude'][n] != float_missing_value and meta_data['longitude'][n] > 180): - meta_data['longitude'][n] = meta_data['longitude'][n] - 360.0 - - # It is NOT IDEAL, but if missing a lat or lon, fill with prior known value for now. - for n, lon in enumerate(meta_data['longitude']): - lat = meta_data['latitude'][n] - if (lon < -180 or lon > 180): - meta_data['longitude'][n] = meta_data['longitude'][n-1] - if (lat < -90 or lat > 90): - meta_data['latitude'][n] = meta_data['latitude'][n-1] - - # Forcably create station_id 5-char string from WMO block+station number. - meta_data['station_id'] = np.full(target_number, string_missing_value, dtype=' 0 and block < 100 and number > 0 and number < 1000): - meta_data['station_id'][n] = "{:02d}".format(block) + "{:03d}".format(number) - if n == 0: - count[3] += 1 - logging.info(f"Processing sonde for station: {meta_data['station_id'][n]}") - - # Very odd, sometimes the first level of data has some variables set to zero. Reset to missing. - if (meta_data['geopotential_height'][0] == 0 or meta_data['air_pressure'][0] == 0): - meta_data['geopotential_height'][0] = float_missing_value - meta_data['air_pressure'][0] = float_missing_value - - # And now processing the observed variables we care about. - nbad = 0 - for variable in raw_obsvars: - vals[variable] = np.full(target_number, float_missing_value) - if temp_data[variable] is not None: - try: - vals[variable] = temp_data[variable][b:e] - if len(vals[variable]) < target_number: - nbad += 1 - logging.warning(f" var {variable} has {len(vals[variable])} " - f"elements while expecting {target_number}") - vals[variable] = np.full(target_number, float_missing_value) - except Exception: - logging.warning(f"Unable to copy {variable} data, " - f"either index [{b},{e}] must be out of range.") - else: - nbad += 1 - - if nbad == len(raw_obsvars): - logging.warning(f"No usable data in this ob, skipping it.") - count[2] += target_number - return data, count, start_pos - - count[1] += target_number - - # Finally transfer the meta_data to the output array. - for key in meta_keys: - data[key] = np.append(data[key], meta_data[key]) - - ''' - Need to transform some variables (wind speed/direction to components for example). - In the ideal world, we could assume that the meteorological variables were given - well-bounded values, but in BUFR, they could be garbage, so ensure that values - are all sensible before calling the transformation functions. - ''' - uwnd = np.full(target_number, float_missing_value) - vwnd = np.full(target_number, float_missing_value) - for n, wdir in enumerate(vals['windDirection']): - wspd = vals['windSpeed'][n] - if wdir and wspd: - if (wdir >= 0 and wdir <= 360 and wspd >= 0 and wspd < 300): - uwnd[n], vwnd[n] = met_utils.dir_speed_2_uv(wdir, wspd) - - spfh = np.full(target_number, float_missing_value) - for n, dewpoint in enumerate(vals['dewpointTemperature']): - pres = meta_data['air_pressure'][n] - if dewpoint and pres: - if (dewpoint > 50 and dewpoint < 325 and pres > 100 and pres < 109900): - spfh[n] = met_utils.specific_humidity(dewpoint, pres) - - airt = np.full(target_number, float_missing_value) - for n, temp in enumerate(vals['airTemperature']): - if temp: - if (temp > 50 and temp < 345): - airt[n] = temp - - tvirt = np.full(target_number, float_missing_value) - for n, temp in enumerate(airt): - pres = meta_data['air_pressure'][n] - if (temp != float_missing_value and spfh[n] and pres < 108000 and pres > 10000): - qvapor = max(1.0e-12, spfh[n]/(1.0-spfh[n])) - tvirt[n] = temp*(1.0 + 0.61*qvapor) - - # Finally fill up the output data dictionary with observed variables. - data['eastward_wind'] = np.append(data['eastward_wind'], uwnd) - data['northward_wind'] = np.append(data['northward_wind'], vwnd) - data['specific_humidity'] = np.append(data['specific_humidity'], spfh) - data['air_temperature'] = np.append(data['air_temperature'], airt) - data['virtual_temperature'] = np.append(data['virtual_temperature'], tvirt) - - obnum += 1 - - logging.info("number of observations so far: " + str(count[1])) - logging.info("number of invalid or useless observations: " + str(count[2])) - logging.info("number of sonde locations: " + str(count[3])) - start_pos = f.tell() - return data, count, start_pos - - -if __name__ == "__main__": - - from argparse import ArgumentParser - - parser = ArgumentParser( - description=( - 'Read a raob BUFR file and convert into IODA output file') - ) - - required = parser.add_argument_group(title='required arguments') - required.add_argument('-i', '--input-files', nargs='+', dest='file_names', - action='store', default=None, required=True, - help='input files') - required.add_argument('-o', '--output-file', dest='output_file', - action='store', default=None, required=True, - help='output file') - - parser.set_defaults(debug=False) - parser.set_defaults(verbose=False) - parser.set_defaults(datetimeReference=" ") - optional = parser.add_argument_group(title='optional arguments') - optional.add_argument('--debug', action='store_true', - help='enable debug messages') - optional.add_argument('--verbose', action='store_true', - help='enable verbose debug messages') - optional.add_argument('--date', dest='datetimeReference', - action='store', default=' ', - help='date reference string (ISO8601)') - - args = parser.parse_args() - - if args.debug: - logging.basicConfig(level=logging.INFO) - elif args.verbose: - logging.basicConfig(level=logging.DEBUG) - else: - logging.basicConfig(level=logging.ERROR) - - for file_name in args.file_names: - if not os.path.isfile(file_name): - parser.error('Input (-i option) file: ', file_name, ' does not exist') - - args.output_file = os.path.abspath(args.output_file) - apath, afile = os.path.split(args.output_file) - # create output directory path if necessary - if not os.path.exists(apath): - print("creating output directory: ", apath) - os.makedirs(apath) - - main(args.file_names, args.output_file, args.datetimeReference) diff --git a/src/conventional/ship_bufr2ioda.py b/src/conventional/ship_bufr2ioda.py index 67adefc5a..64bc156cc 100644 --- a/src/conventional/ship_bufr2ioda.py +++ b/src/conventional/ship_bufr2ioda.py @@ -76,12 +76,12 @@ obserrlist = [1.2, 0.75E-3, 2.2, 1.7, 1.7, 120.0] VarDims = { - 'airTemperature': ['nlocs'], - 'specificHumidity': ['nlocs'], - 'seaSurfaceTemperature': ['nlocs'], - 'windEastward': ['nlocs'], - 'windNorthward': ['nlocs'], - 'stationPressure': ['nlocs'] + 'airTemperature': ['Location'], + 'specificHumidity': ['Location'], + 'seaSurfaceTemperature': ['Location'], + 'windEastward': ['Location'], + 'windNorthward': ['Location'], + 'stationPressure': ['Location'] } metaDataName = iconv.MetaDataName() @@ -94,7 +94,7 @@ 'ioda_version': 2, 'description': 'Surface (Ship) observations converted from BUFR', 'source': 'LDM at NCAR-RAL', - 'source_files': '' + 'sourceFiles': '' } DimDict = { @@ -141,12 +141,12 @@ def main(file_names, output_file): for fname in file_names: logging.debug("Reading file: " + fname) - AttrData['source_files'] += ", " + fname + AttrData['sourceFiles'] += ", " + fname data, count, start_pos = read_file(fname, count, start_pos, data) - AttrData['source_files'] = AttrData['source_files'][2:] - logging.debug("All source files: " + AttrData['source_files']) + AttrData['sourceFiles'] = AttrData['sourceFiles'][2:] + logging.debug("All source files: " + AttrData['sourceFiles']) if not data: logging.critical("ABORT: no message data was captured, stopping execution.") @@ -154,8 +154,7 @@ def main(file_names, output_file): logging.info("--- {:9.4f} BUFR read seconds ---".format(time.time() - start_time)) nlocs = len(data['dateTime']) - DimDict = {'nlocs': nlocs} - AttrData['nlocs'] = np.int32(DimDict['nlocs']) + DimDict = {'Location': nlocs} # Set coordinates and units of the ObsValues. for n, iodavar in enumerate(obsvars): diff --git a/src/conventional/sonde_bufr2ioda.py b/src/conventional/sonde_bufr2ioda.py index 83a22c952..573d0cf00 100644 --- a/src/conventional/sonde_bufr2ioda.py +++ b/src/conventional/sonde_bufr2ioda.py @@ -32,7 +32,7 @@ ("longitude", "float", "degrees_east", "keep"), ("stationElevation", "float", "m", "keep"), ("dateTime", "long", "seconds since 1970-01-01T00:00:00Z", "keep"), - ("releaseTime", "long", "seconds since 1970-01-01T00:00:00Z", "keep"), + ("launchTime", "long", "seconds since 1970-01-01T00:00:00Z", "keep"), ("pressure", "float", "Pa", "keep"), ("geopotentialHeight", "float", "m", "keep"), ("vertSignificance", "integer", "", "toss"), @@ -41,7 +41,12 @@ ("timeDisplacement", "float", "s", "toss"), ("wmoBlockNumber", "integer", "", "toss"), ("wmoStationNumber", "integer", "", "toss"), - ("stationWMO", "string", "", "keep"), + ("stationIdentification", "string", "", "keep"), + ("instrumentType", "integer", "", "keep"), + ("instrumentRadiationCorrectionInfo", "integer", "", "keep"), + ("instrumentHumidityCorrectionInfo", "integer", "", "keep"), + ("temperatureSensorType", "integer", "", "keep"), + ("humiditySensorType", "integer", "", "keep"), ("year", "integer", "", "toss"), ("month", "integer", "", "toss"), ("day", "integer", "", "toss"), @@ -57,7 +62,7 @@ 'stationElevation': ['Constructed', 'heightOfBarometerAboveMeanSeaLevel', 'heightOfStationGroundAboveMeanSeaLevel', 'heightOfStation', 'height'], 'dateTime': ['Constructed'], - 'releaseTime': ['Constructed'], + 'launchTime': ['Constructed'], 'pressure': ['pressure', 'nonCoordinatePressure'], 'geopotentialHeight': ['nonCoordinateGeopotentialHeight', 'geopotentialHeight'], 'vertSignificance': ['extendedVerticalSoundingSignificance', 'verticalSoundingSignificance'], @@ -66,13 +71,15 @@ 'timeDisplacement': ['timePeriod'], 'wmoBlockNumber': ['blockNumber'], 'wmoStationNumber': ['stationNumber'], - 'stationWMO': ['Constructed'], + 'stationIdentification': ['Constructed'], # "stationLongName": 'shipOrMobileLandStationIdentifier', - # "instrumentType": 'radiosondeType', + "instrumentType": ['radiosondeType'], + "instrumentRadiationCorrectionInfo": ['solarAndInfraredRadiationCorrection'], + "instrumentHumidityCorrectionInfo": ['correctionAlgorithmsForHumidityMeasurements'], + "temperatureSensorType": ['temperatureSensorType'], + "humiditySensorType": ['humiditySensorType'], # "instrumentSerialNum": 'radiosondeSerialNumber', # "instrumentSoftwareVersion": 'softwareVersionNumber', - # "instrumentHumidityCorrectionInfo": 'correctionAlgorithmsForHumidityMeasurements', - # "instrumentRadiationCorrectionInfo": 'solarAndInfraredRadiationCorrection', 'year': ['year'], 'month': ['month'], 'day': ['day'], @@ -85,12 +92,13 @@ raw_obsvars = ['airTemperature', 'dewpointTemperature', 'windDirection', 'windSpeed'] # The outgoing IODA variables (ObsValues), their units, and assigned constant ObsError. -obsvars = ['airTemperature', 'specificHumidity', 'windEastward', 'windNorthward'] -obsvars_units = ['K', 'kg kg-1', 'm s-1', 'm s-1'] -obserrlist = [1.2, 0.75E-3, 1.7, 1.7] +obsvars = ['airTemperature', 'virtualTemperature', 'specificHumidity', 'windEastward', 'windNorthward'] +obsvars_units = ['K', 'K', 'kg kg-1', 'm s-1', 'm s-1'] +obserrlist = [1.2, 1.2, 0.75E-3, 1.7, 1.7] VarDims = { 'airTemperature': ['Location'], + 'virtualTemperature': ['Location'], 'specificHumidity': ['Location'], 'windEastward': ['Location'], 'windNorthward': ['Location'] @@ -436,6 +444,11 @@ def read_bufr_message(f, count, start_pos, data): # have to do things differently. compressed = ecc.codes_get(bufr, 'compressedData') + # Unfortunately, there is absolutely no way to handle compressed data with number of subsets>1 + if compressed and nsubsets > 1: + logging.warning(f"CANNOT handle compressed data, skipping ({msg_size} bytes)") + return data, count, start_pos + ''' This will print absolutely every BUFR key in the message. print(" ") @@ -577,7 +590,7 @@ def read_bufr_message(f, count, start_pos, data): elif temp_data['latDisplacement'] is not None: target_number = len(temp_data['latDisplacement']) elif temp_data['pressure'] is not None: - target_number = len(temp_data['pressure']) + target_number = len(temp_data['air_pressure']) elif temp_data['airTemperature'] is not None: target_number = len(temp_data['airTemperature']) elif temp_data['windSpeed'] is not None: @@ -613,13 +626,13 @@ def read_bufr_message(f, count, start_pos, data): meta_data['dateTime'] = specialty_time(temp_data['timeDisplacement'][b:e], meta_data['year'][0], meta_data['month'][0], meta_data['day'][0], # noqa meta_data['hour'][0], meta_data['minute'][0], meta_data['second'][0]) # noqa - meta_data['releaseTime'] = np.full(target_number, meta_data['dateTime'][0]) + meta_data['launchTime'] = np.full(target_number, meta_data['dateTime'][0]) else: meta_data['dateTime'][0] = specialty_time([0], meta_data['year'][0], meta_data['month'][0], meta_data['day'][0], # noqa meta_data['hour'][0], meta_data['minute'][0], meta_data['second'][0]) # noqa meta_data['dateTime'] = np.full(target_number, meta_data['dateTime'][0]) - meta_data['releaseTime'] = np.full(target_number, meta_data['dateTime'][0]) + meta_data['launchTime'] = np.full(target_number, meta_data['dateTime'][0]) # Sondes also have lat/lon displacement from launch/release location. if temp_data['latDisplacement'] is not None and temp_data['lonDisplacement'] is not None: @@ -650,15 +663,15 @@ def read_bufr_message(f, count, start_pos, data): if (lat < -90 or lat > 90): meta_data['latitude'][n] = meta_data['latitude'][n-1] - # Forcably create station_id 5-char string from WMO block+station number. - meta_data['stationWMO'] = np.full(target_number, string_missing_value, dtype=' 0 and block < 100 and number > 0 and number < 1000): - meta_data['stationWMO'][n] = "{:02d}".format(block) + "{:03d}".format(number) + meta_data['stationIdentification'][n] = "{:02d}".format(block) + "{:03d}".format(number) if n == 0: count[3] += 1 - logging.info(f"Processing sonde for station: {meta_data['stationWMO'][n]}") + logging.info(f"Processing sonde for station: {meta_data['stationIdentification'][n]}") # Very odd, sometimes the first level of data has some variables set to zero. Reset to missing. if (meta_data['geopotentialHeight'][0] == 0 or meta_data['pressure'][0] == 0): @@ -721,11 +734,19 @@ def read_bufr_message(f, count, start_pos, data): if (temp > 50 and temp < 345): airt[n] = temp + tvirt = np.full(target_number, float_missing_value) + for n, temp in enumerate(airt): + pres = meta_data['pressure'][n] + if (temp != float_missing_value and spfh[n] and pres < 108000 and pres > 10000): + qvapor = max(1.0e-12, spfh[n]/(1.0-spfh[n])) + tvirt[n] = temp*(1.0 + 0.61*qvapor) + # Finally fill up the output data dictionary with observed variables. data['windEastward'] = np.append(data['windEastward'], uwnd) data['windNorthward'] = np.append(data['windNorthward'], vwnd) data['specificHumidity'] = np.append(data['specificHumidity'], spfh) data['airTemperature'] = np.append(data['airTemperature'], airt) + data['virtualTemperature'] = np.append(data['virtualTemperature'], tvirt) obnum += 1 diff --git a/src/conventional/sonde_tac2ioda.py b/src/conventional/sonde_tac2ioda.py new file mode 100644 index 000000000..cb1aa19dd --- /dev/null +++ b/src/conventional/sonde_tac2ioda.py @@ -0,0 +1,1195 @@ +########################################################################### +# These functions decode WMO format soundings which contain at least the +# mandatory levels (TTAA, TTCC), and also can include significant temperature +# (TTBB, TTDD) and wind (PPBB, PPDD) sections. The sections can be merged into +# a single profile, which interpolates any missing values if possible. +# The code is based on Fortran code originally developed by Peter Neilley, NCAR/RAP. +########################################################################### + +import re +import logging +import math +import os +import sys +import time +import json +from datetime import datetime, timedelta +import dateutil.parser +from pathlib import Path + +import numpy as np +import netCDF4 as nc +from cartopy import geodesic +from copy import deepcopy as dcop + +# set path to ioda_conv_engines module +IODA_CONV_PATH = Path(__file__).parent/"@SCRIPT_LIB_PATH@" +if not IODA_CONV_PATH.is_dir(): + IODA_CONV_PATH = Path(__file__).parent/'..'/'lib-python' +sys.path.append(str(IODA_CONV_PATH.resolve())) + +# These modules need the path to lib-python modules +from collections import defaultdict, OrderedDict +from orddicts import DefaultOrderedDict +import ioda_conv_engines as iconv +import meteo_utils +import meteo_sounding_utils + +os.environ["TZ"] = "UTC" + +logger = logging.getLogger("decodeSounding") + +# Abbreviations for pressure levels found in the WMO datasets +LEVEL_CODES = { + '99': 0, + '00': 1000.0, + '92': 925.0, + '85': 850.0, + '70': 700.0, + '50': 500.0, + '40': 400.0, + '30': 300.0, + '25': 250.0, + '20': 200.0, + '15': 150.0, + '10': 100.0, + '07': 70.0 +} + +# List of sounding data sections +TYPES = ['TTAA', 'TTBB', 'PPBB', 'TTCC', 'TTDD', 'PPDD'] + +STATIONS = {} + +# The outgoing IODA MetaData variables, their data type, and units. +MetaDataKeyList = [ + ("stationIdentification", "string", ""), + ("latitude", "float", "degrees_north"), + ("longitude", "float", "degrees_east"), + ("stationElevation", "float", "m"), + ("height", "float", "m"), + ("pressure", "float", "Pa"), + ("launchTime", "string", ""), + ("dateTime", "long", "seconds since 1970-01-01T00:00:00Z"), +] +meta_keys = [m_item[0] for m_item in MetaDataKeyList] + +# The outgoing IODA variables (ObsValues), their units, and assigned constant ObsError. +obsvars = ['airTemperature', + 'specificHumidity', + 'virtualTemperature', + 'windEastward', + 'windNorthward'] +obsvars_units = ['K', 'kg kg-1', 'K', 'm s-1', 'm s-1'] +obserrlist = [1.2, 0.75E-3, 1.5, 1.7, 1.7] + +VarDims = { + 'airTemperature': ['Location'], + 'specificHumidity': ['Location'], + 'virtualTemperature': ['Location'], + 'windEastward': ['Location'], + 'windNorthward': ['Location'] +} + +metaDataName = iconv.MetaDataName() +obsValName = iconv.OvalName() +obsErrName = iconv.OerrName() +qcName = iconv.OqcName() + +AttrData = { + 'converter': os.path.basename(__file__), + 'ioda_version': 2, + 'description': 'Radiosonde observations converted from text (TAC) format', + 'source': 'LDM at NCAR-RAL', + 'sourceFiles': '' +} + +DimDict = { +} + +float_missing_value = nc.default_fillvals['f4'] +int_missing_value = nc.default_fillvals['i4'] +double_missing_value = nc.default_fillvals['f8'] +long_missing_value = nc.default_fillvals['i8'] +string_missing_value = '_' + +iso8601_string = MetaDataKeyList[meta_keys.index('dateTime')][2] +epoch = datetime.fromisoformat(iso8601_string[14:-1]) + +missing_vals = {'string': string_missing_value, + 'integer': int_missing_value, + 'long': long_missing_value, + 'float': float_missing_value, + 'double': double_missing_value} +dtypes = {'string': object, + 'integer': np.int32, + 'long': np.int64, + 'float': np.float32, + 'double': np.float64} + +geod = geodesic.Geodesic() # generate ellipsoid, defaults to Earth WGS84 parameters + + +def loadStations(stationfile, skipIfLoaded=True): + """ + Load station info from a .json file into the global station list. This info is needed + when parsing WMO data to determine surface elevation at each site + :param stationfile: The filename to load + :param skipIfLoaded: If True, don't load the station file if one has already been loaded + :return: True on success, False on error + """ + + global STATIONS + + if skipIfLoaded and len(STATIONS) > 0: + return True + + try: + fh = open(stationfile, "r") + data = fh.read() + fh.close() + STATIONS = json.loads(data) + return True + except Exception as e: + logger.error("Could not read station info from json file '%s': %s" % (stationfile, e)) + return False + + +def getStationInfo(icaoId=None, synopId=None): + """ + Get info for a station given either an icaoId or a synopId + :param icaoId: + :param synopId: + :return: A dict containing station info, or None + """ + if icaoId is None and synopId is None: + return None + + if len(STATIONS) == 0: + logger.warning("Station file not loaded") + return None + + station = None + if synopId: + station = STATIONS[synopId] if synopId in STATIONS else None + else: + for synop in STATIONS: + st = STATIONS[synop] + if st['id'].lower() == icaoId.lower(): + station = st + synopId = synop + break + + if station: + station['synop'] = synopId + + return station + + +def getProfile(filename, synopId, year, month): + """ + Get a parsed profile for the given station from the given RAOBS file + :param filename: + :param synopId: + :param year: + :param month: + :return: A parsed sounding dict, or None + """ + sections = getSections(filename, [synopId]) + if not sections: + return None + + s = [] + for type in sections[synopId].keys(): + sc = decode(sections[synopId][type], year, month) + if sc is not None: + s.append(sc) + fullyMerged = mergeSections(s) + if fullyMerged is None: + logger.error(f"Can't merge sections - missing mandatory levels! at site: {synopId}") + + return fullyMerged + + +def getSections(filename, stationList=None): + """ + Pull each section from the given file. + :param stationList: A list of synop IDs to pull. If omitted, pulls all stations + :return: A dict keyed by synop ID, containing the unparsed sections + """ + sections = {} + line = None + try: + fh = open(filename, 'rb') + while True: + line = fh.readline() + if not line: + break + + # get rid of non-ASCII characters + line = ''.join(chr(i) for i in line if i < 128) + + (type, tokens, id) = getTokens(line) + if type == "": + continue + if stationList is not None and id not in stationList: + continue + if id not in sections.keys(): + sections[id] = {} + + soundingStr = line + if "NIL=" in soundingStr: + continue + + while True: + line = fh.readline() + # get rid of non-ASCII characters + line = ''.join(chr(i) for i in line if i < 128) + + if line: + line = line.replace(chr(13), "").replace(chr(10), " ") + if line == "" or line == " ": + continue + soundingStr += line + if "=" in line: + break + else: + break + + if "NIL=" not in soundingStr: + sections[id][type] = soundingStr + + fh.close() + return sections + + except Exception as e: + logger.error("Problem reading '%s': %s: '%s'" % (filename, e, line)) + return None + + +def decode(soundingStr, year, month): + """ + Decode a sounding string containing TT* or PP* data + :param soundingStr: The complete sounding string + :return: The decoded data, as a dict + """ + # clean up the data first + str = soundingStr.replace("^M", chr(13)) + (type, tokens, id) = getTokens(str) + if type not in TYPES: + return None + decoded = None + + if len(STATIONS) == 0: + logger.warning("Station file not loaded") + return None + + if id not in STATIONS: + logger.error("No station id found for '%s' in the stations file." % id) + return None + stationAlt = STATIONS[id]['elev'] + + if type == "TTAA": + decoded = decodeMandatory(type, tokens, stationAlt, year, month, above100=False) + elif type == "TTBB": + decoded = decodeSignificant(type, tokens, year, month, above100=False) + elif type == "PPBB": + decoded = decodeWinds(type, tokens, stationAlt) + elif type == "TTCC": + decoded = decodeMandatory(type, tokens, stationAlt, year, month, above100=True) + elif type == "TTDD": + decoded = decodeSignificant(type, tokens, year, month, above100=True) + elif type == "PPDD": + decoded = decodeWinds(type, tokens, stationAlt) + + return decoded + + +def decodeMandatory(type, tokens, stationAlt, year, month, above100=False): + """ + Decode the TTAA or TTBB section of the sounding + :param tokens: The parsed tokens making up the section + :return: A dict containing the mandatory levels and section metadata + """ + index = 0 + if tokens[index] != type: + index += 1 + index += 1 + (day, hour) = getDayHour(tokens[index]) + index += 1 + + mandatory = { + 'type': type, + 'id': tokens[index], + 'year': year, + 'month': month, + 'day': day, + 'hour': hour, + 'levels': {} + } + + index += 1 + last_dewdep = None + while index < len(tokens): + if tokens[index] == "51515": + break + + p = tokens[index][:2] + + if p == "88": + break + + if p in LEVEL_CODES.keys(): + pressure = LEVEL_CODES[p] + if above100: + pressure /= 10 + try: + height = int(tokens[index][2:5]) + except Exception: + index += 3 + continue + + if pressure == 1000 and height >= 500: + height = (height - 500) * -1 + elif pressure == 0: + pressure = height + if pressure < 500: + pressure += 1000 + height = stationAlt + elif pressure == 850: + height += 1000 + elif pressure == 700: + if height > 500: + height += 2000 + else: + height += 3000 + elif pressure in [500, 400, 300]: + height *= 10 + elif pressure <= 20: + if height > 500: + height = (2000 + height) * 10 + else: + height = (3000 + height) * 10 + elif pressure <= 80: + if height > 500: + height = (1000 + height) * 10 + else: + height = (2000 + height) * 10 + elif pressure <= 250: + height = (1000 + height) * 10 + + index += 1 + if index < len(tokens): + (temp, dew) = getTempDew(tokens[index], last_dewdep) + if temp is not None and dew is not None: + last_dewdep = temp - dew + else: + (temp, dew) = (None, None) + index += 1 + if index < len(tokens): + (wspd, wdir, u, v) = getWind(tokens[index]) + else: + (wspd, wdir, u, v) = (None, None, None, None) + + mandatory['levels'][pressure] = { + 'height': height, + 'temp': temp, + 'dew': dew, + 'wspd': wspd, + 'wdir': wdir, + 'u': u, + 'v': v, + 'section': type + } + else: + index += 2 # skip this level and the temp and wind fields + + index += 1 + + return mandatory + + +def decodeSignificant(type, tokens, year, month, above100=False): + """ + Decode the TTBB or TTDD sections of the data + :param tokens: The parsed tokens making up the section + :return: A dict containing the significant levels and section metadata + """ + index = 0 + if tokens[index] != type: + index += 1 + index += 1 + (day, hour) = getDayHour(tokens[index]) + index += 1 + significant = { + 'type': type, + 'id': tokens[index], + 'year': year, + 'month': month, + 'day': day, + 'hour': hour, + 'levels': {} + } + + index += 1 + last_dewdep = None + while index < len(tokens): + if tokens[index] in ["21212", "31313", "41414", "51515", "61616"]: + break + if tokens[index] == "/////": + index += 2 + continue + + try: + pl = int(tokens[index][2:5]) + except Exception: + index += 2 + continue + + if above100: + pl /= 10 + elif pl < 100: + pl += 1000 + index += 1 + if index < len(tokens): + (temp, dew) = getTempDew(tokens[index], last_dewdep) + if temp is not None and dew is not None: + last_dewdep = temp - dew + + index += 1 + + significant['levels'][pl] = { + 'temp': temp, + 'dew': dew, + 'section': type + } + + return significant + + +def decodeWinds(type, tokens, stationAlt): + index = 0 + if tokens[index] != type: + index += 1 + index += 1 + (day, hour) = getDayHour(tokens[index]) + index += 1 + + winds = { + 'type': type, + 'id': tokens[index], + 'day': day, + 'hour': hour, + 'heights': {} + } + + index += 1 + while index < len(tokens): + try: + iadd = 0 + t = tokens[index] + if t[0] == "1": + iadd = 100000 + baseht = int(t[1]) * 10 + + heights = [] + heighttok = tokens[index] + for h in heighttok[2:5]: + if h == "/": + continue + height = (baseht + int(h)) * 1000 + iadd # in feet + height *= 0.3048 # to meters + if height == 0: + height = stationAlt + heights.append(height) + + index += 1 + + # sometimes the number of tokens that should follow isn't right... + nexttoks = [] + while index < len(tokens) and tokens[index][0] != '9': + nexttoks.append(tokens[index]) + index += 1 + + # do we have more tokens than heights? then figure out what tokens to use + if len(nexttoks) > len(heights): + i = 0 + for h in heighttok[2:5]: + if h == '/': + nexttoks.pop(i) + if len(nexttoks) == len(heights): + break + else: + i += 1 + + # do we have less tokens than heights? + while len(nexttoks) < len(heights): + heights = heights[:-1] + + for i in range(0, len(heights)): + if heights[i] not in winds['heights']: + (wspd, wdir, u, v) = getWind(nexttoks[i]) + if heights[i] >= stationAlt: + winds['heights'][heights[i]] = {'wspd': wspd, 'wdir': wdir, 'u': u, 'v': v, 'section': type} + except Exception: + break + + return winds + + +def mergeSections(sections): + """ + Merge all sections into a single profile, interpolating when necessary + :param sections: A list of parsed data sections + :return: A merged dict, or None + """ + sects = {} + types = [] + for s in sections: + sects[s['type']] = s + types.append(s['type']) + + if 'TTAA' not in sects: + return None + + # merge mandatory levels + merged = sects['TTAA'] + if 'TTCC' in sects: + merge(merged, sects['TTCC']) + levels = {} + for lv in merged['levels']: + levels[lv] = merged['levels'][lv] + + # merge in optional levels, which have either + # pressure or height data (but not both) so get those too + if 'TTBB' in sects: + getHeights(sects['TTBB'], levels) + merge(merged, sects['TTBB']) + if 'TTDD' in sects: + getHeights(sects['TTDD'], levels) + merge(merged, sects['TTDD']) + if 'PPBB' in sects: + getPressureLevels(sects['PPBB'], levels) + merge(merged, sects['PPBB']) + if 'PPDD' in sects: + getPressureLevels(sects['PPDD'], levels) + merge(merged, sects['PPDD']) + + # Fill in missing obs data + merged['levels'] = interpolateWindAndTemp(merged['levels']) + + # fill in metadata + synop = merged['id'] + station = STATIONS[synop] + merged.update(station) + merged.pop("type", None) + merged['sections'] = types + merged['synop'] = synop + + return merged + + +def merge(dest, src): + """ + Merge src levels in if they don't already exist in dest + :param dest: The section dict to merge into + :param src: The section dict to merge from + :return: A new set of levels, with levels removed where wind or temp couldn't be determined + """ + for pressure in src['levels'].keys(): + if pressure not in dest['levels']: + dest['levels'][pressure] = src['levels'][pressure] + + +def interpolateWindAndTemp(levels): + """ + Interpolate temp/dew and wind/dir values where needed + :param levels: A dict containing levels. Must include pressure info for each level + :return: A new set of levels, with levels removed where wind or temp couldn't be determined + """ + pl = sorted(levels.keys(), reverse=True) + newlevels = {} + + # go through each pressure level + for i in range(0, len(pl)): + thispres = pl[i] + thislevel = levels[thispres] + haveWind = 'u' in thislevel + haveTemp = 'temp' in thislevel + + # If we have both, skip this level + if (haveWind and haveTemp): + newlevels[thispres] = thislevel + continue + # if we have neither, dump the level + if not (haveWind or haveTemp): + continue + + # whch variable set do we need? + missing = ['temp', 'dew'] if haveWind else ['u', 'v'] + + # find a level above and below that contain the missing data field + # so we can interpolate + i1 = i + below = None + while i1 >= 0: + ll = levels[pl[i1]] + found = True + for m in missing: + if m not in ll or ll[m] is None: + found = False + break + if found: + below = ll + break + i1 -= 1 + if below is None: + continue + + i2 = i + above = None + while i2 < len(pl): + ul = levels[pl[i2]] + found = True + for m in missing: + if m not in ul or ul[m] is None: + found = False + break + if found: + above = ul + break + i2 += 1 + if above is None: + continue + + # get the percentage, i.e. how close this level is to the lower level versus the upper level + belowpres = pl[i1] + abovepres = pl[i2] + ratio = (belowpres - thispres) / (belowpres - abovepres) + + # if we're missing wind, interpolate using the vector values + if not haveWind: + u = below['u'] + (above['u'] - below['u']) * ratio + v = below['v'] + (above['v'] - below['v']) * ratio + wspd = math.sqrt(u ** 2 + v ** 2) + wdir = math.atan2(u, v) * (180 / math.pi) + 180 + thislevel['u'] = u + thislevel['v'] = v + thislevel['wspd'] = wspd + thislevel['wdir'] = wdir + else: + # for temp and dew, use a straight linear interpolation + temp = below['temp'] + (above['temp'] - below['temp']) * ratio + dew = below['dew'] + (above['dew'] - below['dew']) * ratio + thislevel['temp'] = temp + thislevel['dew'] = dew + + newlevels[thispres] = thislevel + + return newlevels + + +def getTokens(soundingStr): + """ + Return tokens and section type from the given string + :param soundingStr: The string to parse + :return: (type, tokens, stationId) + """ + str = soundingStr.replace(chr(13), "").replace(chr(10), " ") + str = str.strip() + toks = re.split(r"\s+", str) + tokens = [] + for t in toks: + t = t.replace("=", "") + if len(t) >= 3: + tokens.append(t) + + type = "" + stationId = "" + if len(tokens) >= 3: + if tokens[0] in TYPES: + type = tokens[0] + stationId = tokens[2] + elif tokens[1] in TYPES: + type = tokens[1] + stationId = tokens[3] + + return (type, tokens, stationId) + + +def getDayHour(str): + """ + Parse day of month and hour from the given string + :param str: The string to parse + :return: (day, hour) + """ + try: + day = int(str[:2]) - 50 + if day < 0: + day += 50 + hour = int(str[2:4]) + return (day, hour) + except Exception: + return (None, None) + + +def getWind(str): + """ + Parse wind speed (ms) and wind direction (deg) + :param str: The string to parse + :return: (wspd,wdir,u,v) + """ + wspd = None + wdir = None + u = None + v = None + + try: + wdir = int(str[0:3]) + wspd = int(str[3:5]) + mid = int(str[2]) + if mid == 1 or mid == 6: + wdir -= 1 + wspd += 100 + + # wspd *= 0.51444, convert to m/s + + # calculate u and v + md = 270 - wdir + if md < 0: + md += 360 + theta = md * (math.pi / 180) + u = wspd * math.cos(theta) + v = wspd * math.sin(theta) + except Exception: + pass + + return (wspd, wdir, u, v) + + +def getTempDew(str, last_dewdep): + """ + Parse temperature and dewpoint from the given string + :param str: The string to parse + :param last_dewdep: The last dewpoint depression, in case it's missing + :return: (temp,dew) + """ + temp = None + dew = None + + if str != "/////": + try: + temp = int(str[0:2]) + dec = int(str[2]) + temp += 0.1 * dec + if dec % 2 != 0: + temp *= -1 + except Exception: + pass + + try: + dewdep = int(str[3:5]) + if dewdep <= 50: + dewdep /= 10 + else: + dewdep -= 50 + + # calculate dewpoint from dewpoint depression + dew = temp - dewdep + except Exception: + # dewpoint is missing or unparesable, so calculate it + if temp and last_dewdep: + dew = temp - last_dewdep + + return (temp, dew) + + +def getPressureLevels(section, levels): + """ + Interpolate missing pressure levels using surrounding levels containing both height and pressure info. + Section dict is appended in-place + :param section: The sounding section to interpolate + :param levels: A dict of levels containing both height and pressure at each level (and hopefully temperature) + :return: + """ + section['levels'] = {} + pressures = sorted(levels.keys(), reverse=True) + + # loop through each height and try to find surrounding heights with defined + # levels to interpolate or extrapolate pressure level from + + for height in section['heights'].keys(): + pressurelo = pressures[0] + pressureup = pressures[-1] + pressure = None + + if height < levels[pressurelo]['height']: + pressure = meteo_sounding_utils.pext_down(pressurelo, levels[pressurelo]['temp'], levels[pressurelo]['height'], height) + elif height > levels[pressureup]['height']: + pressure = meteo_sounding_utils.pext_up(pressureup, levels[pressureup]['temp'], levels[pressureup]['height'], height) + else: + p = 0 + while p < len(pressures) - 1 and levels[pressurelo]['height'] <= height: + p += 1 + pressurelo = pressures[p] + pressureup = pressurelo + pressurelo = pressures[0] if p == 0 else pressures[p-1] + + templo = levels[pressurelo]['temp'] + tempup = levels[pressureup]['temp'] + heightlo = levels[pressurelo]['height'] + heightup = levels[pressureup]['height'] + pressure = meteo_sounding_utils.p_interp(templo, tempup, pressurelo, pressureup, heightlo, heightup, height) + + if pressure is not None: + level = section['heights'][height] + level['height'] = height + section['levels'][pressure] = level + + +def getHeights(section, levels): + """ + Interpolate missing heights using surrounding levels containing both height and pressure info. + Section dict is appended in-place + :param section: The sounding section to interpolate + :param levels: A dict of levels containing both height and pressure at each level (and hopefully temperature) + :return: + """ + pressures = sorted(levels.keys(), reverse=True) + + for pressure in sorted(section['levels'].keys(), reverse=True): + pressurelo = pressures[0] + pressureup = pressures[-1] + height = None + + if pressure > pressurelo and levels[pressurelo]['height'] is not None \ + and levels[pressurelo]['temp'] is not None: + height = meteo_sounding_utils.zext_down(pressure, pressurelo, levels[pressurelo]['temp'], + levels[pressurelo]['height']) + elif pressure < pressureup and levels[pressureup]['height'] is not None \ + and levels[pressureup]['temp'] is not None: + height = meteo_sounding_utils.zext_up(pressureup, pressure, levels[pressureup]['temp'], + levels[pressureup]['height']) + else: + p = 0 + while p < len(pressures) - 1 and pressures[p] > pressure: + p += 1 + pressureup = pressures[p] + pressurelo = pressures[0] if p == 0 else pressures[p - 1] + + templo = levels[pressurelo]['temp'] + tempup = levels[pressureup]['temp'] + heightlo = levels[pressurelo]['height'] + heightup = levels[pressureup]['height'] + height = meteo_sounding_utils.z_interp(templo, tempup, pressurelo, pressureup, pressure, heightlo, heightup) + + if height is not None: + section['levels'][pressure]['height'] = height + + +def printProfile(profile, output=sys.stdout): + """ + Print the profile to the given output stream + :param profile: The profile to print + :param output: The output stream. Defauts to stdout + :return: + """ + # print header + print() + print("%-15s %s" % ("WMO number:", profile['synop'])) + print("%-15s %s" % ("ICAO ID:", profile['id'])) + print("%-15s %s" % ("Name:", profile['name'])) + print("%-15s %s" % ("Latitude:", profile['lat'])) + print("%-15s %s" % ("Longitude:", profile['lon'])) + print("%-15s %s m" % ("Elevation:", profile['elev'])) + print("%-15s %s/%sz" % ("Day/Hour:", profile['day'], profile['hour'])) + print("%-15s %s" % ("Decoded parts:", ",".join(profile['sections']))) + print("%8s %6s %6s %6s %6s %6s %6s %10s %10s %6s" % + ("Pres", "Temp", "Dew", "Wspd", "Wdir", "U", "V", "Height", "Height", "Section"), file=output) + print("%8s %6s %6s %6s %6s %6s %6s %10s %10s %6s" % + ("(mb)", "(C)", "(C)", "(m/s)", "(deg)", "", "", "(m)", "(ft)", ""), file=output) + print("-------- ------ ------ ------ ------ ------ ------ ---------- ---------- ------", file=output) + + levels = profile['levels'] + + for pressure in sorted(levels.keys(), reverse=True): + level = round(pressure, ndigits=1) + temp = "NULL" if 'temp' not in levels[pressure] or levels[pressure]['temp'] is None \ + else round(levels[pressure]['temp'], ndigits=1) + dew = "NULL" if 'dew' not in levels[pressure] or levels[pressure]['dew'] is None \ + else round(levels[pressure]['dew'], ndigits=1) + wspd = "NULL" if 'wspd' not in levels[pressure] or levels[pressure]['wspd'] is None \ + else round(levels[pressure]['wspd'], ndigits=1) + wdir = "NULL" if 'wdir' not in levels[pressure] or levels[pressure]['wdir'] is None \ + else round(levels[pressure]['wdir'], ndigits=0) + height = "NULL" if 'height' not in levels[pressure] or levels[pressure]['height'] is None \ + else round(levels[pressure]['height'], ndigits=1) + u = "NULL" if 'u' not in levels[pressure] or levels[pressure]['u'] is None \ + else round(levels[pressure]['u'], ndigits=1) + v = "NULL" if 'v' not in levels[pressure] or levels[pressure]['v'] is None \ + else round(levels[pressure]['v'], ndigits=1) + section = levels[pressure]['section'] + + print("%8s %6s %6s %6s %6s %6s %6s %10s %6s" % + ("%.1f" % level, temp, dew, wspd, wdir, u, v, height, section), file=output) + + +def change_vars(profile): + met_utils = meteo_utils.meteo_utils() + new_profile = {} + for var_name in meta_keys: + new_profile[var_name] = [] + for var_name in obsvars: + new_profile[var_name] = [] + + # SPECIAL: Most soundings launched 50-55 minutes prior to stated synoptic time, so a 12Z + # launch is usually initiated close to 11:05Z. + this_datetime = datetime(profile['year'], profile['month'], profile['day'], profile['hour'], 0, 0) + launch_time = this_datetime - timedelta(seconds=55*60) + previous_time = launch_time + + heightKm1 = profile['elev'] + levels = profile['levels'] + for pressure in sorted(levels.keys(), reverse=True): + pres = pressure*100.0 + height = float_missing_value + temp = float_missing_value + dewp = float_missing_value + tvirt = float_missing_value + spfh = float_missing_value + u = float_missing_value + v = float_missing_value + if 'temp' in levels[pressure] and levels[pressure]['temp'] is not None: + temp = levels[pressure]['temp'] + 273.15 + if 'dew' in levels[pressure] and levels[pressure]['dew'] is not None: + dewp = levels[pressure]['dew'] + 273.15 + if (temp > 75 and temp < 355 and dewp > 50 and dewp < 325 and dewp <= temp*1.05 and pres > 100 and pres < 109900): + spfh = met_utils.specific_humidity(dewp, pres) + qvapor = max(1.0e-12, spfh/(1.0-spfh)) + tvirt = temp*(1.0 + 0.61*qvapor) + if 'height' in levels[pressure] and levels[pressure]['height'] is not None: + height = levels[pressure]['height'] + dz = height - heightKm1 + # Legacy soundings produce null values at mandatory level below ground. + if (dz < 1.0): + this_datetime = previous_time + else: + # Typical radiosonde ascent rate is 5 m/s + this_datetime = previous_time + timedelta(seconds=dz*0.2) + heightKm1 = height + else: + this_datetime = previous_time + + if 'u' in levels[pressure] and levels[pressure]['u'] is not None: + u = levels[pressure]['u'] + if 'v' in levels[pressure] and levels[pressure]['v'] is not None: + v = levels[pressure]['v'] + + time_offset = round((this_datetime - epoch).total_seconds()) + previous_time = this_datetime + + new_profile['stationIdentification'].append(profile['synop']) + new_profile['latitude'].append(profile['lat']) + new_profile['longitude'].append(profile['lon']) + new_profile['stationElevation'].append(profile['elev']) + new_profile['launchTime'].append(launch_time.strftime("%Y-%m-%dT%H:%M:%SZ")) + new_profile['dateTime'].append(time_offset) + new_profile['pressure'].append(pres) + new_profile['height'].append(height) + new_profile['airTemperature'].append(temp) + new_profile['virtualTemperature'].append(tvirt) + new_profile['specificHumidity'].append(spfh) + new_profile['windEastward'].append(u) + new_profile['windNorthward'].append(v) + + """ + Based on height and time and the wind componenents, predict the lat, lon positions + as the balloon ascends. Generally the balloon ascends at 5 m/s, which was already + assumed in the creation of each timestamp. + """ + + previous_idx = 0 + location = [profile['lon'], profile['lat'], None] + previous_loc = [profile['lon'], profile['lat'], None] + delta_t = np.diff(new_profile['dateTime']) + + for idx in range(1, len(delta_t)): + + if (new_profile['windEastward'][idx-1] != float_missing_value and new_profile['windNorthward'][idx-1] != float_missing_value): + # move north-south + d_north = new_profile['windNorthward'][idx-1] * delta_t[idx-1] + location = geod.direct(points=previous_loc[:2], azimuths=0., distances=d_north)[0] + new_profile['latitude'][idx] = location[1] + # move east-west + d_east = new_profile['windEastward'][idx-1] * delta_t[idx-1] + location = geod.direct(points=location[:2], azimuths=90., distances=d_east)[0] + new_profile['longitude'][idx] = location[0] + else: + new_profile['latitude'][idx] = new_profile['latitude'][idx-1] + new_profile['longitude'][idx] = new_profile['longitude'][idx-1] + + # store location for next step calculations + previous_loc = dcop(location) + + # Be sure to delete the prior location info before exiting + del location + del previous_loc + + return new_profile + + +def append_ioda_data(in_profile, obs_data): + """ + Append each profile to the end of obs_data. + """ + + for var_name in meta_keys: + obs_data[var_name].extend(in_profile[var_name]) + for var_name in obsvars: + obs_data[var_name].extend(in_profile[var_name]) + + return obs_data + + +if __name__ == "__main__": + + import argparse + + start_time = time.time() + today = datetime.today() + + parser = argparse.ArgumentParser( + description=( + 'Read legacy radiosonde text file (WMO TEMPO format) and convert into IODA output file') + ) + + required = parser.add_argument_group(title='required arguments') + required.add_argument('-i', '--input-files', nargs='+', dest='file_names', + action='store', default=None, required=True, + help='input files') + required.add_argument('-o', '--output-file', dest='output_file', + action='store', default=None, required=True, + help='output file') + required.add_argument('-t', '--station-file', dest='station_file', + action='store', default=None, required=True, + help='station table file') + + parser.set_defaults(debug=False) + parser.set_defaults(verbose=False) + parser.set_defaults(netCDF=False) + optional = parser.add_argument_group(title='optional arguments') + optional.add_argument('-y', '--year', dest='year', + action='store', default=None, help='year') + optional.add_argument('-m', '--month', dest='month', + action='store', default=None, help='month') + optional.add_argument('--debug', action='store_true', + help='enable debug messages') + optional.add_argument('--verbose', action='store_true', + help='enable verbose debug messages') + optional.add_argument('--netcdf', action='store_true', + help='enable netCDF output file (IODA/JEDI Data Conventions)') + + args = parser.parse_args() + + if args.year: + year = int(args.year) + else: + year = today.year + if args.month: + month = int(args.month) + else: + month = today.month + + if args.debug: + logging.basicConfig(level=logging.INFO) + elif args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.ERROR) + + """ + #--------------------------------------------------------------------------------------------- + Options captured, now ingest the station list and cycle the stations to get each profile. + #--------------------------------------------------------------------------------------------- + """ + + if not os.path.isfile(args.station_file): + parser.error('Station table (-t option) file: ', args.station_file, ' does not exist') + + loadStations(args.station_file) + + # If using netcdf output option, set up data structures (IODA) for outputs. + if args.netcdf: + varDict = defaultdict(lambda: DefaultOrderedDict(dict)) + varAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) + obs_data = {} # The final outputs. + for var_name in meta_keys: + obs_data[var_name] = [] + for var_name in obsvars: + obs_data[var_name] = [] + + # Loop through input files and decode/convert. + ntotal = 0 + for file_name in args.file_names: + if not os.path.isfile(file_name): + parser.error('Input (-i option) file: ', file_name, ' does not exist') + if args.netcdf: + AttrData['sourceFiles'] += ", " + file_name + logging.debug(f"Reading input file: {file_name}") + + nstations = 0 + for n, station in enumerate(STATIONS.keys()): + logging.debug(f"\n seeking data from station {station} within {file_name} for year:{year} and month:{month}") + profile = getProfile(file_name, station, year, month) + + if profile: + nstations += 1 + nlevels = len(profile['levels']) + if nlevels > 1: + ntotal += nlevels + logging.debug(f" found sounding with {nlevels} levels") + if args.verbose: + printProfile(profile) + if args.netcdf: + new_profile = change_vars(profile) + obs_data = append_ioda_data(new_profile, obs_data) + else: + logging.debug(f" skipping sounding {station} with 1 or fewer ({nlevels}) levels") + + if args.netcdf: + ioda_data = {} + DimDict = {'Location': ntotal} + AttrData['sourceFiles'] = AttrData['sourceFiles'][2:] + # Set coordinates and units of the ObsValues. + for n, iodavar in enumerate(obsvars): + varDict[iodavar]['valKey'] = iodavar, obsValName + varDict[iodavar]['errKey'] = iodavar, obsErrName + varDict[iodavar]['qcKey'] = iodavar, qcName + varAttrs[iodavar, obsValName]['coordinates'] = 'longitude latitude' + varAttrs[iodavar, obsErrName]['coordinates'] = 'longitude latitude' + varAttrs[iodavar, qcName]['coordinates'] = 'longitude latitude' + varAttrs[iodavar, obsValName]['units'] = obsvars_units[n] + varAttrs[iodavar, obsErrName]['units'] = obsvars_units[n] + + # Set units of the MetaData variables and all _FillValues. + for key in meta_keys: + dtypestr = MetaDataKeyList[meta_keys.index(key)][1] + if MetaDataKeyList[meta_keys.index(key)][2]: + varAttrs[(key, metaDataName)]['units'] = MetaDataKeyList[meta_keys.index(key)][2] + varAttrs[(key, metaDataName)]['_FillValue'] = missing_vals[dtypestr] + ioda_data[(key, metaDataName)] = np.array(obs_data[key], dtype=dtypes[dtypestr]) + + # Transfer from the 1-D data vectors and ensure output data (ioda_data) types using numpy. + for n, iodavar in enumerate(obsvars): + ioda_data[(iodavar, obsValName)] = np.array(obs_data[iodavar], dtype=np.float32) + ioda_data[(iodavar, obsErrName)] = np.full(ntotal, obserrlist[n], dtype=np.float32) + ioda_data[(iodavar, qcName)] = np.full(ntotal, 2, dtype=np.int32) + + logging.debug("Writing output file: " + args.output_file) + # setup the IODA writer + writer = iconv.IodaWriter(args.output_file, MetaDataKeyList, DimDict) + # write everything out + writer.BuildIoda(ioda_data, VarDims, varAttrs, AttrData) + + print(f" wrote data for {nstations} number of stations") + logging.info("--- {:9.4f} total seconds ---".format(time.time() - start_time)) diff --git a/src/conventional/synop_bufr2ioda.py b/src/conventional/synop_bufr2ioda.py index 9cddd6d2b..5bc2b614b 100644 --- a/src/conventional/synop_bufr2ioda.py +++ b/src/conventional/synop_bufr2ioda.py @@ -28,22 +28,28 @@ os.environ["TZ"] = "UTC" locationKeyList = [ - ("stationIdentification", "string", ""), - ("stationLongName", "string", ""), - ("wmoBlockNumber", "integer", ""), - ("wmoStationNumber", "integer", ""), - ("latitude", "float", "degrees_north"), - ("longitude", "float", "degrees_east"), - ("stationElevation", "float", "m"), - ("height", "float", "m"), - ("dateTime", "long", "seconds since 1970-01-01T00:00:00Z") + ("stationIdentification", "string", "", "keep"), + # ("station_name", "string", "", "keep"), + ("wmoBlockNumber", "integer", "", "toss"), + ("wmoStationNumber", "integer", "", "toss"), + ("latitude", "float", "degrees_north", "keep"), + ("longitude", "float", "degrees_east", "keep"), + ("stationElevation", "float", "m", "keep"), + ("height", "float", "m", "keep"), + ("dateTime", "long", "seconds since 1970-01-01T00:00:00Z", "keep"), + ("year", "integer", "", "toss"), + ("month", "integer", "", "toss"), + ("day", "integer", "", "toss"), + ("hour", "integer", "", "toss"), + ("minute", "integer", "", "toss"), + ("second", "integer", "", "toss") ] meta_keys = [m_item[0] for m_item in locationKeyList] metaDataKeyList = { 'wmoBlockNumber': ['blockNumber'], 'wmoStationNumber': ['stationNumber'], - 'stationLongName': ['stationOrSiteName'], + # 'station_name': ['stationOrSiteName'], This fails due to unicode characters 'latitude': ['latitude'], 'longitude': ['longitude'], 'stationElevation': ['heightOfStationGroundAboveMeanSeaLevel'], @@ -51,7 +57,13 @@ 'heightOfBarometerAboveMeanSeaLevel', 'heightOfStationGroundAboveMeanSeaLevel'], 'stationIdentification': ['Constructed'], - 'dateTime': ['Constructed'] + 'dateTime': ['Constructed'], + 'year': ['year'], + 'month': ['month'], + 'day': ['day'], + 'hour': ['hour'], + 'minute': ['minute'], + 'second': ['second'] } var_mimic_length = "latitude" @@ -66,18 +78,20 @@ # The outgoing IODA variables (ObsValues), their units, and assigned constant ObsError. obsvars = ['airTemperature', 'specificHumidity', + 'virtualTemperature', 'windEastward', 'windNorthward', 'stationPressure'] -obsvars_units = ['K', 'kg kg-1', 'm s-1', 'm s-1', 'Pa'] -obserrlist = [1.2, 0.75E-3, 1.7, 1.7, 120.0] +obsvars_units = ['K', 'kg kg-1', 'K', 'm s-1', 'm s-1', 'Pa'] +obserrlist = [1.2, 0.75E-3, 1.5, 1.7, 1.7, 120.0] VarDims = { - 'airTemperature': ['nlocs'], - 'specificHumidity': ['nlocs'], - 'windEastward': ['nlocs'], - 'windNorthward': ['nlocs'], - 'stationPressure': ['nlocs'] + 'airTemperature': ['Location'], + 'specificHumidity': ['Location'], + 'virtualTemperature': ['Location'], + 'windEastward': ['Location'], + 'windNorthward': ['Location'], + 'stationPressure': ['Location'] } metaDataName = iconv.MetaDataName() @@ -90,7 +104,7 @@ 'ioda_version': 2, 'description': 'Surface (SYNOP) observations converted from BUFR', 'source': 'LDM at NCAR-RAL', - 'source_files': '' + 'sourceFiles': '' } DimDict = { @@ -137,12 +151,12 @@ def main(file_names, output_file): for fname in file_names: logging.debug("Reading file: " + fname) - AttrData['source_files'] += ", " + fname + AttrData['sourceFiles'] += ", " + fname data, count, start_pos = read_file(fname, count, start_pos, data) - AttrData['source_files'] = AttrData['source_files'][2:] - logging.debug("All source files: " + AttrData['source_files']) + AttrData['sourceFiles'] = AttrData['sourceFiles'][2:] + logging.debug("All source files: " + AttrData['sourceFiles']) if not data: logging.critical("ABORT: no message data was captured, stopping execution.") @@ -150,8 +164,7 @@ def main(file_names, output_file): logging.info("--- {:9.4f} BUFR read seconds ---".format(time.time() - start_time)) nlocs = len(data['dateTime']) - DimDict = {'nlocs': nlocs} - AttrData['nlocs'] = np.int32(DimDict['nlocs']) + DimDict = {'Location': nlocs} # Set coordinates and units of the ObsValues. for n, iodavar in enumerate(obsvars): @@ -167,10 +180,12 @@ def main(file_names, output_file): # Set units of the MetaData variables and all _FillValues. for key in meta_keys: dtypestr = locationKeyList[meta_keys.index(key)][1] - if locationKeyList[meta_keys.index(key)][2]: - varAttrs[(key, metaDataName)]['units'] = locationKeyList[meta_keys.index(key)][2] - varAttrs[(key, metaDataName)]['_FillValue'] = missing_vals[dtypestr] - obs_data[(key, metaDataName)] = np.array(data[key], dtype=dtypes[dtypestr]) + keep_or_toss = locationKeyList[meta_keys.index(key)][3] + if (keep_or_toss == "keep"): + if locationKeyList[meta_keys.index(key)][2]: + varAttrs[(key, metaDataName)]['units'] = locationKeyList[meta_keys.index(key)][2] + varAttrs[(key, metaDataName)]['_FillValue'] = missing_vals[dtypestr] + obs_data[(key, metaDataName)] = np.array(data[key], dtype=dtypes[dtypestr]) # Transfer from the 1-D data vectors and ensure output data (obs_data) types using numpy. for n, iodavar in enumerate(obsvars): @@ -268,11 +283,58 @@ def is_all_missing(data): return False +def specialty_time(year, month, day, hour, minute, second): + + bad_date = False + + # UGH, some BUFR uses 2-digit year, awful. + if (year >= 0 and year <= 50): + year += 2000 + elif (year > 50 and year <= 99): + year += 1900 + + if minute == int_missing_value: + minute = 0 + elif minute < 0 or minute > 59: + minute = 0 + + if second == int_missing_value: + second = 0 + elif second < 0 or second > 59: + second = 0 + + if (year < 1900 or year > 2499 or month < 1 or month > 12 or day < 1 or day > 31 or hour < 0 or hour > 23 or minute < 0 or minute > 59): + bad_date = True + + if bad_date: + year = 1900 + month = 1 + day = 1 + hour = 0 + minute = 0 + + try: + this_datetime = datetime(year, month, day, hour, minute, second) + except Exception: + logging.critical(f"Bogus time info {year}-{month}-{day}T{hour}:{minute}:{second}Z") + year = 1900 + month = 1 + day = 1 + hour = 0 + minute = 0 + this_datetime = datetime(year, month, day, hour, minute, second) + + time_offset = round((this_datetime - epoch).total_seconds()) + + return time_offset + + def read_file(file_name, count, start_pos, data): f = open(file_name, 'rb') while True: + count[0] += 1 # Use eccodes to decode each bufr message in the file data, count, start_pos = read_bufr_message(f, count, start_pos, data) @@ -284,143 +346,140 @@ def read_file(file_name, count, start_pos, data): def read_bufr_message(f, count, start_pos, data): + temp_data = {} # A temporary dictionary to hold things. meta_data = {} # All the various MetaData go in here. vals = {} # Floating-point ObsValues variables go in here. avals = [] # Temporarily hold array of values. call_fail = False - if start_pos == f.tell(): - return data, count, None start_pos = f.tell() - count[0] += 1 - logging.info("BUFR message number: " + str(count[0])) met_utils = meteo_utils.meteo_utils() try: bufr = ecc.codes_bufr_new_from_file(f) - except: # noqa - logging.critical("ABORT, failue when attempting to call: codes_bufr_new_from_file") + try: + msg_size = ecc.codes_get_message_size(bufr) + logging.info(f"Will attempt to read BUFR msg [{count[0]}] with size: ({msg_size} bytes)") + except: # noqa + return data, count, None + except ecc.CodesInternalError: + logging.critical(f"Useless BUFR message (codes_bufr_new_from_file)") + start_pos = f.tell() + return data, count, start_pos try: ecc.codes_set(bufr, 'skipExtraKeyAttributes', 1) # Supposedly this is ~25 percent faster ecc.codes_set(bufr, 'unpack', 1) - except: # noqa - logging.info("finished unpacking BUFR file") - start_pos = None + except ecc.CodesInternalError: + ecc.codes_release(bufr) + logging.info(f"INCOMPLETE BUFR message, skipping ({msg_size} bytes)") + start_pos += int(0.5*msg_size) + f.seek(start_pos) return data, count, start_pos + # Some BUFR messages have subsets (multiple) observations in a single message. + # Therefore we have to loop through a vector of the delayedDescriptorReplicationFactor. + try: + nsubsets = ecc.codes_get(bufr, 'numberOfSubsets') + except ecc.KeyValueNotFoundError: + nsubsets = 1 + pass + + # Are the data compressed or uncompressed? If the latter, then when nsubsets>1, we + # have to do things differently. + compressed = ecc.codes_get(bufr, 'compressedData') + # First, get the MetaData we are interested in (list is in metaDataKeyList) + max_mlen = 0 for k, v in metaDataKeyList.items(): - meta_data[k] = [] + temp_data[k] = [] if (len(v) > 1): for var in v: if (var != 'Constructed'): try: avals = ecc.codes_get_array(bufr, var) - meta_data[k] = assign_values(avals, k) - if not is_all_missing(meta_data[k]): + max_mlen = max(max_mlen, len(avals)) + temp_data[k] = assign_values(avals, k) + if not is_all_missing(temp_data[k]): break except ecc.KeyValueNotFoundError: - logging.warning("Caution: unable to find requested BUFR key: " + var) - except Exception: - logging.warning("Caution, requested BUFR key: " + v[0] + " is somehow bad") + logging.debug("Caution: unable to find requested BUFR key: " + var) + temp_data[k] = None + else: + temp_data[k] = None else: if (v[0] != 'Constructed'): try: avals = ecc.codes_get_array(bufr, v[0]) - meta_data[k] = assign_values(avals, k) + max_mlen = max(max_mlen, len(avals)) + temp_data[k] = assign_values(avals, k) except ecc.KeyValueNotFoundError: - logging.warning("Caution, unable to find requested BUFR key: " + v[0]) - except Exception: - logging.warning("Caution, requested BUFR key: " + v[0] + " is somehow bad") + logging.debug("Caution, unable to find requested BUFR key: " + v[0]) + temp_data[k] = None + else: + temp_data[k] = None + if temp_data[k] is not None: + logging.info(f" length of var {k} is: {len(temp_data[k])}") + + # These meta data elements are so critical that we should quit quickly if lacking them: + if (temp_data['year'] is None) and (temp_data['month'] is None) and \ + (temp_data['day'] is None) and (temp_data['hour'] is None): + logging.warning("Useless ob without date info.") + count[2] += target_number + return data, count, start_pos + if (temp_data['wmoBlockNumber'] is None) and (temp_data['wmoStationNumber'] is None) and \ + (temp_data['latitude'] is None) and (temp_data['longitude'] is None): + logging.warning("Useless ob without lat,lon or station number info.") + count[2] += target_number + return data, count, start_pos - # Determine the target number of observation points from a critical variable (i.e., latitude). - target_number = len(meta_data[var_mimic_length]) - logging.debug("Target number of obs in this BUFR message: " + str(target_number)) + # Next, get the raw observed weather variables we want. + # TO-DO: currently all ObsValue variables are float type, might need integer/other. + max_dlen = 0 + for variable in raw_obsvars: + temp_data[variable] = [] + if not compressed and nsubsets > 1: + for n in range(nsubsets): + var = '/subsetNumber=' + str(n+1) + '/' + variable + try: + avals = ecc.codes_get_array(bufr, var) + temp_data[variable] = np.append(temp_data[variable], assign_values(avals, variable)) + except ecc.KeyValueNotFoundError: + logging.debug("Caution, unable to find requested BUFR variable: " + variable) + temp_data[variable] = None + else: + try: + avals = ecc.codes_get_array(bufr, variable) + max_dlen = max(max_dlen, len(avals)) + temp_data[variable] = assign_values(avals, variable) + except ecc.KeyValueNotFoundError: + logging.debug("Caution, unable to find requested BUFR variable: " + variable) + temp_data[variable] = None + if temp_data[k] is not None: + logging.info(f" length of var {variable} is: {len(temp_data[variable])}") + + # Be done with this BUFR message. + ecc.codes_release(bufr) - # For any of the MetaData elements that were totally lacking, fill entire vector with missing. + # Transfer the meta data from its temporary vector into final its meta data var. + target_number = nsubsets empty = [] for k, v in metaDataKeyList.items(): - if not any(meta_data[k]): - meta_data[k] = assign_missing_meta(empty, k, target_number, 0) - if (len(meta_data[k]) == 1): - meta_data[k] = np.full(target_number, meta_data[k][0]) - elif (len(meta_data[k]) < target_number): - logging.warning(f"Key called {k} contains only {len(meta_data[k])} " - f" elements, wheras {target_number} were expected.") - meta_data[k] = assign_missing_meta(meta_data[k], k, target_number, len(meta_data[k])-1) - - # Plus, to construct a dateTime, we always need its components. - try: - year = ecc.codes_get_array(bufr, 'year') - if (len(year) < target_number): - year = np.full(target_number, year[0]) - year[year < 1900] = 1900 - year[year > 2399] = 1900 - except ecc.KeyValueNotFoundError: - logging.warning("Caution, no data for year") - year = np.full(target_number, 1900) - - try: - month = ecc.codes_get_array(bufr, 'month') - if (len(month) < target_number): - month = np.full(target_number, month[0]) - year[np.logical_or(month < 1, month > 12)] = 1900 - month[np.logical_or(month < 1, month > 12)] = 1 - except ecc.KeyValueNotFoundError: - logging.warning("Caution, no data for month") - year = np.full(target_number, 1900) - month = np.full(target_number, 1) - - try: - day = ecc.codes_get_array(bufr, 'day') - if (len(day) < target_number): - day = np.full(target_number, day[0]) - year[np.logical_or(day < 1, day > 31)] = 1900 - day[np.logical_or(day < 1, day > 31)] = 1 - except ecc.KeyValueNotFoundError: - logging.warning("Caution, no data for day") - year = np.full(target_number, 1900) - day = np.full(target_number, 1) - - try: - hour = ecc.codes_get_array(bufr, 'hour') - if (len(hour) < target_number): - hour = np.full(target_number, hour[0]) - year[np.logical_or(hour < 0, hour > 23)] = 1900 - hour[np.logical_or(hour < 0, hour > 23)] = 0 - except ecc.KeyValueNotFoundError: - logging.warning("Caution, no data for hour") - year = np.full(target_number, 1900) - hour = np.full(target_number, 0) - - try: - minute = ecc.codes_get_array(bufr, 'minute') - if (len(minute) < target_number): - minute = np.full(target_number, minute[0]) - year[np.logical_or(minute < 0, minute > 59)] = 1900 - minute[np.logical_or(minute < 0, minute > 59)] = 0 - except ecc.KeyValueNotFoundError: - logging.warning("Caution, no data for minute") - year = np.full(target_number, 1900) - minute = np.full(target_number, 0) - - second = np.full(target_number, 0) - try: - avals = ecc.codes_get_array(bufr, 'second') # non-integer value, optional - if (len(avals) < target_number): - avals = np.full(target_number, avals[0]) - for n, a in enumerate(avals): - if (a > 0 and a < 60): - second[n] = round(a) - except ecc.KeyValueNotFoundError: - logging.info("Caution, no data for second") - - for n, yyyy in enumerate(year): - this_datetime = datetime(yyyy, month[n], day[n], hour[n], minute[n], second[n]) - time_offset = round((this_datetime - epoch).total_seconds()) - if (time_offset > -1E9): - meta_data['dateTime'][n] = time_offset + meta_data[k] = assign_missing_meta(empty, k, target_number, 0) + if temp_data[k] is None: + next + else: + if len(temp_data[k]) == target_number: + meta_data[k] = temp_data[k] + else: + meta_data[k] = np.full(target_number, temp_data[k][0]) + + # To construct a dateTime, we always need its components. + if temp_data['year'] is not None: + for n in range(nsubsets): + meta_data['dateTime'][n] = specialty_time(meta_data['year'][n], meta_data['month'][n], + meta_data['day'][n], meta_data['hour'][n], # noqa + meta_data['minute'][n], meta_data['second'][n]) # noqa # Force longitude into space of -180 to +180 only. Reset both lat/lon missing if either absent. mask_lat = np.logical_or(meta_data['latitude'] < -90.0, meta_data['latitude'] > 90.0) @@ -437,73 +496,97 @@ def read_bufr_message(f, count, start_pos, data): mask_height = np.logical_or(meta_data['height'] < -425, meta_data['height'] > 8500) meta_data['height'][mask_height] = float_missing_value - # If the height of the observation (sensor) is missing, try to fill it with stationElevation. + # If the height of the observation (sensor) is missing, try to fill it with station_elevation. for n, elev in enumerate(meta_data['stationElevation']): - if (elev > -425 and elev < 8500 and np.abs(meta_data['height'][n]-elev) > 50): + if (elev > -425 and elev < 8500): meta_data['height'][n] = elev + 2 - # Next, get the raw observed weather variables we want. - # TO-DO: currently all ObsValue variables are float type, might need integer/other. - for variable in raw_obsvars: - vals[variable] = [] - try: - avals = ecc.codes_get_array(bufr, variable) - if (len(avals) != target_number): - logging.warning(f"Variable called {variable} contains only {len(avals)} " - f" elements, wheras {target_number} were expected.") - count[2] += target_number - return data, count, start_pos - vals[variable] = assign_values(avals, variable) - except Exception: - logging.warning("Caution, unable to find requested BUFR variable: " + variable) - vals[variable] = np.full(target_number, float_missing_value, dtype=np.float32) - - # Be done with this BUFR message. - ecc.codes_release(bufr) - - # Count the locations for which time, lat, lon, or height is nonsense, therefore ob is useless. - count[1] += target_number - mask_date = np.full(target_number, 0, dtype=np.int32) - mask_ll = np.full(target_number, 0, dtype=np.int32) - mask_z = np.full(target_number, 0, dtype=np.int32) - mask_date[year == 1900] = 1 - mask_ll[meta_data['latitude'] == float_missing_value] = 1 - mask_z[meta_data['stationElevation'] == float_missing_value] = 1 - for n, x in enumerate(mask_date): - if (mask_date[n] == 1 or mask_ll[n] == 1 or mask_z[n] == 1): - count[2] += 1 - # Forcably create station_id 5-char string from WMO block+station number. meta_data['station_id'] = np.full(target_number, string_missing_value, dtype=' 0 and block < 100 and number > 0 and number < 1000): - meta_data['stationIdentification'][n] = "{:02d}".format(block) + "{:03d}".format(number) + meta_data['station_id'][n] = "{:02d}".format(block) + "{:03d}".format(number) + + # And now processing the observed variables we care about. + nbad = 0 + for variable in raw_obsvars: + vals[variable] = np.full(target_number, float_missing_value) + if temp_data[variable] is not None: + logging.info(f" length of var {variable} is: {len(temp_data[variable])}") + if len(temp_data[variable]) == target_number: + vals[variable] = temp_data[variable] + elif len(temp_data[variable]) < target_number: + logging.warning(f" var {variable} has {len(temp_data[variable])} " + f"elements while expecting {target_number}") + lendat = len(temp_data[variable]) + vals[variable][0:lendat] = temp_data[variable] + else: + logging.warning(f" var {variable} has {len(temp_data[variable])} " + f"elements while expecting {target_number}") + vals[variable] = temp_data[variable][0:target_number] + else: + nbad += 1 + + if nbad == len(raw_obsvars): + logging.warning(f"No usable data in this ob.") + count[2] += target_number + return data, count, start_pos + + count[1] += target_number + + # Finally transfer the meta_data to the output array. + for key in meta_keys: + data[key] = np.append(data[key], meta_data[key]) + + ''' + Need to transform some variables (wind speed/direction to components for example). + In the ideal world, we could assume that the meteorological variables were given + well-bounded values, but in BUFR, they could be garbage, so ensure that values + are all sensible before calling the transformation functions. + ''' - # Need to transform some variables to others (wind speed/direction to components for example). uwnd = np.full(target_number, float_missing_value) vwnd = np.full(target_number, float_missing_value) for n, wdir in enumerate(vals['windDirection']): - if (wdir >= 0 and wdir <= 360 and vals['windSpeed'][n] != float_missing_value): - uwnd[n], vwnd[n] = met_utils.dir_speed_2_uv(wdir, vals['windSpeed'][n]) + wspd = vals['windSpeed'][n] + if wdir and wspd: + if (wdir >= 0 and wdir <= 360 and wspd >= 0 and wspd < 300): + uwnd[n], vwnd[n] = met_utils.dir_speed_2_uv(wdir, wspd) + + airt = np.full(target_number, float_missing_value) + for n, temp in enumerate(vals['airTemperature']): + if temp: + if (temp > 50 and temp < 345): + airt[n] = temp + + psfc = np.full(target_number, float_missing_value) + for n, p in enumerate(vals['nonCoordinatePressure']): + if p: + if (p > 30000 and p < 109900): + psfc[n] = p spfh = np.full(target_number, float_missing_value) + tvirt = np.full(target_number, float_missing_value) for n, dewpoint in enumerate(vals['dewpointTemperature']): - psfc = vals['nonCoordinatePressure'][n] - if (dewpoint > 90 and dewpoint < 325 and psfc > 30000 and psfc < 109900): - spfh[n] = met_utils.specific_humidity(dewpoint, psfc) - - # Move everything into the final data dictionary, including metadata. + if dewpoint: + if (dewpoint > 90 and dewpoint < 325 and psfc[n] != float_missing_value): + spfh[n] = met_utils.specific_humidity(dewpoint, psfc[n]) + if (airt[n] != float_missing_value): + qvapor = max(1.0e-12, spfh[n]/(1.0-spfh[n])) + tvirt[n] = airt[n]*(1.0 + 0.61*qvapor) + + # Finally fill up the output data dictionary with observed variables. data['windEastward'] = np.append(data['windEastward'], uwnd) data['windNorthward'] = np.append(data['windNorthward'], vwnd) data['specificHumidity'] = np.append(data['specificHumidity'], spfh) - data['airTemperature'] = np.append(data['airTemperature'], vals['airTemperature']) - data['stationPressure'] = np.append(data['stationPressure'], vals['nonCoordinatePressure']) - for key in meta_keys: - data[key] = np.append(data[key], meta_data[key]) + data['airTemperature'] = np.append(data['airTemperature'], airt) + data['virtualTemperature'] = np.append(data['virtualTemperature'], tvirt) + data['stationPressure'] = np.append(data['stationPressure'], psfc) - logging.info("number of observations so far: " + str(count[1])) - logging.info("number of invalid or useless observations: " + str(count[2])) + logging.info(f"number of observations so far: {count[1]} from {count[0]} BUFR msgs.") + logging.info(f"number of invalid or useless observations: {count[2]}") + start_pos = f.tell() return data, count, start_pos diff --git a/src/gnssro/CMakeLists.txt b/src/gnssro/CMakeLists.txt index 9624894f7..0d4f50190 100644 --- a/src/gnssro/CMakeLists.txt +++ b/src/gnssro/CMakeLists.txt @@ -5,7 +5,7 @@ ecbuild_add_executable( TARGET gnssro_bufr2ioda SOURCES gnssro_bufr2ioda.f90 - LIBS bufr::bufr_4 + LIBS bufr::bufr_d NetCDF::NetCDF_Fortran) ecbuild_add_executable( TARGET gnssro_gsidiag2ioda diff --git a/src/gnssro/gnssro_bufr2ioda.py b/src/gnssro/gnssro_bufr2ioda.py index 2934f647e..3b681430c 100644 --- a/src/gnssro/gnssro_bufr2ioda.py +++ b/src/gnssro/gnssro_bufr2ioda.py @@ -10,7 +10,6 @@ from __future__ import print_function import sys import argparse -import netCDF4 as nc from datetime import datetime, timedelta import dateutil.parser from concurrent.futures import ProcessPoolExecutor @@ -268,7 +267,6 @@ def get_obs_data(bufr, profile_meta_data, add_qc, record_number=None): def quality_control(profile_meta_data, heights, lats, lons): - print('Performing QC Checks') good = (heights > 0.) & (heights < 100000.) & (abs(lats) < 90.) & (abs(lons) < 360.) diff --git a/src/goes/goes.py b/src/goes/goes.py index 11450204d..b1095e901 100644 --- a/src/goes/goes.py +++ b/src/goes/goes.py @@ -25,11 +25,10 @@ class Goes: def __init__(self, input_file_path, goes_util: GoesUtil): """ Constructor - input_file_path - a GOES-16 or GOES-17 raw data file for a single ABI channel + input_file_path - a GOES-R raw data file for a single ABI channel """ self._input_file_path = input_file_path self._goes_util = goes_util - self._get_metadata_from_input_file_path() self._rad_data_array = None self._dqf_data_array = None self._lat_fill_value_index_array = None @@ -37,34 +36,111 @@ def __init__(self, input_file_path, goes_util: GoesUtil): self._obsvalue_bt_data_array = None self._obserror_rf_data_array = None self._obserror_bt_data_array = None + self._input_dataset = None + self._metadata_dict = {} + self._open() + + # Pre-load some important MetaData into the dictionary + self._get_metadata_from_input_file() - def _get_metadata_from_input_file_path(self): + def _get_metadata_from_input_file(self): """ Creates a dictionary of file metadata from input_file_path """ - self._metadata_dict = {'instrument': 'ABI', - 'processing_level': 'L1b', - 'product_acronym': 'Rad'} - metadata_array = os.path.splitext(os.path.basename(self._input_file_path))[0].split('_') - self._metadata_dict['system_environment'] = metadata_array[0] - self._metadata_dict['abi_sector_type'] = Goes._string_to_abisectortype(metadata_array[1]) - self._metadata_dict['abi_mode'] = Goes._string_to_abimode(metadata_array[1]) - self._metadata_dict['abi_channel'] = int(metadata_array[1][-2:]) - self._metadata_dict['platform_identifier'] = metadata_array[2] - year = int(metadata_array[3][1:-1][0:4]) - day_of_year = int(metadata_array[3][1:-1][4:7]) - hour = int(metadata_array[3][1:-1][7:9]) - minute = int(metadata_array[3][1:-1][9:11]) - second = int(metadata_array[3][1:-1][11:13]) - scan_start_datetime = datetime.datetime(year, 1, 1, hour, minute, second) + datetime.timedelta(day_of_year - 1) - self._metadata_dict['day_of_year'] = day_of_year - self._metadata_dict['start_date'] = scan_start_datetime + self._metadata_dict['platform_identifier'] = self._load_metadata_platform() + self._metadata_dict['instrument_identifier'] = self._load_metadata_instrument() + self._title = self._load_metadata_title() + metadata_elements = self._title.split(' ') + self._metadata_dict['instrument'] = metadata_elements[0] + self._metadata_dict['processing_level'] = metadata_elements[1] + self._metadata_dict['product_acronym'] = metadata_elements[2] + self._metadata_dict['abi_sector_type'] = self._load_metadata_scene_id() + self._metadata_dict['yaw_flip_flag'] = self._load_yaw_flip_flag() + time_coverage_start = self._load_metadata_timestamp() + self._metadata_dict['start_date'] = datetime.datetime.fromisoformat(time_coverage_start[:-3]) + + # Only the channel should change per file, nothing else, so we could do steps above only once. + self._metadata_dict['abi_channel'] = self._load_band_variable() def _open(self): """ Opens a netCDF4 dataset using input_file_path. """ self._input_dataset = Dataset(self._input_file_path, 'r') + self._input_dataset.set_auto_scale(True) + + def _load_metadata_platform(self): + """ + Creates a local platform variable + """ + self._platform_id = self._input_dataset.getncattr('platform_ID') + return self._platform_id + + def _load_metadata_instrument(self): + """ + Creates a local instrument variable + """ + self._instrument_id = self._input_dataset.getncattr('instrument_ID') + return self._instrument_id + + def _load_metadata_scene_id(self): + """ + Creates a local scene ID variable (Full Disk, CONUS, Mesoscale) + """ + self._scene_id = self._input_dataset.getncattr('scene_id') + return self._scene_id + + def _load_metadata_title(self): + """ + Creates a local title variable (ABI L1b Radiances) + """ + self._title = self._input_dataset.getncattr('title') + return self._title + + def _load_metadata_timestamp(self): + """ + Creates a local timestamp (ISO-8601) variable + """ + self._timestamp = self._input_dataset.getncattr('time_coverage_start') + return self._timestamp + + def _load_band_variable(self): + """ + Creates a local band (channel number) variable (convert byte to int). + """ + self._band = int(self._input_dataset.variables['band_id'][0]) + return self._band + + def _load_yaw_flip_flag(self): + """ + Creates a local yaw_flip_flag variable + """ + self._yaw_flip_flag = self._input_dataset.variables['yaw_flip_flag'][0] + return self._yaw_flip_flag + + def get_abi_channel(self): + """ + Returns the ABI channel. + """ + return self._metadata_dict['abi_channel'] + + def get_start_date(self): + """ + Returns the scan's start date as a datetime object + """ + return self._metadata_dict['start_date'] + + def get_yaw_flip_flag(self): + """ + Returns the yaw_flip_flag + """ + return self._metadata_dict['yaw_flip_flag'] + + def get_platform_id(self): + """ + Returns the platform_id information + """ + return self._metadata_dict['platform_identifier'] def _load_kappa0_variable(self): """ @@ -106,32 +182,6 @@ def _load_rad_data_array(self): """ self._rad_data_array = ma.getdata(self._input_dataset.variables['Rad'][:].real) - @staticmethod - def _string_to_abimode(string): - """ - Selects the ABI Mode Enum constant from string. - string - the string used for the match - """ - if 'M4' in string: - return ABIMode.ABI_SCAN_MODE_4 - if 'M6' in string: - return ABIMode.ABI_SCAN_MODE_6 - - @staticmethod - def _string_to_abisectortype(string): - """ - Selects the ABI Sector Type constant from string. - string - the string used for the match - """ - if 'F' in string: - return ABISectorType.FULL_DISK - if 'C' in string: - return ABISectorType.CONUS - if 'M1' in string: - return ABISectorType.MESOSCALE_REGION_1 - if 'M2' in string: - return ABISectorType.MESOSCALE_REGION_2 - def _create_obsvalue_rf_data_array(self): """ Creates a local data array variable containing the calculated obsvalue reflectance factor data @@ -167,30 +217,6 @@ def _create_obserror_bt_data_array(self): sqrt_comp = np.power(sqrt_comp_1 * sqrt_comp_2, 2) * np.power(self._std_dev_radiance_value_of_valid_pixels, 2) self._obserror_bt_data_array = np.sqrt(sqrt_comp) / np.sqrt(self._valid_pixel_count) - def get_abi_channel(self): - """ - Returns the ABI channel. - """ - return self._metadata_dict['abi_channel'] - - def get_platform_identifier(self): - """ - Returns the platform identifier. - """ - return self._metadata_dict['platform_identifier'] - - def get_start_date(self): - """ - Returns the scan's start date as a datetime object - """ - return self._metadata_dict['start_date'] - - def get_day_of_year(self): - """ - Returns the scan's day of year. - """ - return self._metadata_dict['day_of_year'] - def get_input_file_path(self): """ Returns the input_file_path. @@ -244,8 +270,6 @@ def load(self): """ Loads, calculates, sub-samples, reshapes, and filters all data arrays required by the GoesConverter class. """ - self._open() - self._input_dataset.set_auto_scale(True) self._load_kappa0_variable() self._load_planck_variables() self._load_std_dev_radiance_value_of_valid_pixels_variable() @@ -286,21 +310,3 @@ def load(self): self._create_obserror_bt_data_array() self.close() - - -# -# This enumeration is for the ABI Mode. -# -class ABIMode(Enum): - ABI_SCAN_MODE_4 = 1 - ABI_SCAN_MODE_6 = 2 - - -# -# This enumeration is for the ABI Sector Type -# -class ABISectorType(Enum): - FULL_DISK = 1 - CONUS = 2 - MESOSCALE_REGION_1 = 3 - MESOSCALE_REGION_2 = 4 diff --git a/src/goes/goes_converter.py b/src/goes/goes_converter.py index cccea23a3..c496db5e3 100755 --- a/src/goes/goes_converter.py +++ b/src/goes/goes_converter.py @@ -14,18 +14,18 @@ # / -> date_time # / -> _ioda_layout # / -> _ioda_layout_version -# /MetaData/datetime -# /MetaData/elevation_angle -# /MetaData/latitude -# /MetaData/longitude -# /MetaData/scan_angle +# /MetaData/dateTime +# /MetaData/latitude -> units +# /MetaData/longitude -> units +# /MetaData/elevation_angle -> units +# /MetaData/scan_angle -> units # /MetaData/scan_position # /MetaData/sensor_azimuth_angle -> units # /MetaData/sensor_view_angle -> units # /MetaData/sensor_zenith_angle -> units # /MetaData/solar_azimuth_angle -> units # /MetaData/solar_zenith_angle -> units -# /MetaData/time +# /MetaData/sensor_channel # /ObsError/reflectance_factor or /ObsError/brightness_temperature # /ObsValue/reflectance_factor or /ObsValue/brightness_temperature # /ObsError/brightness_temperature -> units @@ -33,13 +33,8 @@ # /PreQC/reflectance_factor or /PreQC/brightness_temperature # /PreQC/reflectance_factor -> flag_values or /PreQC/brightness_temperature -> flag_values # /PreQC/reflectance_factor -> flag_meanings or /PreQC/brightness_temperature -> flag_meanings -# /VarMetaData/sensor_channel -# /VarMetaData/variable_names # /nchans -# /ndatetime # /nlocs -# /nstring -# /nvars # import os @@ -47,6 +42,7 @@ import numpy as np from netCDF4 import Dataset from numpy import ma +import datetime from goes import Goes from goes_latlon import GoesLatLon from goes_util import GoesUtil @@ -74,6 +70,11 @@ def __init__(self, input_file_paths, latlon_file_path, output_file_path_rf, outp self._latlon_dataset = None self._check_arguments() + # TODO: make use of default fill values from NetCDF + # self._float_missing_value = nc.default_fillvals['f4'] + # self._int_missing_value = nc.default_fillvals['i4'] + # self._long_missing_value = nc.default_fillvals['i8'] + def _check_arguments(self): """ Checks the input arguments. @@ -83,8 +84,8 @@ def _check_arguments(self): print("ERROR: input_file_paths must contain 16 GOES-16 or GOES-17 data files. One for each ABI channel.") good_args = False good_resolutions = [2, 4, 8, 16, 32, 64] - if self._resolution not in good_resolutions: - print("ERROR: resolution (in km) must be one of the following values: 2 (default), 4, 8, 16, 32, 64.") + if int(self._resolution) not in good_resolutions: + print("ERROR: resolution (in km) must be one of the following values: 2, 4, 8 (default), 16, 32, 64.") good_args = False if not good_args: sys.exit(2) @@ -101,26 +102,17 @@ def _initialize(self): self._goes_dict_bt = {} for input_file_path in self._input_file_paths: goes = Goes(input_file_path, self._goes_util) - abi_channel = int(goes.get_abi_channel()) + abi_channel = goes.get_abi_channel() if abi_channel < 7: self._goes_dict_rf[abi_channel] = goes else: self._goes_dict_bt[abi_channel] = goes - self._day_of_year = self._goes_dict_bt[7].get_day_of_year() - self._start_date = self._goes_dict_bt[7].get_start_date() + self._platform_id = goes.get_platform_id() + self._start_date = goes.get_start_date() self._input_file_path_template = self._goes_dict_bt[7].get_input_file_path() - self._goes_util.set_yaw_flip_flag(self._get_yaw_flip_flag()) + self._goes_util.set_yaw_flip_flag(goes.get_yaw_flip_flag()) self._goes_util.set_resolution(self._resolution) - def _get_yaw_flip_flag(self): - """ - Returns the yaw_flip_flag from channel 1 - """ - dataset = Dataset(self._input_file_paths[0], 'r') - yaw_flip_flag = dataset.variables['yaw_flip_flag'][0] - dataset.close() - return yaw_flip_flag - def _check_nadir(self): """ Returns a boolean variable indicating whether the nadir has changed by comparing the lat_nadir and lon_nadir @@ -171,6 +163,7 @@ def _create_metadata_latitude_variable(self, output_dataset): latitude_data_array = self._latlon_dataset['MetaData'].variables['latitude'][:].real output_dataset.createVariable('/MetaData/latitude', 'f4', 'nlocs', fill_value=-999) output_dataset['/MetaData/latitude'][:] = latitude_data_array + output_dataset['/MetaData/latitude'].setncattr('units', 'degrees_north') def _create_metadata_longitude_variable(self, output_dataset): """ @@ -180,6 +173,7 @@ def _create_metadata_longitude_variable(self, output_dataset): longitude_data_array = self._latlon_dataset['MetaData'].variables['longitude'][:].real output_dataset.createVariable('/MetaData/longitude', 'f4', 'nlocs', fill_value=-999) output_dataset['/MetaData/longitude'][:] = longitude_data_array + output_dataset['/MetaData/longitude'].setncattr('units', 'degrees_east') def _create_metadata_scan_angle_variable(self, output_dataset): """ @@ -189,6 +183,7 @@ def _create_metadata_scan_angle_variable(self, output_dataset): scan_angle_data_array = self._latlon_dataset['MetaData'].variables['scan_angle'][:].real output_dataset.createVariable('/MetaData/scan_angle', 'f4', 'nlocs', fill_value=-999) output_dataset['/MetaData/scan_angle'][:] = scan_angle_data_array + output_dataset['/MetaData/scan_angle'].setncattr('units', 'degrees') def _create_metadata_elevation_angle_variable(self, output_dataset): """ @@ -198,6 +193,7 @@ def _create_metadata_elevation_angle_variable(self, output_dataset): elevation_angle_data_array = self._latlon_dataset['MetaData'].variables['elevation_angle'][:].real output_dataset.createVariable('/MetaData/elevation_angle', 'f4', 'nlocs', fill_value=-999) output_dataset['/MetaData/elevation_angle'][:] = elevation_angle_data_array + output_dataset['/MetaData/elevation_angle'].setncattr('units', 'degrees') def _create_metadata_scan_position_variable(self, output_dataset): """ @@ -328,7 +324,6 @@ def _create_groups(output_dataset): output_dataset.createGroup('ObsError') output_dataset.createGroup('ObsValue') output_dataset.createGroup('PreQC') - output_dataset.createGroup('VarMetaData') @staticmethod def _create_nchans_dimension(output_dataset, nchans): @@ -346,80 +341,24 @@ def _create_nchans_dimension(output_dataset, nchans): output_dataset.variables['nchans'][:] = [7, 8, 9, 10, 11, 12, 13, 14, 15, 16] @staticmethod - def _create_nvars_dimension(output_dataset, nvars): - """ - Creates the nvars dimension in an output netCDF4 dataset. - output_dataset - A netCDF4 Dataset object - nvars - An integer indicating the number of nvars: 6 (for reflectance factor) - or 10 (for brightness temperature) - """ - output_dataset.createDimension('nvars', nvars) - output_dataset.createVariable('nvars', 'i4', 'nvars') - output_dataset.variables['nvars'][:] = np.arange(1, nvars + 1, 1, dtype='int32') - - @staticmethod - def _create_ndatetime_dimension(output_dataset): - """ - Creates the ndatetime dimension in an output netCDF4 dataset. - output_dataset - A netCDF4 Dataset object - """ - ndatetime = 20 - output_dataset.createDimension('ndatetime', ndatetime) - output_dataset.createVariable('ndatetime', 'i4', 'ndatetime') - output_dataset.variables['ndatetime'][:] = np.arange(1, ndatetime + 1, 1, dtype='int32') - - @staticmethod - def _create_nstring_dimension(output_dataset): + def _create_metadata_sensor_channel_variable(output_dataset): """ - Creates the nstring dimension in an output netCDF4 dataset. + Creates the /MetaData/sensor_channel variable in an output netCDF4 dataset. output_dataset - A netCDF4 Dataset object """ - nstring = 50 - output_dataset.createDimension('nstring', nstring) - output_dataset.createVariable('nstring', 'i4', 'nstring') - output_dataset.variables['nstring'][:] = np.arange(1, nstring + 1, 1, dtype='int32') + output_dataset.createVariable('/MetaData/sensor_channel', 'i4', 'nchans') + output_dataset['/MetaData/sensor_channel'][:] = output_dataset['nchans'][:] @staticmethod - def _create_varmetadata_sensor_channel_variable(output_dataset): - """ - Creates the /VarMetaData/sensor_channel variable in an output netCDF4 dataset. - output_dataset - A netCDF4 Dataset object - """ - output_dataset.createVariable('/VarMetaData/sensor_channel', 'i4', 'nchans') - output_dataset['/VarMetaData/sensor_channel'][:] = output_dataset['nchans'][:] - - @staticmethod - def _create_varmetadata_variable_names_reflectance_factor_variable(output_dataset): - """ - Creates the /VarMetaData/variable_names variable for reflectance factor in an output netCDF4 dataset. - output_dataset - A netCDF4 Dataset object - """ - output_dataset.createVariable('/VarMetaData/variable_names', 'str', 'nchans') - temp_data_array = ['reflectance_factor_1', 'reflectance_factor_2', 'reflectance_factor_3', - 'reflectance_factor_4', 'reflectance_factor_5', 'reflectance_factor_6'] - output_dataset['/VarMetaData/variable_names'][:] = np.array(temp_data_array) - - @staticmethod - def _create_varmetadata_variable_names_brightness_temperature_variable(output_dataset): - """ - Creates the /VarMetaData/variable_names variable for brightness temperature in an output netCDF4 dataset. - output_dataset - A netCDF4 Dataset object - """ - output_dataset.createVariable('/VarMetaData/variable_names', 'str', 'nchans') - temp_data_array = ['brightness_temperature_7', 'brightness_temperature_8', 'brightness_temperature_9', - 'brightness_temperature_10', 'brightness_temperature_11', 'brightness_temperature_12', - 'brightness_temperature_13', 'brightness_temperature_14', 'brightness_temperature_15', - 'brightness_temperature_16'] - output_dataset['/VarMetaData/variable_names'][:] = np.array(temp_data_array) - - @staticmethod - def _create_root_group_attributes(output_dataset): + def _create_root_group_attributes(output_dataset, resolution, platform_id): """ Creates several root group attributes in an output netCDF4 dataset. output_dataset - A netCDF4 Dataset object """ output_dataset.setncattr('_ioda_layout', 'ObsGroup') output_dataset.setncattr('_ioda_layout_version', '0') + output_dataset.setncattr('subsampled_resolution (km)', resolution) + output_dataset.setncattr('platform_identifier', platform_id) @staticmethod def _get_nlocs(dataset): @@ -548,18 +487,19 @@ def _create_obserror_brightness_temperature_variable(self, output_dataset): def _create_metadata_time_variable(self, output_dataset): """ - Creates the /MetaData/datetime and MetaData/time variables and /date_time attribute in an output netCDF4 - dataset. + Creates the /MetaData/dateTime variable and /date_time attribute in an output netCDF4 dataset. output_dataset - A netCDF4 Dataset object """ + iso8601_string = 'seconds since 1970-01-01T00:00:00Z' + epoch = datetime.datetime.fromisoformat(iso8601_string[14:-1]) + time_offset = round((self._start_date - epoch).total_seconds()) # seconds since epoch. start_date = self._start_date.strftime('%Y-%m-%dT%H:%M:%SZ') - datetime_array = np.full(self._get_nlocs(output_dataset), start_date) - time_array = np.full(self._get_nlocs(output_dataset), 0.0) - output_dataset.createVariable('/MetaData/datetime', 'str', 'nlocs') - output_dataset['/MetaData/datetime'][:] = datetime_array - output_dataset.createVariable('/MetaData/time', 'f4', 'nlocs') - output_dataset['/MetaData/time'][:] = time_array output_dataset.setncattr("date_time", start_date) + datetime_array = np.full(self._get_nlocs(output_dataset), np.int64(time_offset)) + + output_dataset.createVariable('/MetaData/dateTime', 'i8', 'nlocs') + output_dataset['/MetaData/dateTime'][:] = datetime_array + output_dataset['/MetaData/dateTime'].setncattr('units', iso8601_string) def _load_all_goes(self): """ @@ -597,14 +537,10 @@ def _convert_bt(self): """ dataset = Dataset(self._output_file_path_bt, 'w') GoesConverter._create_groups(dataset) - GoesConverter._create_root_group_attributes(dataset) + GoesConverter._create_root_group_attributes(dataset, self._resolution, self._platform_id) GoesConverter._create_nchans_dimension(dataset, 10) - GoesConverter._create_nvars_dimension(dataset, 10) - GoesConverter._create_ndatetime_dimension(dataset) - GoesConverter._create_nstring_dimension(dataset) self._create_nlocs_dimension(dataset) - GoesConverter._create_varmetadata_sensor_channel_variable(dataset) - GoesConverter._create_varmetadata_variable_names_brightness_temperature_variable(dataset) + GoesConverter._create_metadata_sensor_channel_variable(dataset) self._create_metadata_latitude_variable(dataset) self._create_metadata_longitude_variable(dataset) self._create_metadata_elevation_angle_variable(dataset) @@ -627,14 +563,10 @@ def _convert_rf(self): """ dataset = Dataset(self._output_file_path_rf, 'w') GoesConverter._create_groups(dataset) - GoesConverter._create_root_group_attributes(dataset) + GoesConverter._create_root_group_attributes(dataset, self._resolution, self._platform_id) GoesConverter._create_nchans_dimension(dataset, 6) - GoesConverter._create_nvars_dimension(dataset, 6) - GoesConverter._create_ndatetime_dimension(dataset) - GoesConverter._create_nstring_dimension(dataset) self._create_nlocs_dimension(dataset) - GoesConverter._create_varmetadata_sensor_channel_variable(dataset) - GoesConverter._create_varmetadata_variable_names_reflectance_factor_variable(dataset) + GoesConverter._create_metadata_sensor_channel_variable(dataset) self._create_metadata_latitude_variable(dataset) self._create_metadata_longitude_variable(dataset) self._create_metadata_elevation_angle_variable(dataset) diff --git a/src/goes/goes_util.py b/src/goes/goes_util.py index 3f79a459b..010f035c6 100644 --- a/src/goes/goes_util.py +++ b/src/goes/goes_util.py @@ -33,7 +33,7 @@ def set_resolution(self, resolution): Sets the resolution variable and array slicing increment for sub_sampling resolution - the resolution """ - self._resolution = resolution + self._resolution = int(resolution) self._increment = int(self._resolution / 2) def set_nonexistent_indices_data_array(self, nonexistent_indices_data_array): diff --git a/src/goes/test_goes_converter.py b/src/goes/test_goes_converter.py index 85b8236a4..51eea7874 100644 --- a/src/goes/test_goes_converter.py +++ b/src/goes/test_goes_converter.py @@ -9,18 +9,9 @@ from goes_converter import GoesConverter -def test_goes_converter(input_file_paths, latlon_file_path, output_file_path_rf, output_file_path_bt, include_rf, - resolution): - """ - Test driver for the GoesConverter class. - input_file_paths - a list containing 16 absolute paths corresponding to the output files for each channel of a - single GOERS_16 or GOES-17 scan - latlon_file_path - The path to an existing GoesLatLon file or if it does not exist the path to write the file - output_file_path_rf - The path to write the IODAv2 reflectance factor data file - output_file_path_bt - The path to write the IODAv2 brightness temperature data file - include_rf - Boolean value indicating whether to create the reflectance factor output data file: True or False - resolution - The resolution in km: 2, 4, 8, 16, 32, or 64 - """ +def test_goes_converter(input_file_paths, latlon_file_path, output_file_path_rf, output_file_path_bt, + include_rf, resolution): + goes_converter = \ GoesConverter(input_file_paths, latlon_file_path, output_file_path_rf, output_file_path_bt, include_rf=include_rf, resolution=resolution) @@ -28,16 +19,30 @@ def test_goes_converter(input_file_paths, latlon_file_path, output_file_path_rf, if __name__ == '__main__': + """ + Test driver for the GoesConverter class. + All filenames in argument list MUST be absolute paths, not relative. + + The 1st argument is a comma-separated list of 16 incoming ABI channel files (absolute paths). + The 2nd argument is the reference lat/lon file (absolute path). If already existing, it is + cross-checked for the same nadir longitude in the reference file compared to the files + being processed. If this file does not exist, it gets created for usage thereafter. + The 3rd argument is the output reflectance (first 6 channels) data file (absolute path). + The 4th argument is the output brightness temperature (channels 7-16) data file (absolute path). + The 5th argument is the target resolution (2, 4, 8, 16, 32, or 64 km). + The 6th argument should be True to output reflectances or False to skip, however argument 3 + must be provided regardless. + """ + start_time = time.time() - # input_file_paths should be a comma separated list containing 16 absolute paths corresponding to - # the output files for each channel of a single GOES_16 or GOES-17 scan input_file_paths = sys.argv[1].split(',') latlon_file_path = sys.argv[2] output_file_path_rf = sys.argv[3] output_file_path_bt = sys.argv[4] resolution = sys.argv[5] include_rf = sys.argv[6] - test_goes_converter(input_file_paths, latlon_file_path, output_file_path_rf, output_file_path_bt, include_rf, - resolution) + + test_goes_converter(input_file_paths, latlon_file_path, output_file_path_rf, output_file_path_bt, + include_rf, resolution) elapsed_time = time.time() - start_time print(f'elapsed time:{elapsed_time:.3g}s') diff --git a/src/gsi-ncdiag/combine_obsspace.py b/src/gsi-ncdiag/combine_obsspace.py index e5bb294ef..b743814bd 100755 --- a/src/gsi-ncdiag/combine_obsspace.py +++ b/src/gsi-ncdiag/combine_obsspace.py @@ -218,12 +218,12 @@ def combine_obsspace(FileList, OutFile, GeoDir): if v in obsspace.variables: _var = obsspace.Variable(v) tmpdata = np.array(_var.read_data()) + del _var else: tmpdata = np.full((obsspace.nlocs), iconv.get_default_fill_val(GeoVarTypes[v]), dtype=GeoVarTypes[v]) tmpgeodata.append(tmpdata) tmpgeoidx.append(np.ones_like(tmpdata).astype(int)*int(idx2)) - del _var del obsspace tmpgeodata = np.hstack(tmpgeodata) tmpgeoidx = np.hstack(tmpgeoidx) @@ -244,12 +244,12 @@ def combine_obsspace(FileList, OutFile, GeoDir): if v in obsspace.variables: _var = obsspace.Variable(v) tmpdata = np.array(_var.read_data()) + del _var else: tmpdata = np.full((obsspace.nlocs, nlevs), iconv.get_default_fill_val(GeoVarTypes[v]), dtype=GeoVarTypes[v]) tmpgeodata.append(tmpdata) tmpgeoidx.append(np.ones_like(tmpdata).astype(int)*int(idx2)) - del _var del obsspace tmpgeodata = np.vstack(tmpgeodata) tmpgeoidx = np.vstack(tmpgeoidx) @@ -268,12 +268,12 @@ def combine_obsspace(FileList, OutFile, GeoDir): if v in obsspace.variables: _var = obsspace.Variable(v) tmpdata = np.array(_var.read_data()) + del _var else: tmpdata = np.full((obsspace.nlocs, ninterfaces), iconv.get_default_fill_val(GeoVarTypes[v]), dtype=GeoVarTypes[v]) tmpgeodata.append(tmpdata) tmpgeoidx.append(np.ones_like(tmpdata).astype(int)*int(idx2)) - del _var del obsspace tmpgeodata = np.vstack(tmpgeodata) tmpgeoidx = np.vstack(tmpgeoidx) diff --git a/src/gsi-ncdiag/gsi_ncdiag.py b/src/gsi-ncdiag/gsi_ncdiag.py index 547ff46ca..2fcf71169 100644 --- a/src/gsi-ncdiag/gsi_ncdiag.py +++ b/src/gsi-ncdiag/gsi_ncdiag.py @@ -438,7 +438,7 @@ 'height_above_mean_sea_level': 'm', 'heightOfSurface': 'm', 'surface_pressure': 'Pa', - 'station{ressure': 'Pa', + 'station{ressure': 'Pa', # Is this for real, curly-brace??? or a typo that should be "p"? 'sea_surface_temperature': 'K', 'surface_temperature': 'K', 'surface_roughness_length': 'm', diff --git a/src/gsi-ncdiag/proc_gsi_ncdiag.py b/src/gsi-ncdiag/proc_gsi_ncdiag.py index 05d5bee2c..106a8f72f 100755 --- a/src/gsi-ncdiag/proc_gsi_ncdiag.py +++ b/src/gsi-ncdiag/proc_gsi_ncdiag.py @@ -142,7 +142,8 @@ def run_radiances_obsdiag(radfile, outdir): # ozone for radfile in radfiles: process = False - for p in gsid.oz_sensors: + oz_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors + for p in oz_sensors: if p in radfile: process = True if process: @@ -172,7 +173,8 @@ def run_radiances_obsdiag(radfile, outdir): # ozone for radfile in radfiles: process = False - for p in gsid.oz_sensors: + oz_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors + for p in oz_sensors: if p in radfile: process = True if process: diff --git a/src/land/smap9km_ssm2ioda.py b/src/land/smap9km_ssm2ioda.py index 0d8f30c75..b6b5ab41f 100644 --- a/src/land/smap9km_ssm2ioda.py +++ b/src/land/smap9km_ssm2ioda.py @@ -25,7 +25,8 @@ locationKeyList = [ ("latitude", "float"), ("longitude", "float"), - ("dateTime", "string") + ("depthBelowSoilSurface", "float"), + ("datetime", "string") ] obsvars = { @@ -44,9 +45,10 @@ class smap(object): - def __init__(self, filename, mask): - self.filename = filename - self.mask = mask + def __init__(self, args): + self.filename = args.input + self.mask = args.maskMissing + self.assumedSoilDepth = args.assumedSoilDepth self.varDict = defaultdict(lambda: defaultdict(dict)) self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) self.varAttrs = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) @@ -89,14 +91,16 @@ def _read(self): ecoli = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['EASE_column_index'][:].ravel() refsec = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['tb_time_seconds'][:].ravel() + deps = np.full_like(vals, self.assumedSoilDepth) times = np.empty_like(vals, dtype=object) - if self.mask == "maskout": + if self.mask: with np.errstate(invalid='ignore'): mask = (vals > valid_min) & (vals < valid_max) vals = vals[mask] lats = lats[mask] lons = lons[mask] + deps = deps[mask] errs = errs[mask] qflg = qflg[mask] sflg = sflg[mask] @@ -111,6 +115,7 @@ def _read(self): vals = vals.astype('float32') lats = lats.astype('float32') lons = lons.astype('float32') + deps = deps.astype('float32') errs = errs.astype('float32') qflg = qflg.astype('int32') sflg = sflg.astype('int32') @@ -126,9 +131,12 @@ def _read(self): self.outdata[('dateTime', 'MetaData')] = times self.outdata[('latitude', 'MetaData')] = lats self.outdata[('longitude', 'MetaData')] = lons - self.varAttrs[('latitude', 'MetaData')]['units'] = 'degrees_north' - self.varAttrs[('longitude', 'MetaData')]['units'] = 'degrees_east' - self.outdata[('earthSurfaceType', 'MetaData')] = sflg + self.varAttrs[('latitude', 'MetaData')]['units'] = 'degree_north' + self.varAttrs[('longitude', 'MetaData')]['units'] = 'degree_east' + self.outdata[('depthBelowSoilSurface', 'MetaData')] = deps + self.varAttrs[('depthBelowSoilSurface', 'MetaData')]['units'] = 'm' + self.outdata[('surfaceFlag', 'MetaData')] = sflg + self.varAttrs[('surfaceFlag', 'MetaData')]['units'] = 'unitless' self.outdata[('vegetationOpacity', 'MetaData')] = vegop self.outdata[('easeRowIndex', 'MetaData')] = erowi self.varAttrs[('easeRowIndex', 'MetaData')]['units'] = '1' @@ -158,14 +166,18 @@ def main(): type=str, required=True) optional = parser.add_argument_group(title='optional arguments') optional.add_argument( - '-m', '--mask', - help="maskout missing values: maskout/default, default=none", - type=str, required=True) + '-m', '--maskMissing', + help="switch to mask missing values: default=False", + default=False, action='store_true', required=False) + optional.add_argument( + '-d', '--assumedSoilDepth', + help="default assumed depth of soil moisture in meters", + type=float, default=0.025, required=False) args = parser.parse_args() # Read in the SMAP volumetric soil moisture data - ssm = smap(args.input, args.mask) + ssm = smap(args) # setup the IODA writer writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) diff --git a/src/land/smap_ssm2ioda.py b/src/land/smap_ssm2ioda.py index 88352893b..b477f23f0 100644 --- a/src/land/smap_ssm2ioda.py +++ b/src/land/smap_ssm2ioda.py @@ -25,7 +25,8 @@ locationKeyList = [ ("latitude", "float"), ("longitude", "float"), - ("dateTime", "string") + ("depthBelowSoilSurface", "float"), + ("datetime", "string") ] obsvars = { @@ -44,9 +45,10 @@ class smap(object): - def __init__(self, filename, mask): - self.filename = filename - self.mask = mask + def __init__(self, args): + self.filename = args.input + self.mask = args.maskMissing + self.assumedSoilDepth = args.assumedSoilDepth self.varDict = defaultdict(lambda: defaultdict(dict)) self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) self.varAttrs = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) @@ -83,14 +85,17 @@ def _read(self): lons = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['longitude'][:].ravel() errs = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['soil_moisture_error'][:].ravel() qflg = ncd.groups['Soil_Moisture_Retrieval_Data'].variables['retrieval_qual_flag'][:].ravel() + + deps = np.full_like(vals, self.assumedSoilDepth) times = np.empty_like(vals, dtype=object) - if self.mask == "maskout": + if self.mask: with np.errstate(invalid='ignore'): mask = (vals > valid_min) & (vals < valid_max) vals = vals[mask] lats = lats[mask] lons = lons[mask] + deps = deps[mask] errs = errs[mask] qflg = qflg[mask] times = times[mask] @@ -103,6 +108,7 @@ def _read(self): vals = vals.astype('float32') lats = lats.astype('float32') lons = lons.astype('float32') + deps = deps.astype('float32') errs = errs.astype('float32') qflg = qflg.astype('int32') @@ -124,6 +130,8 @@ def _read(self): self.outdata[('dateTime', 'MetaData')] = times self.outdata[('latitude', 'MetaData')] = lats self.outdata[('longitude', 'MetaData')] = lons + self.outdata[('depthBelowSoilSurface', 'MetaData')] = deps + self.varAttrs[('depthBelowSoilSurface', 'MetaData')]['units'] = 'm' for iodavar in ['soilMoistureVolumetric']: self.outdata[self.varDict[iodavar]['valKey']] = vals @@ -148,14 +156,18 @@ def main(): type=str, required=True) optional = parser.add_argument_group(title='optional arguments') optional.add_argument( - '-m', '--mask', - help="maskout missing values: maskout/default, default=none", - type=str, required=True) + '-m', '--maskMissing', + help="switch to mask missing values: default=False", + default=False, action='store_true', required=False) + optional.add_argument( + '-d', '--assumedSoilDepth', + help="default assumed depth of soil moisture in meters", + type=float, default=0.025, required=False) args = parser.parse_args() # Read in the SMAP volumetric soil moisture data - ssm = smap(args.input, args.mask) + ssm = smap(args) # setup the IODA writer writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) diff --git a/src/lib-python/CMakeLists.txt b/src/lib-python/CMakeLists.txt index 49493f36d..7425b507a 100644 --- a/src/lib-python/CMakeLists.txt +++ b/src/lib-python/CMakeLists.txt @@ -4,6 +4,7 @@ list( APPEND libs ioda_conv_engines.py collect_sources.py meteo_utils.py + meteo_sounding_utils.py orddicts.py ) diff --git a/src/lib-python/meteo_sounding_utils.py b/src/lib-python/meteo_sounding_utils.py new file mode 100644 index 000000000..73c161421 --- /dev/null +++ b/src/lib-python/meteo_sounding_utils.py @@ -0,0 +1,165 @@ +# --+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ +# Set of routines to do interpolation/extrapolation of P/Z/T sounding data. +# These routines should allow one to compute a full set of P/Z/T data given +# the various sounding parts (e.g. TTAA (mandatory P,Z,T), TTBB (significant +# P,T) and PPBB (winds given height alone). +# +# Peter Neilley, NCAR/RAP 8/93 +# +# --+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + +import math + +# Constants +Rd = 287.05 +Cp = 1004.0 +G = 9.8 +XMS2KTS = 1.94 +RCP = Rd/Cp +CPR = Cp/Rd +CTOK = 273.15 + + +def pext_down(pres_upper, temperature, hght_upper, hght_lower): + """ + Returns pressure of a lower height given a + temperature (assume T=constant) and heights. + :param pres_upper: Pressure of upper level (mb) + :param temperature: Constant temperature (C) + :param hght_upper: Height of upper layer (m) + :param hght_lower: Height of lower layer (m) + :return: Extrapolated pressure (mb) + """ + if pres_upper <= 0 or temperature is None or hght_upper is None or hght_lower is None: + return None + + return pres_upper * math.exp((G / (Rd * (temperature + CTOK))) * (hght_upper - hght_lower)) + +# --+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + + +def pext_up(pres_lower, temperature, hght_upper, hght_lower): + """ + Returns pressure of an upper height given a + temperature (assume T=constant) and heights. + :param pres_upper: Pressure of upper level (mb) + :param temperature: Constant temperature (C) + :param hght_upper: Height of upper layer (m) + :param hght_lower: Height of lower layer (m) + :return: Extrapolated pressure (mb) + """ + if pres_lower <= 0 or temperature is None or hght_upper is None or hght_lower is None: + return None + + return pres_lower * math.exp((G / (Rd * (temperature + CTOK))) * (hght_lower - hght_upper)) + +# --+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + + +def zext_up(pres_lower, pres_upper, temperature, hght_lower): + """ + Returns height of an upper pressure given a + temperature (assume T=constant) and pressures. + :param pres_lower: Pressure of lower level (mb) + :param pres_upper: Pressure of upper level (mb) + :param temperature: Constant temperature (C) + :param hght_lower: Height of lower layer (m) + :return: Extrapolated height (m) + """ + return hght_lower + (Rd * (temperature + CTOK) / G) * math.log(pres_lower / pres_upper) + +# --+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + + +def zext_down(pres_lower, pres_upper, temperature, hght_upper): + """ + Returns height of a lower pressure given a + temperature (assume T=constant) and pressures. + :param pres_lower: Pressure of lower level (mb) + :param pres_upper: Pressure of upper level (mb) + :param temperature: Constant temperature (C) + :param hght_upper: Height of upper layer (m) + :return: Extrapolated height (m) + """ + return hght_upper - (Rd * (temperature + CTOK) / G) * math.log(pres_lower / pres_upper) + +# --+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + + +def p_interp(temp_lower, temp_upper, pres_lower, pres_upper, hght_lower, hght_upper, height): + """ + Returns the pressure level of an intermediate height, given pressure,temperature and height at + two surrounding levels. Make an upward and downward integration of the hydrostatic equation to + the desired level, and then use the mean of the two computation for the result. + Assumes temperature varies linearly with log(P). + :param temp_lower: Temperature of lower layer (C) + :param temp_upper: Temperature of upper layer (C) + :param pres_lower: Pressure of lower layer (mb) + :param pres_upper: Pressure of upper layer (mb) + :param hght_lower: Height of lower layer (m) + :param hght_upper: Height of upper layer (m) + :param height: Height of target layer (m) + :return: Interpolated pressure (mb) + """ + if pres_lower <= 0 or pres_upper > 1200 or pres_upper <= 0 or pres_upper > 1200 \ + or hght_lower <= -1000 or hght_lower > 40000 or hght_upper <= -1000 or hght_upper > 40000 \ + or temp_lower is None or temp_upper is None: + return None + + # Speed up if t structure simple. + if temp_upper == temp_lower: + s = G / (Rd * (temp_lower + CTOK)) + pm1 = pres_lower * math.exp(s * (hght_lower - height)) + pm2 = pres_upper * math.exp(s * (hght_upper - height)) + else: + tl = temp_lower + CTOK + tu = temp_upper + CTOK + s = (tu - tl) / (hght_upper - hght_lower) + t0 = tl - s * hght_lower + a1 = math.log((t0 + s * height) / (t0 + s * hght_lower)) + a2 = math.log((t0 + s * hght_upper) / (t0 + s * height)) + b1 = G * a1 / (s * Rd) + b2 = G * a2 / (s * Rd) + pm1 = pres_lower / math.exp(b1) + pm2 = pres_upper * math.exp(b2) + + return (pm1 + pm2) / 2 + +# --+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + + +def z_interp(temp_lower, temp_upper, pres_lower, pres_upper, pressure, hght_lower, hght_upper): + """ + Returns the height of an intermediate pressure, given pressure,temperature and height at + two surrounding levels. Make an upward and downward integration of the hydrostatic equation to + the desired level, and then use the mean of the two computation for the result. + Assumes temperature varies linearly with log(P). + :param temp_lower: Temperature of lower layer (C) + :param temp_upper: Temperature of upper layer (C) + :param pres_lower: Pressure of lower layer (mb) + :param pres_upper: Pressure of upper layer (mb) + :param pressure: Target pressure (mb) + :param hght_lower: Height of lower layer (m) + :param hght_upper: Height of upper layer (m) + :return: Interpolated height (m) + """ + if pres_lower <= 0 or pres_upper > 1200 or pres_upper <= 0 or pres_upper > 1200 \ + or hght_lower <= -1000 or hght_lower > 40000 or hght_upper <= -1000 or hght_upper > 40000 \ + or temp_lower is None or temp_upper is None: + return None + + if temp_lower == temp_upper: + s = Rd * (temp_lower + CTOK) / G + z1 = hght_lower - s * math.log(pressure / pres_lower) + z2 = hght_upper + s * math.log(pres_upper / pressure) + else: + tl = temp_lower + CTOK + tu = temp_upper + CTOK + s = (tu - tl) / (hght_upper - hght_lower) + f1 = (s * Rd / G) * math.log(pres_lower / pressure) + f2 = (s * Rd / G) * math.log(pressure / pres_upper) + t0 = tl - s * hght_lower + z1 = ((t0 + s * hght_lower) * math.exp(f1) - t0) / s + z2 = ((t0 + s * hght_upper) * math.exp(-f2) - t0) / s + + return (z1 + z2) / 2 diff --git a/src/marine/CMakeLists.txt b/src/marine/CMakeLists.txt index bf55e6925..2ab3b9c94 100644 --- a/src/marine/CMakeLists.txt +++ b/src/marine/CMakeLists.txt @@ -1,4 +1,6 @@ list(APPEND programs + nsidc_l4cdr_ice2ioda.py + amsr2_icec2ioda.py copernicus_l4adt2ioda.py copernicus_l3swh2ioda.py cryosat_ice2ioda.py @@ -16,12 +18,14 @@ list(APPEND programs rads_adt2ioda.py smap_sss2ioda.py smos_sss2ioda.py + swot_l2adt2ioda.py argoClim2ioda.py viirs_modis_l2_oc2ioda.py viirs_modis_l3_oc2ioda.py godae_bgc_argo2ioda.py avhrr_radiance2ioda.py pace_oc2ioda.py + pace_radiance2ioda.py ostia_l4sst2ioda.py ) diff --git a/src/marine/amsr2_icec2ioda.py b/src/marine/amsr2_icec2ioda.py new file mode 100755 index 000000000..ee1494f10 --- /dev/null +++ b/src/marine/amsr2_icec2ioda.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +# +# (C) Copyright 2021 UCAR +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# + +import sys +import argparse +import numpy as np +from datetime import datetime, timedelta +import netCDF4 as nc +from pathlib import Path + +IODA_CONV_PATH = Path(__file__).parent/"../lib/pyiodaconv" +if not IODA_CONV_PATH.is_dir(): + IODA_CONV_PATH = Path(__file__).parent/'..'/'lib-python' +sys.path.append(str(IODA_CONV_PATH.resolve())) + +import ioda_conv_engines as iconv +from orddicts import DefaultOrderedDict + + +vName = "sea_ice_area_fraction" + +locationKeyList = [ + ("latitude", "float"), + ("longitude", "float"), + ("datetime", "string") +] + +GlobalAttrs = {} + + +class iceconc(object): + def __init__(self, filenames, date): + self.filenames = filenames + self.date = date + self.data = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) + self.VarAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) + self._read() + + # Open obs file and read/load relevant info + def _read(self): + valKey = vName, iconv.OvalName() + errKey = vName, iconv.OerrName() + qcKey = vName, iconv.OqcName() + + for f in self.filenames: + print(" Reading file: ", f) + ncd = nc.Dataset(f, 'r') + + source_var_name = { + 'lat': 'Latitude', + 'lon': 'Longitude', + 'icec': 'NASA_Team_2_Ice_Concentration', + 'icec_qc': 'Flags' + } + lon = ncd.variables['Longitude'][:] + lat = ncd.variables['Latitude'][:] + icec = ncd.variables['NASA_Team_2_Ice_Concentration'][:] + icec_FillValue = ncd.variables['NASA_Team_2_Ice_Concentration']._FillValue + icec_units = ncd.variables['NASA_Team_2_Ice_Concentration'].units + icec_qc = ncd.variables['Flags'][:] + qc_FillValue = ncd.variables['Flags']._FillValue + qc_units = ncd.variables['Flags'].units + icec_qc = icec_qc.astype(int) + mask = np.logical_not(icec.mask) + lon = lon[mask] + lat = lat[mask] + icec = icec[mask] + icec_qc = icec_qc[mask] + + for i in range(len(lon)): + # get date from filename + datestart = ncd.getncattr('time_coverage_start') + dateend = ncd.getncattr('time_coverage_end') + date1 = datetime.strptime(datestart, "%Y-%m-%dT%H:%M:%S.%fZ") + date2 = datetime.strptime(dateend, "%Y-%m-%dT%H:%M:%S.%fZ") + avg = date1 + (date2 - date1) * 0.5 + locKey = lat[i], lon[i], avg.strftime("%Y-%m-%dT%H:%M:%SZ") + self.data[locKey][valKey] = icec[i] * 0.01 + self.VarAttrs[locKey][valKey]['_FillValue'] = icec_FillValue + self.VarAttrs[locKey][valKey]['units'] = icec_units + self.data[locKey][errKey] = 0.1 + self.VarAttrs[locKey][errKey]['_FillValue'] = icec_FillValue + self.VarAttrs[locKey][errKey]['units'] = icec_units + self.data[locKey][qcKey] = icec_qc[i] + self.VarAttrs[locKey][qcKey]['_FillValue'] = qc_FillValue + self.VarAttrs[locKey][qcKey]['units'] = qc_units + ncd.close() + + +def main(): + + parser = argparse.ArgumentParser( + description=( + 'Read AMSR-2 sea ice concentration file(s) and convert' + ' to a concatenated IODA formatted output file.') + ) + required = parser.add_argument_group(title='required arguments') + required.add_argument( + '-i', '--input', + help="name of icec input file(s)", + type=str, nargs='+', required=True) + required.add_argument( + '-o', '--output', + help="name of ioda output file", + type=str, required=True) + required.add_argument( + '-d', '--date', + help="base date for the center of the window", + metavar="YYYYMMDDHH", type=str, required=True) + args = parser.parse_args() + fdate = datetime.strptime(args.date, '%Y%m%d%H') +# + VarDims = { + 'sea_ice_area_fraction': ['nlocs'], + } + + # Read in the Ice concentration + icec = iceconc(args.input, fdate) + + # write them out + ObsVars, nlocs = iconv.ExtractObsData(icec.data, locationKeyList) + + DimDict = {'nlocs': nlocs} + writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) + + writer.BuildIoda(ObsVars, VarDims, icec.VarAttrs, GlobalAttrs) + + +if __name__ == '__main__': + main() diff --git a/src/marine/nsidc_l4cdr_ice2ioda.py b/src/marine/nsidc_l4cdr_ice2ioda.py new file mode 100755 index 000000000..82ac05c39 --- /dev/null +++ b/src/marine/nsidc_l4cdr_ice2ioda.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 + +# +# (C) Copyright 2019 UCAR +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# + +from __future__ import print_function +import sys +import argparse +import netCDF4 as nc +from datetime import datetime, timedelta +import dateutil.parser +import numpy as np +from pathlib import Path + +IODA_CONV_PATH = Path(__file__).parent/"../lib/pyiodaconv" +if not IODA_CONV_PATH.is_dir(): + IODA_CONV_PATH = Path(__file__).parent/'..'/'lib-python' +sys.path.append(str(IODA_CONV_PATH.resolve())) + +import ioda_conv_engines as iconv +from orddicts import DefaultOrderedDict + + +class Observation(object): + + def __init__(self, filename, thin, date): + self.filename = filename + self.thin = thin + self.date = date + self.data = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) + self.VarAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) + self._read() + + def _read(self): + print(" Reading file: ", self.filename) + ncd = nc.Dataset(self.filename) + datein = ncd.variables['time'][:] + 0.5 + reftime = dateutil.parser.parse(ncd.variables['time'].units[-20:]) + obs_date = reftime + timedelta(days=float(datein)) + + data_in = {} + input_vars = ( + 'stdev_of_cdr_seaice_conc', + 'cdr_seaice_conc', + 'longitude', + 'latitude') + + # apply mask to constrain the ice concentration to within 1.0 (100% coverage) + data_in['cdr_seaice_conc'] = ncd.variables['cdr_seaice_conc'][:].ravel() + mask = data_in['cdr_seaice_conc'] <= 1.0 + for v in input_vars: + data_in[v] = ncd.variables[v][:].ravel()[mask] + mask = mask[mask] + vals_FillValue = ncd.variables['cdr_seaice_conc']._FillValue + vals_units = ncd.variables['cdr_seaice_conc'].units + errs_FillValue = ncd.variables['cdr_seaice_conc']._FillValue + errs_units = ncd.variables['cdr_seaice_conc'].units + # also, sometimes certain input variables have their own mask due to + # missing values + for v in input_vars: + if np.ma.is_masked(data_in[v]): + mask = np.logical_and(mask, np.logical_not(data_in[v].mask)) + + for v in input_vars: + data_in[v] = data_in[v][mask] + + ncd.close() + + vals = data_in['cdr_seaice_conc'] + errs = data_in['stdev_of_cdr_seaice_conc'] + lons = data_in['longitude'] + lats = data_in['latitude'] + + valKey = vName, iconv.OvalName() + errKey = vName, iconv.OerrName() + qcKey = vName, iconv.OqcName() + + # apply thinning mask + if self.thin > 0.0: + mask_thin = np.random.uniform(size=len(lons)) > self.thin + datein = datein[mask_thin] + timein = timein[mask_thin] + lons = lons[mask_thin] + lats = lats[mask_thin] + vals = vals[mask_thin] + qc = qc[mask_thin] + + for i in range(len(lons)): + locKey = lats[i], lons[i], obs_date.strftime("%Y-%m-%dT%H:%M:%SZ") + self.data[locKey][valKey] = vals[i] + self.VarAttrs[locKey][valKey]['_FillValue'] = vals_FillValue + self.VarAttrs[locKey][valKey]['units'] = vals_units + self.data[locKey][errKey] = errs[i] + if errs[i] == 0: + self.data[locKey][errKey] = 0.0001 + self.VarAttrs[locKey][errKey]['_FillValue'] = errs_FillValue + self.VarAttrs[locKey][errKey]['units'] = errs_units + self.data[locKey][qcKey] = 1 + self.VarAttrs[locKey][qcKey]['units'] = 'unitless' + + +vName = "sea_ice_area_fraction" + + +locationKeyList = [ + ("latitude", "float"), + ("longitude", "float"), + ("datetime", "string") +] + +GlobalAttrs = { + 'odb_version': 1, +} + + +def main(): + + parser = argparse.ArgumentParser( + description=('Read sea-ice concentration from NSIDC L4 CDR files' + ' and convert to IODA format') + ) + + required = parser.add_argument_group(title='required arguments') + required.add_argument( + '-i', '--input', + help="EMC ice fraction obs input file(s)", + type=str, required=True) + required.add_argument( + '-o', '--output', + help="name of ioda output file", + type=str, required=True) + required.add_argument( + '-d', '--date', + help="base date for the center of the window", + metavar="YYYYMMDDHH", type=str, required=True) + + optional = parser.add_argument_group(title='optional arguments') + optional.add_argument( + '-t', '--thin', + help="percentage of random thinning, from 0.0 to 1.0. Zero indicates" + " no thinning is performed. (default: %(default)s)", + type=float, default=0.0) + + args = parser.parse_args() + fdate = datetime.strptime(args.date, '%Y%m%d%H') + VarDims = { + vName: ['nlocs'], + } + # Read in + ice = Observation(args.input, args.thin, fdate) + + # write them out + ObsVars, nlocs = iconv.ExtractObsData(ice.data, locationKeyList) + DimDict = {'nlocs': nlocs} + writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) + + writer.BuildIoda(ObsVars, VarDims, ice.VarAttrs, GlobalAttrs) + + +if __name__ == '__main__': + main() diff --git a/src/marine/ostia_l4sst2ioda.py b/src/marine/ostia_l4sst2ioda.py index 836b75856..8d826c250 100755 --- a/src/marine/ostia_l4sst2ioda.py +++ b/src/marine/ostia_l4sst2ioda.py @@ -27,7 +27,7 @@ locationKeyList = [ ("latitude", "float"), ("longitude", "float"), - ("datetime", "string"), + ("dateTime", "long"), ] obsvars = { @@ -43,7 +43,7 @@ } VarDims = { - 'sst': ['nlocs'], + 'sst': ['Location'], } @@ -57,19 +57,16 @@ def __init__(self, filename): self.lons = self.lons.ravel() self.lats = self.lats.ravel() self.sst = np.squeeze(ncd.variables['analysed_sst'][:]).ravel() - self.sst = self.sst-273.15 self.err = np.squeeze(ncd.variables['analysis_error'][:]).ravel() - self.time = ncd.variables['time'][:] + this_datetime = ncd.variables['time'][:].astype(np.int) ncd.close() # Same time stamp for all obs within 1 file - self.datetime = np.empty_like(self.sst, dtype=object) - base_date = datetime(1981, 1, 1) - dt = base_date + timedelta(days=float(self.time/86400.0)) - self.datetime[:] = dt.strftime("%Y-%m-%dT%H:%M:%SZ") + self.datetime = np.full(len(self.lats), this_datetime) + self.time_units = 'seconds since 1981-01-01T00:00:00Z' # Remove observations out of sane bounds - qci = np.where(np.abs(self.sst) < 99.0) + qci = np.where(np.abs(self.sst) < 355.0) self.nlocs = len(qci[0]) self.lons = self.lons[qci].astype(np.single) self.lats = self.lats[qci].astype(np.single) @@ -82,7 +79,6 @@ class ostia_l4sst2ioda(object): def __init__(self, filename): self.filename = filename self.varDict = defaultdict(lambda: defaultdict(dict)) - self.metaDict = defaultdict(lambda: defaultdict(dict)) self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) self.var_mdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) self._read() @@ -90,26 +86,29 @@ def __init__(self, filename): # Open input file and read relevant info def _read(self): # set up variable names for IODA - iodavar = 'sea_surface_temperature' + iodavar = 'seaSurfaceTemperature' self.varDict[iodavar]['valKey'] = iodavar, iconv.OvalName() self.varDict[iodavar]['errKey'] = iodavar, iconv.OerrName() self.varDict[iodavar]['qcKey'] = iodavar, iconv.OqcName() - self.var_mdata[iodavar, iconv.OvalName()]['units'] = 'c' - self.var_mdata[iodavar, iconv.OerrName()]['units'] = 'c' + self.var_mdata[iodavar, iconv.OvalName()]['units'] = 'K' + self.var_mdata[iodavar, iconv.OerrName()]['units'] = 'K' # read input filename sst = ostia(self.filename) # map ostia to ioda data structure - self.outdata[('datetime', 'MetaData')] = sst.datetime + self.outdata[('dateTime', 'MetaData')] = sst.datetime + self.var_mdata[('dateTime', 'MetaData')]['units'] = sst.time_units self.outdata[('latitude', 'MetaData')] = sst.lats + self.var_mdata[('latitude', 'MetaData')]['units'] = 'degrees_north' self.outdata[('longitude', 'MetaData')] = sst.lons + self.var_mdata[('longitude', 'MetaData')]['units'] = 'degrees_east' self.outdata[self.varDict[iodavar]['valKey']] = sst.sst self.outdata[self.varDict[iodavar]['errKey']] = sst.err self.outdata[self.varDict[iodavar]['qcKey']] = np.zeros(sst.nlocs, dtype=np.int32) - DimDict['nlocs'] = sst.nlocs - AttrData['nlocs'] = np.int32(DimDict['nlocs']) + DimDict['Location'] = sst.nlocs + AttrData['Location'] = np.int32(DimDict['Location']) def main(): diff --git a/src/marine/pace_radiance2ioda.py b/src/marine/pace_radiance2ioda.py new file mode 100755 index 000000000..0d81cb2f1 --- /dev/null +++ b/src/marine/pace_radiance2ioda.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 + +# +# (C) Copyright 2022 UCAR +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# + +import sys +import argparse +import netCDF4 as nc +from datetime import datetime, timedelta +import dateutil.parser +import numpy as np +from multiprocessing import Pool +import os +from pathlib import Path + +IODA_CONV_PATH = Path(__file__).parent/"../lib/pyiodaconv" +if not IODA_CONV_PATH.is_dir(): + IODA_CONV_PATH = Path(__file__).parent/'..'/'lib-python' +sys.path.append(str(IODA_CONV_PATH.resolve())) + +import ioda_conv_engines as iconv +from orddicts import DefaultOrderedDict + +output_var_names = ["radiance"] + +locationKeyList = [ + ("latitude", "float"), + ("longitude", "float"), + ("datetime", "string") +] + +GlobalAttrs = { + 'odb_version': 1, +} + +VarAttrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) + +DimDict = {} + +VarDims = {} + +chan_number = range(1, 250) # we havew 120 Blue band 120 Red band and 9 SWRI + + +def read_input(input_args): + """ + Reads/converts a single input file, performing optional thinning also. + + Arguments: + + input_args: A tuple of input filename and global_config + input_file: The name of file to read + global_config: structure for global arguments related to read + + Returns: + + A tuple of (obs_data, GlobalAttrs) needed by the IODA writer. + """ + data_in = {} + input_file = input_args[0] + global_config = input_args[1] + ncd = nc.Dataset(input_file, 'r') + # get the base time (should only have 1 or 2 time slots) + time_base = ncd.groups['scan_line_attributes'].variables['time'][:] + basetime = dateutil.parser.parse(ncd.groups['scan_line_attributes'].variables['time'].units[-20:]) + + # Determine the lat/lon grid. + lons = ncd.groups['geolocation_data'].variables['longitude'][:].ravel() + lats = ncd.groups['geolocation_data'].variables['latitude'][:].ravel() + pixels = ncd.groups['geolocation_data'].variables['longitude'][:].shape[1] + + # calculate the time + time = np.tile((time_base), (pixels)) + + # load in all the other data + geo_vars = ( + 'solar_zenith', + 'sensor_zenith', + 'sensor_azimuth', + 'solar_azimuth') + + band_vars = ( + 'blue_wavelength', + 'red_wavelength', + 'SWIR_wavelength') + + obs_vars = ( + 'Lt_blue', + 'Lt_red', + 'Lt_SWIR') + + input_vars = geo_vars + band_vars + obs_vars + + for v in geo_vars: + if v not in data_in: + data_in[v] = ncd.groups['geolocation_data'].variables[v][:].ravel() + + for v in band_vars: + if v not in data_in: + data_in[v] = ncd.groups['sensor_band_parameters'].variables[v][:].ravel() + + for v in obs_vars: + if v not in data_in: + data_in[v] = ncd.groups['observation_data'].variables[v][:] + + ncd.close() + + # Create a mask for optional random thinning + mask = np.random.uniform(size=len(lons)) > global_config['thin'] + + # also, sometimes certain input variables have their own mask due to + # missing values + for v in geo_vars: + if np.ma.is_masked(data_in[v]): + mask = np.logical_and(mask, np.logical_not(data_in[v].mask)) + + time = time[mask] + lons = lons[mask] + lats = lats[mask] + + # create a string version of the date for each observation + dates = [] + for i in range(len(lons)): + obs_date = basetime + timedelta(seconds=float(time[i])) + dates.append(obs_date.strftime("%Y-%m-%dT%H:%M:%SZ")) + + # output values + nchans = len(chan_number) + obs_dim = (len(lons)) + val_radiance = np.concatenate((data_in['Lt_blue'], data_in['Lt_red'], data_in['Lt_SWIR']), axis=0) + val_radiance = np.reshape(val_radiance, (val_radiance.shape[0], val_radiance.shape[1]*val_radiance.shape[2])) + val_radiance = val_radiance.T + + # as there is not any obs error in data we use the same obs error for all chans for now + err = np.zeros((obs_dim, nchans))+0.5 + + # the quality flaq is not developed for this data set yet, we need to change this part when + # they add the qC to the obs data + qc = np.zeros((obs_dim, nchans)) + + wavelength = np.concatenate((data_in['blue_wavelength'], data_in['red_wavelength'], data_in['SWIR_wavelength']), axis=0) + + # allocate space for output depending on which variables are to be saved + + obs_data = {} + obs_data[('datetime', 'MetaData')] = np.empty(len(dates), dtype=object) + obs_data[('datetime', 'MetaData')][:] = dates + obs_data[('latitude', 'MetaData')] = lats + obs_data[('longitude', 'MetaData')] = lons + obs_data[('time', 'MetaData')] = time.astype('float32') + obs_data[('height_above_mean_sea_level', 'MetaData')] = np.zeros((obs_dim), dtype=np.float32) + obs_data[('sensor_azimuth_angle', 'MetaData')] = data_in['sensor_azimuth'] + obs_data[('sensor_zenith_angle', 'MetaData')] = data_in['sensor_zenith'] + obs_data[('sensor_view_angle', 'MetaData')] = data_in['sensor_zenith'] + obs_data[('solar_zenith_angle', 'MetaData')] = data_in['solar_zenith'] + obs_data[('solar_azimuth_angle', 'MetaData')] = data_in['solar_azimuth'] + obs_data[('sensor_band_central_radiation_wavenumber', 'VarMetaData')] = wavelength.astype('float32') + obs_data[output_var_names[0], global_config['oval_name']] = val_radiance.astype('float32') + obs_data[output_var_names[0], global_config['oerr_name']] = err.astype('float32') + obs_data[output_var_names[0], global_config['opqc_name']] = qc.astype('int32') + + return (obs_data, GlobalAttrs) + + +def main(): + + # Get command line arguments + parser = argparse.ArgumentParser( + description=( + 'Reads the radiance from any PACE L1B Data ' + ' and converts into IODA formatted output files.' + ' Multiple files are concatenated ') + ) + required = parser.add_argument_group(title='required arguments') + required.add_argument( + '-i', '--input', + help="Path to a pace L1B observation input file", + type=str, nargs=1, required=True) + required.add_argument( + '-o', '--output', + help="path of IODA output file", + type=str, required=True) + required.add_argument( + '-d', '--date', + metavar="YYYYMMDDHH", + help="base date for the center of the window", + type=str, required=True) + + optional = parser.add_argument_group(title='optional arguments') + optional.add_argument( + '-t', '--thin', + help="percentage of random thinning, from 0.0 to 1.0. Zero indicates" + " no thinning is performed. (default: %(default)s)", + type=float, default=0.0) + + args = parser.parse_args() + args.date = datetime.strptime(args.date, '%Y%m%d%H') + + # Setup the configuration that is passed to each worker process + # Note: Pool.map creates separate processes, and can only take iterable + # objects. Rather than using global variables, embed them into + # the iterable object together with argument array passed in (args.input) + global_config = {} + global_config['date'] = args.date + global_config['thin'] = args.thin + global_config['oval_name'] = iconv.OvalName() + global_config['oerr_name'] = iconv.OerrName() + global_config['opqc_name'] = iconv.OqcName() + + # concatenate the data from the files + obs_data = read_input((args.input[0], global_config))[0] + + for i in range(1, len(args.input)): + for k in obs_data: + obs_data[k] = np.concatenate( + (obs_data[k], args.input[i]), axis=0) + + # prepare global attributes we want to output in the file, + # in addition to the ones already loaded in from the input file + GlobalAttrs['date_time_string'] = args.date.strftime("%Y-%m-%dT%H:%M:%SZ") + GlobalAttrs['thinning'] = args.thin + GlobalAttrs['converter'] = os.path.basename(__file__) + + # determine which variables we are going to output + selected_names = [] + selected_names.append(output_var_names[0]) + + # pass parameters to the IODA writer + # (needed because we are bypassing ExtractObsData within BuildNetcdf) + VarDims = { + 'radiance': ['nlocs', 'nchans'], + 'sensor_band_central_radiation_wavenumber': ['nchans'] + } + + nchans = len(chan_number) + nlocs = len(obs_data[('longitude', 'MetaData')]) + ndatetime = np.zeros((20), dtype=np.float32) + DimDict = { + 'nlocs': nlocs, + 'nchans': list(chan_number), + 'nvars': list(chan_number), + 'ndatetime': list(ndatetime) + } + + writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) + + VarAttrs[('radiance', 'ObsValue')]['units'] = 'W m^-2 um^-1 sr^-1' + VarAttrs[('radiance', 'ObsError')]['units'] = 'W m^-2 um^-1 sr^-1' + VarAttrs[('radiance', 'PreQC')]['units'] = 'unitless' + VarAttrs[('radiance', 'ObsValue')]['_FillValue'] = -32767 + VarAttrs[('radiance', 'ObsError')]['_FillValue'] = 999 + VarAttrs[('radiance', 'PreQC')]['_FillValue'] = 999 + + writer.BuildIoda(obs_data, VarDims, VarAttrs, GlobalAttrs) + + +if __name__ == '__main__': + main() diff --git a/src/marine/swot_l2adt2ioda.py b/src/marine/swot_l2adt2ioda.py new file mode 100755 index 000000000..5f0927742 --- /dev/null +++ b/src/marine/swot_l2adt2ioda.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +# +# (C) Copyright 2022 UCAR +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# + +import sys +import argparse +import netCDF4 as nc +import numpy as np +from datetime import datetime, timedelta +import pytz +import os +from pathlib import Path + +IODA_CONV_PATH = Path(__file__).parent/"../lib/pyiodaconv" +if not IODA_CONV_PATH.is_dir(): + IODA_CONV_PATH = Path(__file__).parent/'..'/'lib-python' +sys.path.append(str(IODA_CONV_PATH.resolve())) + +import ioda_conv_engines as iconv +from collections import defaultdict, OrderedDict +from orddicts import DefaultOrderedDict + +locationKeyList = [ + ("latitude", "float"), + ("longitude", "float"), + ("dateTime", "long"), +] + +obsvars = { + 'adt': 'adt', +} + +AttrData = { + 'converter': os.path.basename(__file__), + 'nvars': np.int32(len(obsvars)), +} + +DimDict = { +} + +VarDims = { + 'adt': ['Location'], +} + + +class swot_l2adt2ioda(object): + def __init__(self, filename): + self.filename = filename + self.varDict = defaultdict(lambda: defaultdict(dict)) + self.outdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) + self.var_mdata = defaultdict(lambda: DefaultOrderedDict(OrderedDict)) + self._read() + + def _read(self): + # Open the input file and read the relevant data + ncd = nc.Dataset(self.filename, 'r') + self.lons = ncd.variables['longitude'][:].ravel() + self.lats = ncd.variables['latitude'][:].ravel() + self.geoid = ncd.variables['geoid'][:].ravel() + self.ssha = ncd.variables['ssha_karin'][:].ravel() + Fillvalue = ncd.variables['ssha_karin']._FillValue + units = ncd.variables['ssha_karin'].units + scale_factor = ncd.variables['ssha_karin'].scale_factor + self.mssh = ncd.variables['mean_sea_surface_cnescls'][:].ravel() + self.err = ncd.variables['ssh_karin_uncert'][:].ravel() + err_Fillvalue = ncd.variables['ssh_karin']._FillValue + err_units = ncd.variables['ssh_karin'].units + err_scale_factor = ncd.variables['ssh_karin'].scale_factor + self.qcflag = ncd.variables['ssha_karin_qual'][:].ravel() + # get the time data, chop off milliseconds, set the units + time_var = ncd.variables['time'] + time_units = time_var.units[:-2] + "Z" + s = list(time_units) + s[24] = "T" + time_units = "".join(s) + num_pixels = ncd.dimensions['num_pixels'].size + self.time = np.zeros(len(time_var)*num_pixels, dtype=np.int64) + for t in range(len(time_var)): + for n in range(num_pixels): + self.time[n + t*num_pixels] = np.round(time_var[t]) + + ncd.close() + + # estimate adt from SSH and Geoid height + adt = np.where(self.ssha == Fillvalue, Fillvalue, self.ssha + self.mssh - self.geoid) + + # set up variable names for IODA + iodavar = 'absoluteDynamicTopography' + self.varDict[iodavar]['valKey'] = iodavar, iconv.OvalName() + self.varDict[iodavar]['errKey'] = iodavar, iconv.OerrName() + self.varDict[iodavar]['qcKey'] = iodavar, iconv.OqcName() + self.var_mdata[iodavar, iconv.OvalName()]['units'] = units + self.var_mdata[iodavar, iconv.OerrName()]['units'] = err_units + self.var_mdata[iodavar, iconv.OvalName()]['_FillValue'] = Fillvalue + self.var_mdata[iodavar, iconv.OerrName()]['_FillValue'] = err_Fillvalue + self.var_mdata[iodavar, iconv.OvalName()]['scale_factor'] = scale_factor + self.var_mdata[iodavar, iconv.OerrName()]['scale_factor'] = err_scale_factor + + # map swot adt to ioda data structure + self.outdata[('dateTime', 'MetaData')] = self.time + self.var_mdata[('dateTime', 'MetaData')]['units'] = time_units + self.outdata[('latitude', 'MetaData')] = self.lats.astype('float32') + self.var_mdata[('latitude', 'MetaData')]['units'] = "degrees_north" + self.outdata[('longitude', 'MetaData')] = self.lons.astype('float32') + self.var_mdata[('longitude', 'MetaData')]['units'] = "degrees_east" + self.outdata[self.varDict[iodavar]['valKey']] = adt + # The current uncertainity values seem to be wrong so setting error to 1 + self.outdata[self.varDict[iodavar]['errKey']] = np.ones(np.shape(self.err)) + self.outdata[self.varDict[iodavar]['qcKey']] = self.qcflag.astype('int32') + + DimDict['Location'] = len(adt) + AttrData['Location'] = np.int32(DimDict['Location']) + + +def main(): + + # get command line arguments + parser = argparse.ArgumentParser( + description=( + 'Reads SWOT L2 SSH data, estimates the ADT ' + 'and converts into IODA formatted output files.') + ) + + required = parser.add_argument_group(title='required arguments') + required.add_argument( + '-i', '--input', + help="path of SWOT L2 SSH observation netCDF input file", + type=str, required=True) + required.add_argument( + '-o', '--output', + help="path of IODA output file", + type=str, required=True) + + args = parser.parse_args() + + # Read in the ADT data + adt = swot_l2adt2ioda(args.input) + + # setup the IODA writer + writer = iconv.IodaWriter(args.output, locationKeyList, DimDict) + + # write everything out + writer.BuildIoda(adt.outdata, VarDims, adt.var_mdata, AttrData) + + +if __name__ == '__main__': + main() diff --git a/src/ncar-bufr2nc-fortran/CMakeLists.txt b/src/ncar-bufr2nc-fortran/CMakeLists.txt index 280e53086..f55a4e114 100644 --- a/src/ncar-bufr2nc-fortran/CMakeLists.txt +++ b/src/ncar-bufr2nc-fortran/CMakeLists.txt @@ -18,5 +18,5 @@ list(APPEND bufr2nc_source ecbuild_add_executable( TARGET bufr2nc_fortran.x SOURCES ${bufr2nc_source} - LIBS bufr::bufr_4 + LIBS bufr::bufr_d NetCDF::NetCDF_Fortran) diff --git a/src/obserror/obserror2ioda.f90 b/src/obserror/obserror2ioda.f90 index f63236f27..f4e0b2f6a 100644 --- a/src/obserror/obserror2ioda.f90 +++ b/src/obserror/obserror2ioda.f90 @@ -6,7 +6,7 @@ program obserror2ioda use netcdf implicit none -character(len=256) :: filename_in, filename_out +character(len=256) :: arg, filename_in, filename_out, varid_name integer :: lu, ioflag integer :: nch_active, nctot, iprec integer :: ii @@ -14,9 +14,22 @@ program obserror2ioda integer, allocatable, dimension(:) :: indxR !< (nch_active) real(4), allocatable, dimension(:,:) :: readR4 !< (nch_active, nch_active) -!> get filenames from command line arguments -call getarg(1,filename_in) -call getarg(2,filename_out) +!> set defaults +varid_name = 'obserror_correlations' + +!> get command line arguments +do ii = 1, iargc() + call getarg(ii,arg) + if (ii == 1) filename_in = arg + if (ii == 2) filename_out = arg + if (ii == 3) varid_name = arg +enddo + +!> sanity check for options +if (trim(varid_name) /= 'obserror_correlations' .and. trim(varid_name) /= 'obserror_covariances') then + print *, 'Unknown option ', trim(varid_name) + call abort() +end if !> read GSI obs error covariances open(lu,file=trim(filename_in),convert='little_endian',form='unformatted') @@ -33,17 +46,17 @@ program obserror2ioda call check( nf90_create(trim(filename_out), NF90_CLOBBER + NF90_NETCDF4,ncid_out)) call check( nf90_def_dim(ncid_out, 'channels', nch_active, nch_dimid) ) call check( nf90_def_var(ncid_out, 'channels', NF90_INT, nch_dimid, varid_nch) ) -call check( nf90_def_var(ncid_out, "obserror_correlations", NF90_FLOAT, (/nch_dimid, nch_dimid/), varid_corr) ) +call check( nf90_def_var(ncid_out, trim(varid_name), NF90_FLOAT, (/nch_dimid, nch_dimid/), varid_corr) ) call check( nf90_put_var(ncid_out, varid_nch, indxR) ) call check( nf90_put_var(ncid_out, varid_corr, readR4) ) call check( nf90_close(ncid_out) ) contains - subroutine check(status) + subroutine check(status) integer, intent ( in) :: status if(status /= nf90_noerr) then - print *, trim(nf90_strerror(status)) - stop "Stopped" + print *, trim(nf90_strerror(status)) + call abort() end if end subroutine check diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b0ed46a92..7d14bdd0e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -8,6 +8,8 @@ ################################################################################ list( APPEND test_input + testinput/nsidc_l4_icec.nc + testinput/amsr2_icec_l2p.nc testinput/gds2_sst_l2p.nc testinput/gds2_sst_l3u.nc testinput/giirs_fy4a-test.nc @@ -38,7 +40,8 @@ list( APPEND test_input testinput/viirs_aod.nc testinput/tropomi_no2.nc testinput/mopitt_co.he5 - testinput/modis_aod.nc + testinput/modis_aod.hdf + testinput/imssnow_24km.grib2 testinput/2020100106_metars_small.csv testinput/singleob.yaml testinput/ghcn_20200228.csv @@ -51,6 +54,7 @@ list( APPEND test_input testinput/aeronet_aod.dat testinput/dt_global_twosat_phy_l4_20190101_vDT2021.nc testinput/global_vavh_l3_rt_s3a_20210930T18.nc + testinput/SWOT_L2_SSHA.nc testinput/aeronet_cad.dat testinput/aeronet_tab.dat testinput/imsscf_20191215_c48.nc @@ -64,13 +68,17 @@ list( APPEND test_input testinput/SMAP_L2_SM_P_E_36365_D_20211122T013945_R18240_001.h5 testinput/satwinds_ssec2021080103.txt testinput/pace_oc_l2.nc + testinput/pace_radiance_L1B.nc testinput/mls_o3_l2.he5 testinput/omi_o3_l2.he5 testinput/ompsnm_o3_l2.he5 testinput/sst_ostia.nc + testinput/2021081612_RAOB_small.txt ) list( APPEND test_output + testoutput/nsidc_l4_icec.nc + testoutput/amsr2_icec_l2p.nc testoutput/gds2_sst_l2p.nc testoutput/gds2_sst_l3u.nc testoutput/giirs_fy4a_obs_2017030208.nc4 @@ -100,7 +108,8 @@ list( APPEND test_output testoutput/ioda.NC001007.2020031012.nc testoutput/viirs_aod.nc testoutput/viirs_bias.nc - testoutput/tropomi_no2.nc + testoutput/tropomi_no2_total.nc + testoutput/tropomi_no2_tropo.nc testoutput/mopitt_co.nc testoutput/modis_aod.nc testoutput/2020100106_metars_small.nc @@ -114,6 +123,7 @@ list( APPEND test_output testoutput/aeronet_aod.nc testoutput/ioda_dt_global_twosat_phy_l4_20190101_vDT2021.nc testoutput/ioda_global_vavh_l3_rt_s3a_20210930T18.nc + testoutput/SWOT_L2_ADT.nc testoutput/aeronet_aaod.nc testoutput/imsfv3_scf.nc testoutput/gnssro_cosmic2_2021080212.nc4 @@ -125,10 +135,12 @@ list( APPEND test_output testoutput/smap9km_ssm.nc testoutput/satwinds_ssec2021080103.nc testoutput/pace_oc_l2.nc + testoutput/pace_radiance_L1B.nc testoutput/mls_o3_l2.nc testoutput/omi_o3_l2.nc testoutput/ompsnm_o3_l2.nc testoutput/sst_ostia.nc + testoutput/2021081612_sonde_small.nc ) if( iodaconv_gnssro_ENABLED ) @@ -144,26 +156,24 @@ endif() if( iodaconv_bufr_ENABLED ) list( APPEND test_input testinput/bufr_tables + testinput/gdas.t12z.1bamua.tm00.bufr_d + testinput/gdas.t12z.esamua.tm00.bufr_d + testinput/gdas.t12z.1bmhs.tm00.bufr_d + testinput/gdas.t12z.esmhs.tm00.bufr_d + testinput/gdas.t12z.mtiasi.tm00.bufr_d testinput/gdas.t18z.1bmhs.tm00.bufr_d testinput/gdas.t00z.1bhrs4.tm00.bufr_d - testinput/gdas.t00z.airsev.tm00.bufr_d - testinput/gdas.t00z.1bamua.tm00.bufr_d - testinput/gdas.t00z.avcsam.tm00.bufr_d - testinput/gdas.t00z.atms.tm00.bufr_d - testinput/gdas.t06z.adpsfc.tm00.bufr_d - testinput/gdas.t00z.crisf4.tm00.bufr_d - testinput/gdas.t00z.mtiasi.tm00.bufr_d testinput/gdas.t00z.sevcsr.tm00.bufr_d - testinput/gdas.t00z.ssmisu.tm00.bufr_d + testinput/gdas.t18z.satwnd_avhrr.tm00.bufr_d testinput/bufr_read_2_dim_blocks.bufr testinput/bufr_read_wmo_radiosonde.bufr testinput/bufr_satwnd_new_format.bufr testinput/bufr_satwnd_old_format.bufr - testinput/bufr_airs.yaml - testinput/bufr_amsua.yaml - testinput/bufr_atms.yaml - testinput/bufr_avhrr.yaml - testinput/bufr_cris.yaml + testinput/bufr_simple_groupby.bufr + testinput/bufr_empty_fields.bufr_d + testinput/bufr_sfcshp.bufr + testinput/ADPUPA.prepbufr + testinput/bufr_mhs.yaml testinput/bufr_hrs.yaml testinput/bufr_iasi.yaml testinput/bufr_mhs.yaml @@ -172,12 +182,28 @@ if( iodaconv_bufr_ENABLED ) testinput/bufr_filtering.yaml testinput/bufr_splitting.yaml testinput/bufr_filter_split.yaml - testinput/bufr_adpsfc.yaml - testinput/bufr_snow_adpsfc.yaml + testinput/bufr_ncep_prepbufr_adpsfc.yaml + testinput/bufr_ncep_adpsfc.yaml + testinput/gdas.t12z.adpsfc.prepbufr + testinput/gdas.t06z.adpsfc.tm00.bufr_d + testinput/gdas.t12z.adpsfc.tm00.bufr_d + testinput/bufr_ncep_snow_adpsfc.yaml testinput/bufr_read_2_dim_blocks.yaml + testinput/bufr_ncep_1bamua.yaml + testinput/bufr_ncep_1bamua_n15.yaml + testinput/bufr_ncep_1bmhs.yaml + testinput/bufr_ncep_esamua.yaml + testinput/bufr_ncep_esmhs.yaml + testinput/bufr_ncep_satwind_avhrr.yaml testinput/bufr_read_wmo_radiosonde.yaml testinput/bufr_satwnd_old_format.yaml testinput/bufr_satwnd_new_format.yaml + testinput/bufr_ncep_sevcsr.yaml + testinput/bufr_ncep_mtiasi.yaml + testinput/bufr_simple_groupby.yaml + testinput/bufr_empty_fields.yaml + testinput/bufr_sfcshp.yaml + testinput/adpupa_prepbufr.yaml testinput/aircar_BUFR2ioda.yaml testinput/gdas.aircar.t00z.20210801.bufr testinput/airep_wmoBUFR2ioda.yaml @@ -202,6 +228,9 @@ if( iodaconv_bufr_ENABLED ) testinput/satwind_Insat_wmo.bufr testinput/vadwinds_wmoBUFR2ioda.yaml testinput/vadwinds_wmo_multi.bufr + testinput/gdas.t12z.aircft.tm00.bufr_d + testinput/bufr_specific_subsets_by_query.yaml + testinput/bufr_specifying_subsets.yaml ) list( APPEND test_output @@ -210,20 +239,25 @@ if( iodaconv_bufr_ENABLED ) testoutput/gdas.t18z.1bmhs.tm00.filtering.nc testoutput/gdas.t18z.1bmhs.tm00.15.seven.split.nc testoutput/gdas.t18z.1bmhs.tm00.15.7.filter_split.nc - testoutput/gdas.t00z.airsev.tm00.nc - testoutput/gdas.t00z.1bamsua.tm00.nc - testoutput/gdas.t00z.avcsam.tm00.nc - testoutput/gdas.t00z.atms.tm00.nc - testoutput/gdas.t06z.adpsfc.tm00.nc + testoutput/gdas.t12z.1bamua.noaa-15.tm00.nc + testoutput/gdas.t12z.1bamua.metop-c.tm00.nc + testoutput/gdas.t12z.esamua.noaa-18.tm00.nc + testoutput/gdas.t12z.1bmhs.metop-b.tm00.nc + testoutput/gdas.t12z.esmhs.noaa-19.tm00.nc + testoutput/gdas.t12z.adpsfc.prepbufr.nc + testoutput/gdas.t12z.adpsfc.tm00.nc testoutput/gdas.t06z.adpsfc_snow.tm00.nc - testoutput/gdas.t00z.crisf4.tm00.nc - testoutput/gdas.t00z.mtiasi.tm00.nc testoutput/gdas.t00z.sevcsr.tm00.nc - testoutput/gdas.t00z.ssmisu.tm00.nc testoutput/bufr_read_2_dim_blocks.nc testoutput/bufr_read_wmo_radiosonde.nc testoutput/NC005031.nc testoutput/NC005066.nc + testoutput/gdas.t18z.satwnd_avhrr.tm00.nc + testoutput/gdas.t12z.mtiasi.metop-c.tm00.nc + testoutput/bufr_simple_groupby.nc + testoutput/bufr_empty_fields.nc + testoutput/bufr_sfcshp.nc + testoutput/adpupa_prepbufr.nc testoutput/gdas.aircar.t00z.20210801.nc testoutput/airep_multi.nc testoutput/amdar_wmo_multi.nc @@ -240,6 +274,7 @@ if( iodaconv_bufr_ENABLED ) testoutput/satwind_Himawari.nc testoutput/satwind_Insat.nc testoutput/vadwinds_wmo_multi.nc + testoutput/bufr_specifying_subsets.nc ) endif() @@ -247,6 +282,7 @@ if (iodaconv_satbias_ENABLED ) list( APPEND test_input testinput/satbias_converter_amsua.yaml testinput/satbias_converter_cris.yaml + testinput/satbias_converter_gmi_gpm.yaml testinput/satbias_converter_ssmis.yaml testinput/satbias_crtm_in testinput/satbias_crtm_pc @@ -255,6 +291,7 @@ if (iodaconv_satbias_ENABLED ) list( APPEND test_output testoutput/satbias_amsua_n18.nc4 testoutput/satbias_cris_npp.nc4 + testoutput/satbias_gmi_gpm.nc4 testoutput/satbias_ssmis_f16.nc4 testoutput/viirs_bias.nc ) @@ -281,6 +318,11 @@ if( iodaconv_pbfortran_ENABLED ) testoutput/gnssro_obs_2018041500.nc4 testoutput/sondes_obs_2020093018.nc4 testoutput/mhs_metop-b_obs_2020101215.nc4 + testoutput/mhs_metop-b_obs_2020110112.nc4 + testoutput/mhs_n19_obs_2020110112.nc4 + testoutput/amsua_metop-b_obs_2020110112.nc4 + testoutput/amsua_n18_obs_2020110112.nc4 + testoutput/iasi_metop-b_obs_2022021912.nc4 ) endif() @@ -295,6 +337,16 @@ foreach(FILENAME ${test_input} ${test_output}) ${CMAKE_CURRENT_BINARY_DIR}/${FILENAME} ) endforeach(FILENAME) +# create the share directory and link files to test subdir +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/share) +file( GLOB_RECURSE SHARE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../share/* ) +foreach ( f ${SHARE_FILES} ) + get_filename_component(filename ${f} NAME) + execute_process( COMMAND ${CMAKE_COMMAND} -E create_symlink + ${CMAKE_CURRENT_SOURCE_DIR}/../share/${filename} + ${CMAKE_CURRENT_BINARY_DIR}/share/${filename} ) +endforeach() + #=============================================================================== # The following tests use odb compare or nccmp to check the difference between the output # file, and the reference copy of what the ouput file should look like. @@ -342,6 +394,30 @@ set(IODACONV_PYTHONPATH "$/../:$ENV{PYTHONPATH}") # Marine converters #=============================================================================== +ecbuild_add_test( TARGET test_${PROJECT_NAME}_nsidc_l4cdr_icec + TYPE SCRIPT + ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/nsidc_l4cdr_ice2ioda.py + -i testinput/nsidc_l4_icec.nc + -o testrun/nsidc_l4_icec.nc + -d 2019010112" + nsidc_l4_icec.nc ${IODA_CONV_COMP_TOL_ZERO}) + +ecbuild_add_test( TARGET test_${PROJECT_NAME}_amsr2_l2icec + TYPE SCRIPT + ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/amsr2_icec2ioda.py + -i testinput/amsr2_icec_l2p.nc + -o testrun/amsr2_icec_l2p.nc + -d 2021080112" + amsr2_icec_l2p.nc ${IODA_CONV_COMP_TOL_ZERO}) + ecbuild_add_test( TARGET test_${PROJECT_NAME}_gds2_sst_l2p TYPE SCRIPT ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" @@ -571,6 +647,17 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_copernicus_l3swh -o testrun/ioda_global_vavh_l3_rt_s3a_20210930T18.nc" ioda_global_vavh_l3_rt_s3a_20210930T18.nc ${IODA_CONV_COMP_TOL_ZERO}) +ecbuild_add_test( TARGET test_${PROJECT_NAME}_swot_l2adt + TYPE SCRIPT + ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/swot_l2adt2ioda.py + -i testinput/SWOT_L2_SSHA.nc + -o testrun/SWOT_L2_ADT.nc" + SWOT_L2_ADT.nc ${IODA_CONV_COMP_TOL_ZERO}) + ecbuild_add_test( TARGET test_${PROJECT_NAME}_sst_ostia TYPE SCRIPT ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" @@ -607,6 +694,18 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_pace_oc_l2 -t 0.5" pace_oc_l2.nc ${IODA_CONV_COMP_TOL_ZERO}) +ecbuild_add_test( TARGET test_${PROJECT_NAME}_pace_radiance_L1B + TYPE SCRIPT + ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/pace_radiance2ioda.py + -i testinput/pace_radiance_L1B.nc + -o testrun/pace_radiance_L1B.nc + -d 2019032112" + pace_radiance_L1B.nc ${IODA_CONV_COMP_TOL_ZERO}) + ecbuild_add_test( TARGET test_${PROJECT_NAME}_mls_o3_l2 TYPE SCRIPT ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" @@ -684,6 +783,23 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_metar -d 2020100106" 2020100106_metars_small.nc ${IODA_CONV_COMP_TOL_ZERO}) +#=============================================================================== +# Conventional data, radiosonde, text/alpha (legacy) converter +#=============================================================================== + +ecbuild_add_test( TARGET test_${PROJECT_NAME}_sondetac + TYPE SCRIPT + ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/sonde_tac2ioda.py + -i testinput/2021081612_RAOB_small.txt + -o testrun/2021081612_sonde_small.nc + -t share/raob_stations_small.json + -y 2021 -m 8 --netcdf" + 2021081612_sonde_small.nc ${IODA_CONV_COMP_TOL_ZERO}) + #=============================================================================== # MISC converters #=============================================================================== @@ -886,12 +1002,12 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_satbias_viirs_aod TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh - netcdf - "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/viirs_biaswriter.py - -o testrun/viirs_bias.nc" - viirs_bias.nc ${IODA_CONV_COMP_TOL_ZERO}) + netcdf + "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/viirs_biaswriter.py + -o testrun/viirs_bias.nc" + viirs_bias.nc ${IODA_CONV_COMP_TOL_ZERO}) -ecbuild_add_test( TARGET test_${PROJECT_NAME}_tropomi_no2 +ecbuild_add_test( TARGET test_${PROJECT_NAME}_tropomi_no2_total TYPE SCRIPT ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" COMMAND bash @@ -899,8 +1015,21 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_tropomi_no2 netcdf "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/tropomi_no2_nc2ioda.py -i testinput/tropomi_no2.nc - -o testrun/tropomi_no2.nc" - tropomi_no2.nc ${IODA_CONV_COMP_TOL_ZERO}) + -o testrun/tropomi_no2_total.nc + -c total" + tropomi_no2_total.nc ${IODA_CONV_COMP_TOL_ZERO}) + +ecbuild_add_test( TARGET test_${PROJECT_NAME}_tropomi_no2_tropo + TYPE SCRIPT + ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/tropomi_no2_nc2ioda.py + -i testinput/tropomi_no2.nc + -o testrun/tropomi_no2_tropo.nc + -c tropo" + tropomi_no2_tropo.nc ${IODA_CONV_COMP_TOL}) ecbuild_add_test( TARGET test_${PROJECT_NAME}_mopitt_co TYPE SCRIPT @@ -910,8 +1039,9 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_mopitt_co netcdf "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/mopitt_co_nc2ioda.py -i testinput/mopitt_co.he5 - -o testrun/mopitt_co.nc" - mopitt_co.nc ${IODA_CONV_COMP_TOL_ZERO}) + -o testrun/mopitt_co.nc + -r 2021092903 2021092921" + mopitt_co.nc ${IODA_CONV_COMP_TOL_ZERO}) ecbuild_add_test( TARGET test_${PROJECT_NAME}_modis_aod TYPE SCRIPT @@ -920,8 +1050,9 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_modis_aod ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/modis_aod2ioda.py - -i testinput/modis_aod.nc - -t 2016060100 + -i testinput/modis_aod.hdf + -t 2021080100 + -p Terra -o testrun/modis_aod.nc" modis_aod.nc ${IODA_CONV_COMP_TOL_ZERO}) @@ -934,7 +1065,7 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_aeronet_aod "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/aeronet_aod2ioda.py -i testinput/aeronet_aod.dat -o testrun/aeronet_aod.nc" - aeronet_aod.nc ${IODA_CONV_COMP_TOL_ZERO}) + aeronet_aod.nc ${IODA_CONV_COMP_TOL}) ecbuild_add_test( TARGET test_${PROJECT_NAME}_aeronet_aaod TYPE SCRIPT @@ -959,7 +1090,6 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_airnow -o testrun/airnow_2020081306.nc" airnow_2020081306.nc ${IODA_CONV_COMP_TOL_ZERO}) - #============================================================================== # Bufr Ingester tests #============================================================================== @@ -1088,178 +1218,316 @@ if(iodaconv_bufr_ENABLED) gdas.t18z.1bmhs.tm00.15.7.filter_split.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_adpsfc2ioda + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_1bamua2ioda TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_adpsfc.yaml" - gdas.t06z.adpsfc.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_1bamua.yaml" + gdas.t12z.1bamua.metop-c.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_snowadpsfc2ioda + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_1bamua2ioda_n15 TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_snow_adpsfc.yaml" - gdas.t06z.adpsfc_snow.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_1bamua_n15.yaml" + gdas.t12z.1bamua.noaa-15.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - - ecbuild_add_test( TARGET test_iodaconv_bufr_read_2_dim_blocks + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_esamua2ioda TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_read_2_dim_blocks.yaml" - bufr_read_2_dim_blocks.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_esamua.yaml" + gdas.t12z.esamua.noaa-18.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_read_wmo_radiosonde + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_1bmhs2ioda TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_read_wmo_radiosonde.yaml" - bufr_read_wmo_radiosonde.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_1bmhs.yaml" + gdas.t12z.1bmhs.metop-b.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_satwnd_old_format + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_esmhs2ioda TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_satwnd_old_format.yaml" - NC005066.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_esmhs.yaml" + gdas.t12z.esmhs.noaa-19.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_satwnd_new_format + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_adpsfc2ioda TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_satwnd_new_format.yaml" - NC005031.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_adpsfc.yaml" + gdas.t12z.adpsfc.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_aircar + ecbuild_add_test( TARGET test_iodaconv_prepbufr_ncep_adpsfc2ioda TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/aircar_BUFR2ioda.yaml" - gdas.aircar.t00z.20210801.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_prepbufr_adpsfc.yaml" + gdas.t12z.adpsfc.prepbufr.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_airep_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_snowadpsfc2ioda TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/airep_wmoBUFR2ioda.yaml" - airep_multi.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_snow_adpsfc.yaml" + gdas.t06z.adpsfc_snow.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_amdar_wmo_bufr + + ecbuild_add_test( TARGET test_iodaconv_bufr_read_2_dim_blocks TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/amdar_wmoBUFR2ioda.yaml" - amdar_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_read_2_dim_blocks.yaml" + bufr_read_2_dim_blocks.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_gnssro_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_read_wmo_radiosonde TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/gnssro_wmoBUFR2ioda.yaml" - gnssro_2020-306-2358C2E6.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_read_wmo_radiosonde.yaml" + bufr_read_wmo_radiosonde.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_generic_gnssro_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_satwnd_old_format TYPE SCRIPT - ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh - netcdf - "${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/src/gnssro/gnssro_bufr2ioda.py - -d 2021080212 - -i testinput/bfrPrf_C2E6.2021.214.12.00.G16_0001.0001_bufr - -o testrun/gnssro_cosmic2_2021080212.nc4" - gnssro_cosmic2_2021080212.nc4 ${IODA_CONV_COMP_TOL_ZERO}) - - ecbuild_add_test( TARGET test_iodaconv_bufr_buoy_wmo_bufr + netcdf + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_satwnd_old_format.yaml" + NC005066.nc ${IODA_CONV_COMP_TOL_ZERO} + DEPENDS bufr2ioda.x ) + + ecbuild_add_test( TARGET test_iodaconv_bufr_satwnd_new_format TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/buoy_wmoBUFR2ioda.yaml" - buoy_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_satwnd_new_format.yaml" + NC005031.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_rass_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_satwind_avhrr TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/rass_wmoBUFR2ioda.yaml" - rass_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_satwind_avhrr.yaml" + gdas.t18z.satwnd_avhrr.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_ship_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_mtiasi TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/ship_wmoBUFR2ioda.yaml" - ship_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_mtiasi.yaml" + gdas.t12z.mtiasi.metop-c.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_synop_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_simple_groupby TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/synop_wmoBUFR2ioda.yaml" - synop_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO}) + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_simple_groupby.yaml" + bufr_simple_groupby.nc ${IODA_CONV_COMP_TOL_ZERO} + DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_satwind_EUMet_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_empty_fields TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/satwind_EUMet_wmoBUFR2ioda.yaml" - satwind_EUMet.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_empty_fields.yaml" + bufr_empty_fields.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_satwind_Himawari_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_sfcshp TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/satwind_Himawari_wmoBUFR2ioda.yaml" - satwind_Himawari.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_sfcshp.yaml" + bufr_sfcshp.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_satwind_Insat_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_adpupa TYPE SCRIPT - COMMAND ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh - ARGS netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/satwind_Insat_wmoBUFR2ioda.yaml" - satwind_Insat.nc ${IODA_CONV_COMP_TOL_ZERO} + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/adpupa_prepbufr.yaml" + adpupa_prepbufr.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) - ecbuild_add_test( TARGET test_iodaconv_bufr_vadwinds_wmo_bufr + ecbuild_add_test( TARGET test_iodaconv_bufr_ncep_sevcsr TYPE SCRIPT COMMAND bash ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh netcdf - "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/vadwinds_wmoBUFR2ioda.yaml" - vadwinds_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_ncep_sevcsr.yaml" + gdas.t00z.sevcsr.tm00.nc ${IODA_CONV_COMP_TOL_ZERO} + DEPENDS bufr2ioda.x ) + + ecbuild_add_test( TARGET test_iodaconv_bufr_specific_subsets_by_query + TYPE SCRIPT + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_specific_subsets_by_query.yaml" + bufr_specifying_subsets.nc ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2ioda.x ) + + ecbuild_add_test( TARGET test_iodaconv_bufr_specifying_subsets + TYPE SCRIPT + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/bufr_specifying_subsets.yaml" + bufr_specifying_subsets.nc ${IODA_CONV_COMP_TOL_ZERO} + DEPENDS bufr2ioda.x ) + +# FIXME: Greg Thompson +# ecbuild_add_test( TARGET test_iodaconv_bufr_aircar +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/aircar_BUFR2ioda.yaml" +# gdas.aircar.t00z.20210801.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_airep_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/airep_wmoBUFR2ioda.yaml" +# airep_multi.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_amdar_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/amdar_wmoBUFR2ioda.yaml" +# amdar_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_gnssro_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/gnssro_wmoBUFR2ioda.yaml" +# gnssro_2020-306-2358C2E6.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) + + ecbuild_add_test( TARGET test_iodaconv_generic_gnssro_bufr + TYPE SCRIPT + ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/src/gnssro/gnssro_bufr2ioda.py + -d 2021080212 + -i testinput/bfrPrf_C2E6.2021.214.12.00.G16_0001.0001_bufr + -o testrun/gnssro_cosmic2_2021080212.nc4" + gnssro_cosmic2_2021080212.nc4 ${IODA_CONV_COMP_TOL_ZERO}) + +# FIXME: Greg Thompson +# ecbuild_add_test( TARGET test_iodaconv_bufr_buoy_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/buoy_wmoBUFR2ioda.yaml" +# buoy_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) + +# ecbuild_add_test( TARGET test_iodaconv_bufr_rass_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/rass_wmoBUFR2ioda.yaml" +# rass_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_ship_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/ship_wmoBUFR2ioda.yaml" +# ship_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_synop_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/synop_wmoBUFR2ioda.yaml" +# synop_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO}) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_satwind_EUMet_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/satwind_EUMet_wmoBUFR2ioda.yaml" +# satwind_EUMet.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_satwind_Himawari_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/satwind_Himawari_wmoBUFR2ioda.yaml" +# satwind_Himawari.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_satwind_Insat_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/satwind_Insat_wmoBUFR2ioda.yaml" +# satwind_Insat.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) +# +# ecbuild_add_test( TARGET test_iodaconv_bufr_vadwinds_wmo_bufr +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x testinput/vadwinds_wmoBUFR2ioda.yaml" +# vadwinds_wmo_multi.nc ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2ioda.x ) endif() @@ -1385,6 +1653,48 @@ if( iodaconv_pbfortran_ENABLED ) mhs_metop-b_obs_2020101215.nc4 ${IODA_CONV_COMP_TOL_ZERO} DEPENDS bufr2nc_fortran.x) +# FIXME: Emily Liu +# ecbuild_add_test( TARGET test_${PROJECT_NAME}_ears_mhs_conv +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2nc_fortran.x +# -i testinput -o testrun gdas.t12z.esmhs.tm00.bufr_d" +# mhs_n19_obs_2020110112.nc4 ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2nc_fortran.x) + + ecbuild_add_test( TARGET test_${PROJECT_NAME}_amsua_conv + TYPE SCRIPT + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${CMAKE_BINARY_DIR}/bin/bufr2nc_fortran.x + -i testinput -o testrun gdas.t12z.1bamua.tm00.bufr_d" + amsua_metop-b_obs_2020110112.nc4 ${IODA_CONV_COMP_TOL_ZERO} + DEPENDS bufr2nc_fortran.x) + +# FIXME: Emily Liu +# ecbuild_add_test( TARGET test_${PROJECT_NAME}_ears_amsua_conv +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2nc_fortran.x +# -i testinput -o testrun gdas.t12z.esamua.tm00.bufr_d" +# amsua_n18_obs_2020110112.nc4 ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2nc_fortran.x) + +# ecbuild_add_test( TARGET test_${PROJECT_NAME}_iasi_conv +# TYPE SCRIPT +# COMMAND bash +# ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh +# netcdf +# "${CMAKE_BINARY_DIR}/bin/bufr2nc_fortran.x +# -i testinput -o testrun gdas.t12z.mtiasi.tm00.bufr_d" +# iasi_metop-b_obs_2022021912.nc4 ${IODA_CONV_COMP_TOL_ZERO} +# DEPENDS bufr2nc_fortran.x) + endif() @@ -1412,6 +1722,15 @@ if( iodaconv_satbias_ENABLED ) satbias_cris_npp.nc4 ${IODA_CONV_COMP_TOL_ZERO} DEPENDS satbias2ioda.x ) + ecbuild_add_test( TARGET test_iodaconv_satbias_gmi + TYPE SCRIPT + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf + "${CMAKE_BINARY_DIR}/bin/satbias2ioda.x testinput/satbias_converter_gmi_gpm.yaml" + satbias_gmi_gpm.nc4 ${IODA_CONV_COMP_TOL_ZERO} + DEPENDS satbias2ioda.x ) + ecbuild_add_test( TARGET test_iodaconv_satbias_ssmis TYPE SCRIPT COMMAND bash @@ -1467,7 +1786,7 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_smap_ssm "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/smap_ssm2ioda.py -i testinput/SMAP_L2_SM_P_NRT_24342_A_20190822T224858_N16023_002.h5 -o testrun/smap_ssm.nc - -m maskout" + --maskMissing" smap_ssm.nc ${IODA_CONV_COMP_TOL_ZERO}) ecbuild_add_test( TARGET test_${PROJECT_NAME}_smos_ssm @@ -1514,14 +1833,15 @@ ecbuild_add_test( TARGET test_${PROJECT_NAME}_smap9km_ssm "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/smap9km_ssm2ioda.py -i testinput/SMAP_L2_SM_P_E_36365_D_20211122T013945_R18240_001.h5 -o testrun/smap9km_ssm.nc - -m maskout" + --maskMissing" smap9km_ssm.nc ${IODA_CONV_COMP_TOL_ZERO}) ecbuild_add_test( TARGET test_${PROJECT_NAME}_land_owp_snow_obs_dup_thin TYPE SCRIPT ENVIRONMENT "PYTHONPATH=${IODACONV_PYTHONPATH}" - COMMAND ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh - ARGS netcdf + COMMAND bash + ARGS ${CMAKE_BINARY_DIR}/bin/iodaconv_comp.sh + netcdf "${Python3_EXECUTABLE} ${CMAKE_BINARY_DIR}/bin/owp_snow_obs.py -i testinput/owp_snow_obs.csv -o testrun/owp_snow_obs_dup_thin_err_fn.nc diff --git a/test/bufr/TestBufrDescription.h b/test/bufr/TestBufrDescription.h index 1dfa2b022..bde696944 100644 --- a/test/bufr/TestBufrDescription.h +++ b/test/bufr/TestBufrDescription.h @@ -24,7 +24,6 @@ #include "oops/util/IntSetParser.h" #include "BufrParser/BufrDescription.h" -#include "BufrParser/BufrMnemonicSet.h" #include "IodaEncoder/IodaDescription.h" @@ -47,7 +46,7 @@ namespace Ingester auto bufrConf = obsConf.getSubConfiguration("obs space"); auto description = Ingester::BufrDescription(bufrConf); - EXPECT(description.getMnemonicSets().size() > 0); + // EXPECT(description.getMnemonicSets().size() > 0); EXPECT(description.getExport().getVariables().size() > 0); } else diff --git a/test/bufr/TestBufrParser.h b/test/bufr/TestBufrParser.h index 43ce973ab..46f27eabb 100644 --- a/test/bufr/TestBufrParser.h +++ b/test/bufr/TestBufrParser.h @@ -22,9 +22,7 @@ #include "test/TestEnvironment.h" #include "BufrParser/BufrDescription.h" -#include "BufrParser/BufrMnemonicSet.h" #include "BufrParser/BufrParser.h" -#include "BufrParser/BufrTypes.h" #include "DataContainer.h" #include "ParserFactory.h" diff --git a/test/testinput/2021081612_RAOB_small.txt b/test/testinput/2021081612_RAOB_small.txt new file mode 100644 index 000000000..7e65a98cc --- /dev/null +++ b/test/testinput/2021081612_RAOB_small.txt @@ -0,0 +1,297 @@ + +956 +USUS50 KWBC 161200 RRA +TTAA 66121 72469 99840 15058 18504 00099 ///// ///// 92782 ///// +///// 85507 ///// ///// 70170 13464 32001 50589 07370 12009 40758 +21758 07511 30964 36948 01514 25087 46756 36016 20232 52380 32518 +15415 58777 34022 10667 63374 31516 88123 62175 35010 77999 31313 +51108 81102= + +051 +UKUS50 KWBC 161200 RRB +TTCC 72469 NIL= + +051 +UKUS50 KWBC 161200 RRD +TTBB 66128 72469 00840 15058 11831 18262 22826 21066 33804 22469 +44700 13464 55589 00256 66579 00266 77547 02374 88518 04772 99435 +16565 11378 24327 22344 28359 33308 35345 44285 39750 55271 42159 +66225 52550 77216 51370 88199 51981 99123 62175 11101 63374 31313 +51108 81102 41414 00900 51515 10164 00091 10194 35502 /////= + +071 +UEUS50 KWBC 161200 +TTDD 72469 NIL= + +097 +UHUS50 KWBC 161200 +PPCC 72469 NIL= + +103 +UGUS50 KWBC 161200 RRA +PPBB 66128 72469 90056 18504 18504 28003 90789 01003 04005 01504 +91012 34503 13502 10002 91345 07003 05506 05508 9169/ 08004 12009 +92034 11509 11506 07508 92567 07512 05010 05511 93056 01009 01020 +35514 938// 32513 94023 30520 33519 32530 9489/ 35016 01511 95012 +35011 36011 34005 9534/ 30015 30517= + +105 +UPUS50 KWBC 161200 +PPAA 72469 NIL= + +223 +UEUS90 KWBC 161200 +PPDD 72469 NIL= + +808 +UEUS50 KWBC 161200 RRF +TTDD 6612/ 72469 11928 64773 22712 62974 33628 59178 44572 60777 +55505 57180 66441 59578 77382 54582 88326 55581 99302 52184 11199 +48587 22135 47987 33099 42790 44094 39592 55080 40791 31313 51108 +81102= + +817 +ULUS50 KWBC 161200 RRC +TTCC 66121 72469 70886 63774 14006 50096 57380 08017 30421 52184 +07522 20684 48787 07520 10143 42790 09026 88999 77999 31313 51108 +81102= + +940 +UQUS50 KWBC 161200 RRE +PPDD 66128 72469 95567 32516 01011 01504 9589/ 03013 08012 96024 +09514 15006 02505 96578 06510 07508 06514 969// 08518 97012 06524 +11019 13006 97346 08510 09009 06011 977// 07519 98013 07522 09021 +07018 99026 08518 09523 07028 10012 10027 10525 08529 10345 09026 +07528 09039 108// 10028= + +985 +USUS41 KLWX 161200 +MANIAD +72403 TTAA 66121 72403 99011 22010 00000 00183 22019 16003 +92857 18412 19013 85580 14816 21008 70199 06220 24013 50589 +07726 24514 40759 17546 23018 30968 33356 22527 25093 43759 +25032 20239 56158 27534 15417 66562 27527 10662 62162 31013 +88142 68563 26521 77999 31313 51108 81106= + +206 +UMUS41 KLWX 161200 +SGLIAD +72403 TTBB 66128 72403 00011 22010 11006 22419 22932 18610 +33563 03513 44367 22532 55341 25558 66311 31548 77252 43359 +88215 51557 99185 60760 11142 68563 22137 68363 33134 68762 +44130 66563 55111 67562 66101 62562 31313 51108 81106 41414 +864// 51515 10164 00000 10194 18010 24011= + +PPBB 66128 72403 90012 00000 16012 19013 90345 19012 18511 +18011 90678 24008 25010 24513 909// 24014 91246 23014 22513 +22512 9205/ 24016 23018 9305/ 23023 24534 950// 24529= + +045 +ULUS50 KWBC 161200 +TTCC 72403 NIL= + +069 +UKUS50 KWBC 161200 RRG +TTDD 72403 NIL= + +104 +ULAK02 KWBC 161200 +PPAA 72403 NIL= +PPCC 72403 NIL= + +482 +UFUS41 KLWX 161200 +ABVIAD +72403 TTCC 66121 72403 70882 61964 32008 50092 58564 04012 +30416 53162 06024 20680 49764 10525 10140 43985 08534 07382 +37789 88999 77999 31313 51108 81106= + +TTDD 6612/ 72403 11664 60163 22329 55763 33266 50163 44234 +51963 55094 41387 66081 42186 77069 36990 31313 51108 81106= + +PPDD 66128 72403 960// 35502 970// 06514 990// 10016 100// +07528 110// 09529= + +314 +UKTU10 LTAA 161200 +TTBB ///// 17030 NIL= +TTBB 66128 17064 00012 30065 11000 27263 22879 16240 33871 +15444 44856 16465 55848 18279 66842 18080 77811 15671 88795 +14879 99787 15291 11782 15097 22727 11293 33721 11495 44705 +10489 55626 04496 66572 00089 77551 02370 88435 14791 99376 +22580 11228 43182 22141 60377 33138 59977 44120 63176 55113 +63576 66101 62775 77100 62975 21212 00012 02005 11980 02512 +22947 06013 33915 05008 44794 10007 55782 11003 66761 36001 +77751 17002 88723 18003 99706 31502 11616 31003 22589 34006 +33532 31508 44509 26010 55480 28512 66289 26523 77227 26560 +88185 26568 99143 26050 11139 25550 22110 25034 33100 26021 +31313 45408 81133 41414 00900 51515 11913 05008 22800 09508 +33600 34505= + +321 +UETU10 LTAA 161200 +TTDD ///// 17030 NIL= +TTDD 6612/ 17064 11871 65775 22084 36579 33078 37579 21212 +11773 35505 22760 13005 33735 21014 44673 11004 55078 09537 +31313 45408 81133= + +322 +USTU10 LTAA 161200 +TTAA ///// 17030 NIL= +TTAA 66121 17064 99012 30065 02005 00121 26262 01509 92802 +20257 04510 85525 17876 09008 70158 10291 32502 50586 08371 +26511 40756 18985 29515 30964 33980 27019 25090 40380 27045 +20240 48781 26566 15425 58178 26055 10676 62975 26021 88120 +63176 26040 77185 26568 40619 31313 45408 81133= + +329 +ULTU10 LTAA 161200 +TTCC ///// 17030 NIL= +TTCC 66121 17064 70893 62575 22502 50102 57377 14009 30430 +50977 10012 20696 47777 16012 10162 40378 09517 88999 77999 +31313 45408 81133= + +576 +UKBZ04 SBBR 161200 +TTBB 66128 83840 00915 13200 11910 12803 22895 15246 +33866 16256 44801 13058 55686 02018 66624 02500 77553 05509 +21212 00915 05006 11877 22511 22822 23024 33776 27024 44751 +25531 55680 25540 66615 27517 77553 29034 31313 42308 81148 +41414 77000= + +726 +UEBZ04 SBBR 161200 +TTDD ///// 83840 NIL= + +728 +USBZ04 SBBR 161200 +TTAA 66127 83840 99915 13200 05006 00140 ///// ///// +92819 ///// ///// 85536 15456 23518 70151 03431 25539 88999 +77999 31313 42308 81148= + +749 +ULBZ04 SBBR 161200 +TTCC ///// 83840 NIL= + +514 +UGBZ04 SBBR 161200 +PPBB 66128 83840 90/46 05006 22511 23024 90789 24528 +26028 25531 91147 25540 27517 29034= + +554 +UHBZ04 SBBR 161200 +PPCC ///// 83840 NIL= + +571 +UPBZ04 SBBR 161200 +PPAA 66128 83840 44285 23518 25539 77999= + +529 +UQBZ04 SBBR 161200 +PPDD ///// 83840 NIL= + +153 +UPZA01 FAPR 161200 +PPAA 66128 68263 +55385 34504 22510 26013 55340 29517 25030 24534 55320 26563 +27540 26035 +71219 26564 42527= + +730 +UGZA01 FAPR 161200 +PPBB 66128 68263 +90123 ///// ///// ///// 9078/ 02508 01007 91357 21023 21022 +22512 92137 26517 28519 23514 929// 24017= + +734 +USZA01 FAPR 161200 RRA +TTAA 66121 68263 +99856 20264 03008 00188 ///// ///// 92857 ///// ///// 85583 +16260 34504 70197 05059 22510 50585 10997 26013 40752 26167 +29517 30954 39576 25030 25077 46171 24534 20225 49779 26563 +15409 61569 27540 10653 70566 26035 +88112 71563 27538 +77202 26564 42527 +31313 48408 81143= + +735 +UEZA01 FAPR 161200 RRA +TTDD 6612/ 68263 +11945 71565 22833 68370 33794 70369 44582 64380 55547 59781 +66500 57982 77374 58581 88275 50384 99197 50184 +21212 11987 25535 22857 24022 33828 27014 44807 29016 55745 +25014 66716 28015 77699 28017 88646 20510 99619 26010 11607 +27010 22554 11501 33494 14510 44477 15010 55408 10515 66386 +05513 77324 07016 88296 10013 99271 10020 11260 07021 22248 +08522 33232 06524 44221 08524 55210 05520 66197 09521 +31313 48408 81143= + +741 +UKZA01 FAPR 161200 RRA +TTBB 66128 68263 +00856 20264 11856 18863 22855 17661 33856 16861 44852 16460 +55776 09045 66763 07847 77752 08860 88749 08660 99707 05057 +11668 03064 22667 03064 33597 05558 44593 05959 55582 05188 +66579 05191 77566 06398 88509 09997 99389 27966 11387 28366 +22375 28791 33369 29190 44286 42170 55243 46973 66220 46382 +77206 49379 88186 51177 99151 61569 11121 67965 22112 71563 +33102 69966 +21212 11856 03008 22737 28005 33705 24010 44688 21011 55660 +21025 66603 20525 77592 21019 88581 19511 99580 19511 11566 +22011 22534 23512 33520 26011 44489 25017 55449 29019 66412 +28019 77398 29517 88375 24514 99365 23514 11363 23014 22300 +25030 33265 25027 44248 24538 55232 26541 66202 26564 77174 +25041 88170 25042 99152 28040 11127 26525 22124 27022 33119 +29031 44113 27538 55108 26030 +31313 48408 81143 +41414 00900 +51515 77359 09046 01507 60433 05158 20524= + +446 +ULZA01 FAPR 161200 RRA +TTCC 66122 68263 +70866 66572 28017 50072 57982 13007 30396 53583 10013 20659 +51184 08519 +88999 +77999 +31313 48408 81143= + +447 +UHZA01 FAPR 161200 +PPCC 66128 68263 +55370 28017 13007 10013 55120 08519 +77999= + +886 +UQZA01 FAPR 161200 +PPDD 66128 68263= + +396 +UKRS10 RUMS 161200 +TTBB 16123 27713 00992 28065 11959 23264 22812 09444 33736 +03832 44711 00056 55701 03462 66683 02666 77673 02066 88620 +03961 99593 04569 11571 07563 22501 12773 33330 34763 44241 +54560 55208 59560 66160 51562 77136 53764 88100 50368 21212 +00992 24002 11978 21005 22912 23005 33868 22508 44768 27508 +55749 25511 66710 26515 77449 24017 88380 24518 99221 28028 +11195 24529 22182 25522 31313 56203 81130 41414 31630= + +445 +USRS02 RUMS 161200 +TTAA 16121 27713 99992 28065 24002 00123 ///// ///// 92804 20662 +22506 85527 13458 23507 70122 03462 26014 50576 12973 25015 40742 +24169 24518 30945 39962 25526 25066 51961 27030 20208 58760 25027 +15392 51763 24019 10654 50368 24510 88211 59560 26025 77999= + +155 +UERS02 RUMS 161200 +TTDD 1612/ 27713 11950 52967 22868 50969 33757 53570 44675 51170 +55448 53372 66309 51374 77265 46576 88225 49175 99196 44576 11145 +43376 22128 39377 21212 11786 26009 22692 25003 33419 13502 44369 +22003 55326 11502 66288 12002 77255 10503 88226 15502 99130 11504= + +739 +ULRS02 RUMS 161200 +TTCC 16122 27713 70885 51770 25004 50103 52372 17503 30434 50174 +12002 20702 46576 15002 88999 77999= + diff --git a/test/testinput/ADPUPA.prepbufr b/test/testinput/ADPUPA.prepbufr new file mode 100644 index 000000000..c95db4987 --- /dev/null +++ b/test/testinput/ADPUPA.prepbufr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa48d03dca71b8d7558802e8eadfe4967e4d22f35ce108c85b20369e962add30 +size 486008 diff --git a/test/testinput/SWOT_L2_SSHA.nc b/test/testinput/SWOT_L2_SSHA.nc new file mode 100644 index 000000000..71e0eff61 --- /dev/null +++ b/test/testinput/SWOT_L2_SSHA.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5da6df19ac85a0ddce727ea2dc682cf2c2b327ff3df7f10420a76f915aafe073 +size 341876 diff --git a/test/testinput/acft_profiles.data_thinned b/test/testinput/acft_profiles.data_thinned new file mode 100644 index 000000000..212cc030a Binary files /dev/null and b/test/testinput/acft_profiles.data_thinned differ diff --git a/test/testinput/adpupa_prepbufr.yaml b/test/testinput/adpupa_prepbufr.yaml new file mode 100644 index 000000000..17381a260 --- /dev/null +++ b/test/testinput/adpupa_prepbufr.yaml @@ -0,0 +1,365 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/ADPUPA.prepbufr" + + exports: + variables: + timestamp: + timeoffset: + timeOffset: "*/PRSLEVEL/DRFTINFO/HRDR" + transforms: + - scale: 3600 + referenceTime: "2020-11-01T12:00:00Z" + longitude: + query: "*/PRSLEVEL/DRFTINFO/XDR" + latitude: + query: "*/PRSLEVEL/DRFTINFO/YDR" + timeOffset: + query: "*/PRSLEVEL/DRFTINFO/HRDR" + transforms: + - scale: 3600 + stationIdentification: + query: "*/SID" + stationElevation: + query: "*/ELV" + type: float + + pressure: + query: "*/PRSLEVEL/P___INFO/P__EVENT/POB" + type: float + transforms: + - scale: 100.0 + height: + query: "*/PRSLEVEL/Z___INFO/Z__EVENT/ZOB" + type: float + airTemperature: + query: "*/PRSLEVEL/T___INFO/T__EVENT/TOB" + transforms: + - offset: 273.15 + dewpointTemperature: + query: "*/PRSLEVEL/Q___INFO/TDO" + transforms: + - offset: 273.15 + virtualTemperature: + query: "*/PRSLEVEL/T___INFO/TVO" + transforms: + - offset: 273.15 + specificHumidity: + query: "*/PRSLEVEL/Q___INFO/Q__EVENT/QOB" + type: float + transforms: + - scale: 0.000001 + windEastward: + query: "*/PRSLEVEL/W___INFO/W__EVENT/UOB" + windNorthward: + query: "*/PRSLEVEL/W___INFO/W__EVENT/VOB" + seaSurfaceTemperature: + query: "*/SST_INFO/SSTEVENT/SST1" + depthBelowSeaSurface: + query: "*/SST_INFO/DBSS_SEQ/DBSS" + +# Quality Markers + pressureQM: + query: "*/PRSLEVEL/P___INFO/P__EVENT/PQM" + heightQM: + query: "*/PRSLEVEL/Z___INFO/Z__EVENT/ZQM" + airTemperatureQM: + query: "*/PRSLEVEL/T___INFO/T__EVENT/TQM" + specificHumidityQM: + query: "*/PRSLEVEL/Q___INFO/Q__EVENT/QQM" + windQM: + query: "*/PRSLEVEL/W___INFO/W__EVENT/WQM" + seaSurfaceTemperatureQM: + query: "*/SST_INFO/SSTEVENT/SSTQM" + +# ObsErrors + pressureError: + query: "*/PRSLEVEL/P___INFO/P__BACKG/POE" + transforms: + - scale: 100 + airTemperatureError: + query: "*/PRSLEVEL/T___INFO/T__BACKG/TOE" + relativeHumidityError: + query: "*/PRSLEVEL/Q___INFO/Q__BACKG/QOE" + transforms: + - scale: 0.1 + windError: + query: "*/PRSLEVEL/W___INFO/W__BACKG/WOE" + seaSurfaceTemperatureError: + query: "*/SST_INFO/SSTBACKG/SSTOE" + +# Extra Info + verticalSignificance: + query: "*/CLOUDSEQ/VSSO" + prepbufrReportType: + query: "*/TYP" + dumpReportType: + query: "*/T29" + prepbufrDataLvlCat: + query: "*/PRSLEVEL/CAT" + waterTemperatureMethod: + query: "*/SST_INFO/MSST" + presentWeather: + query: "*/PREWXSEQ/PRWE" + cloudAmount: + query: "*/CLOUDSEQ/CLAM" + cloudType: + query: "*/CLOUDSEQ/CLTP" + cloudCoverTotal: + query: "*/CLOU2SEQ/TOCC" + type: float + transforms: + - scale: 0.01 + heightOfBaseOfCloud: + query: "*/CLOUDSEQ/HOCB" + type: float + + ioda: + backend: netcdf + obsdataout: "./testrun/adpupa_prepbufr.nc" + + dimensions: + - name: Level + path: "*/PRSLEVEL" + - name: Depth + path: "*/SST_INFO/SSTEVENT" + - name: cloudseq_Dim + path: "*/CLOUDSEQ" + - name: pevent_Dim + path: "*/PRSLEVEL/P___INFO/P__EVENT" + - name: qevent_Dim + path: "*/PRSLEVEL/Q___INFO/Q__EVENT" + - name: tevent_Dim + path: "*/PRSLEVEL/T___INFO/T__EVENT" + - name: zevent_Dim + path: "*/PRSLEVEL/Z___INFO/Z__EVENT" + - name: wevent_Dim + path: "*/PRSLEVEL/W___INFO/W__EVENT" + + variables: + + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "dateTime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [0, 360] + + - name: "MetaData/timeOffset" + coordinates: "longitude latitude Level" + source: variables/timeOffset + longName: "time offset from cycle time" + units: "s" + + - name: "MetaData/stationIdentification" + coordinates: "longitude latitude" + source: variables/stationIdentification + longName: "Station identification" + + - name: "MetaData/stationElevation" + coordinates: "longitude latitude" + source: variables/stationElevation + longName: "Station elevation" + units: "m" + + - name: "MetaData/air_pressure" + coordinates: "longitude latitude Level" + source: variables/pressure + longName: "Pressure" + units: "Pa" + + - name: "MetaData/height" + coordinates: "longitude latitude Level" + source: variables/height + longName: "Height" + units: "m" + + - name: "ObsValue/air_temperature" + coordinates: "longitude latitude Level" + source: variables/airTemperature + longName: "Temperature" + units: "K" + + - name: "ObsValue/dewpoint_temperature" + coordinates: "longitude latitude Level" + source: variables/dewpointTemperature + longName: "Dewpoint temperature" + units: "K" + + - name: "ObsValue/virtual_temperature" + coordinates: "longitude latitude Level" + source: variables/virtualTemperature + longName: "Virtual temperature" + units: "K" + + - name: "ObsValue/specific_humidity" + coordinates: "longitude latitude Level" + source: variables/specificHumidity + longName: "Specific humidity" + units: "kg kg-1" + + - name: "ObsValue/eastward_wind" + coordinates: "longitude latitude Level" + source: variables/windEastward + longName: "Eastward wind" + units: "m s-1" + + - name: "ObsValue/northward_wind" + coordinates: "longitude latitude Level" + source: variables/windNorthward + longName: "Northward wind" + units: "m s-1" + + - name: "ObsValue/sea_surface_temperature" + coordinates: "longitude latitude Depth" + source: variables/seaSurfaceTemperature + longName: "Sea surface temperature" + units: "K" + +# Quality Markers + - name: "QualityMarker/pressure" + coordinates: "longitude latitude Level" + source: variables/pressureQM + longName: "Pressure quality marker" + + - name: "QualityMarker/height" + coordinates: "longitude latitude Level" + source: variables/heightQM + longName: "Height quality marker" + + - name: "QualityMarker/air_temperature" + coordinates: "longitude latitude Level" + source: variables/airTemperatureQM + longName: "Temperature quality marker" + + - name: "QualityMarker/specific_humidity" + coordinates: "longitude latitude Level" + source: variables/specificHumidityQM + longName: "Moisture quality marker" + + - name: "QualityMarker/eastward_wind" + coordinates: "longitude latitude Level" + source: variables/windQM + longName: "U-Component of wind quality marker" + + - name: "QualityMarker/northward_wind" + coordinates: "longitude latitude Level" + source: variables/windQM + longName: "V-Component of wind quality marker" + + - name: "QualityMarker/sea_surface_temperature" + coordinates: "longitude latitude Depth" + source: variables/seaSurfaceTemperatureQM + longName: "Sea surface temperature quality marker" + +# ObsErrors + - name: "ObsError/pressure" + coordinates: "longitude latitude Level" + source: variables/pressureError + longName: "Pressure ObsError" + units: "Pa" + + - name: "ObsError/air_temperature" + coordinates: "longitude latitude Level" + source: variables/airTemperatureError + longName: "Temperature ObsError" + units: "K" + + - name: "ObsError/relative_humidity" + coordinates: "longitude latitude Level" + source: variables/relativeHumidityError + longName: "Relative Humidity ObsError" + units: "1" + + - name: "ObsError/eastward_wind" + coordinates: "longitude latitude Level" + source: variables/windError + longName: "Easthward wind ObsError" + units: "m s-1" + + - name: "ObsError/northward_wind" + coordinates: "longitude latitude Level" + source: variables/windError + longName: "Northward wind ObsError" + units: "m s-1" + + - name: "ObsError/sea_surface_temperature" + coordinates: "longitude latitude Depth" + source: variables/seaSurfaceTemperatureError + longName: "Sea surface temperature ObsError" + units: "K" + +# Extra Info + - name: "MetaData/verticalSignificance" + coordinates: "longitude latitude" + source: variables/verticalSignificance + longName: "Vertical Significance" + + - name: "MetaData/prepbufrType" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "Prepbufr report type" + + - name: "MetaData/dumpReportType" + coordinates: "longitude latitude" + source: variables/dumpReportType + longName: "Data dump report type" + + - name: "MetaData/prepbufrDataLvlCat" + coordinates: "longitude latitude Level" + source: variables/prepbufrDataLvlCat + longName: "Prepbufr data level category" + + - name: "MetaData/waterTemperatureMethod" + coordinates: "longitude latitude" + source: variables/waterTemperatureMethod + longName: "Method of sea surface temperature measurement" + + - name: "ObsValue/presentWeather" + coordinates: "longitude latitude" + source: variables/presentWeather + longName: "Present Weather" + + - name: "ObsValue/cloudAmount" + coordinates: "longitude latitude" + source: variables/cloudAmount + longName: "Cloud Amount" + + - name: "ObsValue/cloudType" + coordinates: "longitude latitude" + source: variables/cloudType + longName: "Cloud Type" + + - name: "ObsValue/heightOfBaseOfCloud" + coordinates: "longitude latitude" + source: variables/heightOfBaseOfCloud + longName: "Height of Base of Cloud" + units: "m" + + - name: "ObsValue/cloudCoverTotal" + coordinates: "longitude latitude" + source: variables/cloudCoverTotal + longName: "Cloud Cover" + units: "1" + + - name: "MetaData/depthBelowSeaSurface" + coordinates: "longitude latitude Depth" + source: variables/depthBelowSeaSurface + longName: "Depth below sea surface" + units: "m" diff --git a/test/testinput/aircftpro_prepbufr.yaml b/test/testinput/aircftpro_prepbufr.yaml new file mode 100644 index 000000000..e83848106 --- /dev/null +++ b/test/testinput/aircftpro_prepbufr.yaml @@ -0,0 +1,327 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + + obsdatain: "./acft_profiles.data_thinned" + + exports: + variables: + stationIdentification: + query: "*/SID" + prepbufrDataLvlCat: + query: "*/PRSLEVLA/CAT" + prepbufrReportType: + query: "*/TYP" + dumpReportType: + query: "*/T29" + obsTimeMinusCycleTime: + query: "*/DHR" +# presentWeather: +# query: "*/PRSLEVLA/PREWXSEQ/PRWE" + longitude: + query: "*/XOB" + latitude: + query: "*/YOB" + group_by: longitude + stationElevation: + query: "*/ELV" + + latitudeProfileLevel: + query: "*/PRSLEVLA/DRFTINFO/YDR" + group_by: longitude + longitudeProfileLevel: + query: "*/PRSLEVLA/DRFTINFO/XDR" + group_by: longitude + timeProfileLevel: + query: "*/PRSLEVLA/DRFTINFO/HRDR" + group_by: longitude + instantaneousAltitudeRate: + query: "*/PRSLEVLA/IALR" + group_by: longitude + aircraftPhase: + query: "*/PRSLEVLA/ACFT_SEQ/POAF" + group_by: longitude + + height: + query: "*/PRSLEVLA/Z___INFO/Z__EVENT/ZOB" + group_by: longitude + pressure: + query: "*/PRSLEVLA/P___INFO/P__EVENT/POB" + transforms: + - scale: 100 + group_by: longitude + pressureError: + query: "*/PRSLEVLA/P___INFO/P__BACKG/POE" + transforms: + - scale: 100 + group_by: longitude + airTemperature: + query: "*/PRSLEVLA/T___INFO/T__EVENT/TOB" + transforms: + - offset: 273.15 + group_by: longitude + airTemperatureError: + query: "*/PRSLEVLA/T___INFO/T__BACKG/TOE" + group_by: longitude + dewpointTemperature: + query: "*/PRSLEVLA/Q___INFO/TDO" + transforms: + - offset: 273.15 + group_by: longitude + specificHumidity: + query: "*/PRSLEVLA/Q___INFO/Q__EVENT/QOB" + transforms: + - scale: 0.000001 + group_by: longitude +# relativeHumidityError: +# query: "*/PRSLEVLA/Q___INFO/Q__EVENT/QOE" + +# cloudAmount: +# query: "*/PRSLEVLA/CLOUDSEQ/CLAM" +# heightOfBaseOfCloud: +# query: "*/PRSLEVLA/CLOUDSEQ/HOCB" + + windEastward: + query: "*/PRSLEVLA/W___INFO/W__EVENT/UOB" + group_by: longitude + windNorthward: + query: "*/PRSLEVLA/W___INFO/W__EVENT/VOB" + group_by: longitude + windError: + query: "*/PRSLEVLA/W___INFO/W__BACKG/WOE" + group_by: longitude + + heightQM: + query: "*/PRSLEVLA/Z___INFO/Z__EVENT/ZQM" + group_by: longitude + pressureQM: + query: "*/PRSLEVLA/P___INFO/P__EVENT/PQM" + group_by: longitude + airTemperatureQM: + query: "*/PRSLEVLA/T___INFO/T__EVENT/TQM" + group_by: longitude + specificHumidityQM: + query: "*/PRSLEVLA/Q___INFO/Q__EVENT/QQM" + group_by: longitude +# verticalSignificanceSurfaceObservations: +# query: "*/PRSLEVLA/CLOUDSEQ/VSSO" +# group_by: longitude + windQM: + query: "*/PRSLEVLA/W___INFO/W__EVENT/WQM" + group_by: longitude + + ioda: + backend: netcdf + obsdataout: "../testoutput/aircraft_prepbufr_acftprofiles.nc" + + variables: + + - name: "MetaData/stationIdentification" + coordinates: "longitude latitude" + source: variables/stationIdentification + longName: "Station ID" + units: "(8)CCITT IA5" + + - name: "MetaData/prepbufrDataLvlCat" + coordinates: "longitude latitude" + source: variables/prepbufrDataLvlCat + longName: "Prepbufr Data Level Category" + units: "CODE TABLE" + + - name: "MetaData/prepbufrReportType" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "Prepbufr Report Type" + units: "CODE TABLE" + + - name: "MetaData/dumpReportType" + coordinates: "longitude latitude" + source: variables/dumpReportType + longName: "Data Dump Report Type" + units: "CODE TABLE" + + - name: "MetaData/obsTimeMinusCycleTime" + coordinates: "longitude latitude" + source: variables/obsTimeMinusCycleTime + longName: "Observation Time Minus Cycle Time" + units: "Hour" + +# - name: "ObsValue/presentWeather" +# coordinates: "longitude latitude" +# source: variables/presentWeather +# longName: "Present weather" +# units: "CODE TABLE" + + - name: "MetaData/latitude" + coordinates: "longitude latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/longitude" + coordinates: "longitude latitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [0, 360] +# range: [-180, 180] + + - name: "ObsValue/stationElevation" + coordinates: "longitude latitude" + source: variables/stationElevation + longName: "Height of Station" + units: "Meter" + + - name: "MetaData/latitudeProfileLevel" + coordinates: "longitude latitude" + source: variables/latitudeProfileLevel + longName: "Profile Level Latitude (for ROAB/PIBAL) based on balloon drift" + units: "degrees_north" + + - name: "MetaData/longitudeProfileLevel" + coordinates: "longitude latitude" + source: variables/longitudeProfileLevel + longName: "Profile Level Longitude (for ROAB/PIBAL) based on balloon drift" + units: "degrees_east" + + - name: "MetaData/timeProfileLevel" + coordinates: "longitude latitude" + source: variables/timeProfileLevel + longName: "Time Profile Level" + units: "Hour" + + - name: "ObsValue/instantaneousAltitudeRate" + coordinates: "longitude latitude" + source: variables/instantaneousAltitudeRate + longName: "Instantaneous Rate Altitue" + units: "Meter Second-1" + + - name: "MetaData/aircraftPhase" + coordinates: "longitude latitude" + source: variables/aircraftPhase + longName: "Phase of Aircraft Flight" + units: "CODE TABLE" + + - name: "ObsValue/height" + coordinates: "longitude latitude" + source: variables/height + longName: "Height of Observation" + units: "Meter" + + - name: "ObsValue/pressure" + coordinates: "longitude latitude" + source: variables/pressure + longName: "Pressure" + units: "Pascals" + range: [20000, 110000] + + - name: "ObsValue/pressureError" + coordinates: "longitude latitude" + source: variables/pressureError + longName: "Pressure Error" + units: "Pascals" + + - name: "ObsValue/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperature + longName: "Temperature" + units: "Kelvin" + range: [193, 325] + + - name: "ObsValue/airTemperatureError" + coordinates: "longitude latitude" + source: variables/airTemperatureError + longName: "Temperature Error" + units: "Kelvin" + + - name: "ObsValue/dewpointTemperature" + coordinates: "longitude latitude" + source: variables/dewpointTemperature + longName: "Dew Point Temperature" + units: "Kelvin" + range: [193, 325] + + - name: "ObsValue/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidity + longName: "Specific Humidity" + units: "Kilogram Kilogram-1" + +# - name: "ObsValue/relativeHumidityError" +# coordinates: "longitude latitude" +# source: variables/relativeHumidityError +# longName: "Relative Humidity Error" +# units: "Percent divided by 10" + +# - name: "ObsValue/cloudAmount" +# coordinates: "longitude latitude" +# source: variables/cloudAmount +# longName: "Cloud Amount" +# units: "CODE TABLE" + +# - name: "ObsValue/heightOfBaseOfCloud" +# coordinates: "longitude latitude" +# source: variables/heightOfBaseOfCloud +# longName: "Height of base of cloud" +# units: "Meter" + + - name: "ObsValue/windEastward" + coordinates: "longitude latitude" + source: variables/windEastward + longName: "U component of Wind" + units: "Meter Second-1" + + - name: "ObsValue/windNorthward" + coordinates: "longitude latitude" + source: variables/windNorthward + longName: "V component of Wind" + units: "Meter Second-1" + + - name: "ObsValue/windError" + coordinates: "longitude latitude" + source: variables/windError + longName: "U, V component of wind error" + units: "Meter Second-1" + + - name: "QualityMarker/heightQM" + coordinates: "longitude latitude" + source: variables/heightQM + longName: "Height QM" + units: "CODE TABLE" + + - name: "QualityMarker/pressureQM" + coordinates: "longitude latitude" + source: variables/pressureQM + longName: "Pressure QM" + units: "CODE TABLE" + + - name: "QualityMarker/airTemperatureQM" + coordinates: "longitude latitude" + source: variables/airTemperatureQM + longName: "Temperature QM" + units: "CODE TABLE" + + - name: "QualityMarker/specificHumidityQM" + coordinates: "longitude latitude" + source: variables/specificHumidityQM + longName: "specific Humidity QM" + units: "CODE TABLE" + +# - name: "QualityMarker/verticalSignificanceSurfaceObservations" +# coordinates: "longitude latitude" +# source: variables/verticalSignificanceSurfaceObservations +# longName: "Vertical Significance (Surface Observations)" +# units: "CODE TABLE" + + - name: "QualityMarker/windQM" + coordinates: "longitude latitude" + source: variables/windQM + longName: "U, V Component of Wind QM" + units: "CODE TABLE" + diff --git a/test/testinput/amsr2_icec_l2p.nc b/test/testinput/amsr2_icec_l2p.nc new file mode 100644 index 000000000..828306b08 --- /dev/null +++ b/test/testinput/amsr2_icec_l2p.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f248ff9834c9ad89277ac4bb5b25c71ff7e8a51bd6ee55835bd910e29a706e18 +size 31971 diff --git a/test/testinput/bufr_empty_fields.bufr_d b/test/testinput/bufr_empty_fields.bufr_d new file mode 100644 index 000000000..1e928db28 --- /dev/null +++ b/test/testinput/bufr_empty_fields.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b56665725197c79d435efa15f871024d6294f157332b9c72f5ea0bb373ab3a67 +size 38120 diff --git a/test/testinput/bufr_empty_fields.yaml b/test/testinput/bufr_empty_fields.yaml new file mode 100644 index 000000000..f742dfd1f --- /dev/null +++ b/test/testinput/bufr_empty_fields.yaml @@ -0,0 +1,56 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# # +# # This software is licensed under the terms of the Apache Licence Version 2.0 +# # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +observations: + - obs space: + name: bufr + + obsdatain: "./testinput/bufr_empty_fields.bufr_d" + + exports: + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + swellWavesDirection: + query: "NC000000/WAVSQ3/DOSW" + heightOfSwellWaves: + query: "NC000000/WAVSQ3/HOSW" + periodOfSwellWaves: + query: "NC000000/WAVSQ3/POSW" + + + ioda: + backend: netcdf + obsdataout: "./testrun/bufr_empty_fields.nc" + + variables: + - name: "MetaData/datetime" + source: variables/timestamp + longName: "Datetime" + units: "datetime" + + - name: "ObsValue/swellWavesDirection" + coordinates: "longitude latitude" + source: variables/swellWavesDirection + longName: "Swell Waves Direction" + units: "Degrees True" + + - name: "ObsValue/heightOfSwellWaves" + coordinates: "longitude latitude" + source: variables/heightOfSwellWaves + longName: "Height of Swell Waves" + units: "Meter" + + - name: "ObsValue/periodOfSwellWaves" + coordinates: "longitude latitude" + source: variables/periodOfSwellWaves + longName: "Period of Swell Waves" + units: "Second" + diff --git a/test/testinput/bufr_filtering.yaml b/test/testinput/bufr_filtering.yaml index 12bc95870..463fd4466 100644 --- a/test/testinput/bufr_filtering.yaml +++ b/test/testinput/bufr_filtering.yaml @@ -7,79 +7,68 @@ observations: - obs space: name: bufr obsdatain: "./testinput/gdas.t18z.1bmhs.tm00.bufr_d" - mnemonicSets: - - mnemonics: [SAID, FOVN, YEAR, MNTH, DAYS, HOUR, MINU, SECO, CLAT, CLON, CLATH, CLONH, HOLS] - - mnemonics: [SAZA, SOZA, BEARAZ, SOLAZI] - - mnemonics: [TMBR] - channels : 1-5 exports: + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + longitude: + query: "*/CLON" + latitude: + query: "*/CLAT" + radiance: + query: "[*/BRITCSTC/TMBR, */BRIT/TMBR]" + filters: - bounding: - mnemonic: CLAT + variable: latitude upperBound: 42.5 - bounding: - mnemonic: CLAT + variable: latitude lowerBound: 35 - bounding: - mnemonic: CLON + variable: longitude upperBound: -68 lowerBound: -86.3 - variables: - timestamp: - datetime: - year: YEAR - month: MNTH - day: DAYS - hour: HOUR - minute: MINU - second: SECO - longitude: - mnemonic: CLON - transforms: - - offset: 0 - latitude: - mnemonic: CLAT - brightnessTemperature: - mnemonic: TMBR - ioda: backend: netcdf obsdataout: "./testrun/gdas.t18z.1bmhs.tm00.filtering.nc" dimensions: - - name: "Location" - size: variables/brightnessTemperature.nrows - - name: "Channel" - size: 5 + - name: Channel + paths: + - "*/BRITCSTC" + - "*/BRIT" variables: - name: "MetaData/dateTime" source: variables/timestamp - dimensions: [ "Location" ] longName: "dateTime" units: "seconds since 1970-01-01T00:00:00Z" - name: "MetaData/latitude" source: variables/latitude - dimensions: ["Location"] longName: "Latitude" units: "degrees_north" range: [-90, 90] - name: "MetaData/longitude" source: variables/longitude - dimensions: ["Location"] longName: "Longitude" units: "degrees_east" range: [-180, 180] - - name: "ObsValue/brightnessTemperature" - source: variables/brightnessTemperature + - name: "radiance@ObsValue" coordinates: "longitude latitude Channel" - dimensions: ["Location", "Channel"] - longName: "Brightness temperature" + source: variables/radiance + longName: "Radiance" units: "K" range: [120, 500] chunks: [1000, 15] diff --git a/test/testinput/bufr_high_dims.yaml b/test/testinput/bufr_high_dims.yaml new file mode 100644 index 000000000..bf547586a --- /dev/null +++ b/test/testinput/bufr_high_dims.yaml @@ -0,0 +1,70 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/gnssro_kompsat5_20180415_00Z.bufr" + + exports: + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + longitude: + query: "*/ROSEQ1/CLONH" + group_by: longitude + latitude: + query: "*/ROSEQ1/CLATH" + group_by: longitude + impactParameter: + query: "*/ROSEQ1/ROSEQ2/IMPP" + group_by: longitude + + ioda: + backend: netcdf + obsdataout: "./testrun/bufr_high_dims.nc" + + dimensions: + - name: roseq1Dim + path: "*/ROSEQ1" + - name: roseq2Dim + path: "*/ROSEQ1/ROSEQ2" + + variables: + - name: "datetime@MetaData" + source: variables/timestamp + longName: "Datetime" + units: "datetime" + + - name: "longitude@MetaData" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + + - name: "latitude@MetaData" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "impactParameter@ObsVal" + source: variables/impactParameter + coordinates: "longitude latitude" + longName: "Impact Parameter" + units: "hz" + + + + + + + diff --git a/test/testinput/bufr_mhs.yaml b/test/testinput/bufr_mhs.yaml index 4fc7f3bb5..d5e1d0dbd 100644 --- a/test/testinput/bufr_mhs.yaml +++ b/test/testinput/bufr_mhs.yaml @@ -7,61 +7,54 @@ observations: - obs space: name: bufr obsdatain: "./testinput/gdas.t18z.1bmhs.tm00.bufr_d" - mnemonicSets: - - mnemonics: [SAID, SIID, FOVN, YEAR, MNTH, DAYS, HOUR, MINU, SECO, CLAT, CLON, CLATH, CLONH] - - mnemonics: [LSQL, HOLS, HMSL] - - mnemonics: [SAZA, SOZA, BEARAZ, SOLAZI] - - mnemonics: [CHNM, TMBR] - channels : 1-5 exports: variables: timestamp: datetime: - year: YEAR - month: MNTH - day: DAYS - hour: HOUR - minute: MINU - second: SECO - latitude: - mnemonic: CLAT + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + height: + query: "*/HMSL" + hols: + query: "*/HOLS" + fovn: + query: "*/FOVN" + lsql: + query: "*/LSQL" longitude: - mnemonic: CLON - satelliteIdentifier: - mnemonic: SAID - stationElevation: - mnemonic: HMSL - heightOfSurface: - mnemonic: HOLS - fieldOfViewNumber: - mnemonic: FOVN - landOrSeaQualifier: - mnemonic: LSQL - fieldOfViewNumber: - mnemonic: FOVN - solarZenithAngle: - mnemonic: SOZA - solarAzimuthAngle: - mnemonic: SOLAZI - sensorZenithAngle: - mnemonic: SAZA - sensorAzimuthAngle: - mnemonic: BEARAZ - sensorChannelNumber: - mnemonic: CHNM - brightnessTemperature: - mnemonic: TMBR + query: "*/CLON" + transforms: + - offset: 50 + latitude: + query: "*/CLAT" + sza: + query: "*/SOZA" + saz: + query: "*/SOLAZI" + vza: + query: "*/SAZA" + vaz: + query: "*/BEARAZ" + channels: + query: "[*/BRITCSTC/CHNM, */BRIT/CHNM]" + radiance: + query: "[*/BRITCSTC/TMBR, */BRIT/TMBR]" ioda: backend: netcdf obsdataout: "./testrun/gdas.t18z.1bmhs.tm00.nc" dimensions: - - name: "Location" - size: variables/latitude.nrows - - name: "Channel" - size: variables/brightnessTemperature.ncols + - name: Channel + paths: + - "*/BRIT" + - "*/BRITCSTC" + source: variables/channels globals: @@ -79,88 +72,70 @@ observations: variables: - - name: "MetaData/dateTime" + - name: "stationElevation@MetaData" + source: variables/height + longName: "Station Elevation" + units: "meters" + + - name: "heightOfSurface@MetaData" + source: variables/hols + longName: "Height of Land Surface" + units: "meters" + + - name: "fieldOfViewNumber@MetaData" + source: variables/fovn + longName: "Field of View Number" + units: "none" + + - name: "landSeaQualifier@MetaData" + source: variables/lsql + longName: "Land/Sea Qualifier" + units: "none" + + - name: "dateTime@MetaData" source: variables/timestamp - dimensions: [ "Location" ] longName: "dateTime" units: "seconds since 1970-01-01T00:00:00Z" - name: "MetaData/latitude" source: variables/latitude - dimensions: ["Location"] longName: "Latitude" units: "degrees_north" range: [-90, 90] - name: "MetaData/longitude" source: variables/longitude - dimensions: ["Location"] longName: "Longitude" units: "degrees_east" range: [-180, 180] - - name: "MetaData/stationElevation" - source: variables/stationElevation - dimensions: ["Location"] - longName: "Height" - units: "m" - - - name: "MetaData/heightOfSurface" - source: variables/heightOfSurface - dimensions: ["Location"] - longName: "Height of Land Surface" - units: "m" - - - name: "MetaData/fieldOfViewNumber" - source: variables/fieldOfViewNumber - dimensions: ["Location"] - longName: "Field of View Number" - units: "none" - - - name: "MetaData/landOrSeaQualifier" - source: variables/landOrSeaQualifier - dimensions: ["Location"] - longName: "Land/Sea Qualifier" - units: "none" - - - name: "MetaData/solarZenithAngle" - source: variables/solarZenithAngle - dimensions: ["Location"] + - name: "solarZenithAngle@MetaData" + source: variables/sza longName: "Solar Zenith Angle" units: "degrees" range: [0, 180] - - name: "MetaData/solarAzimuthAngle" - source: variables/solarAzimuthAngle - dimensions: ["Location"] + - name: "solarAzimuthAngle@MetaData" + source: variables/saz longName: "Solar Azimuth Angle" units: "degrees" range: [-180, 180] - - name: "MetaData/sensorZenithAngle" - source: variables/sensorZenithAngle - dimensions: ["Location"] + - name: "sensorZenithAngle@MetaData" + source: variables/vza longName: "Sensor Zenith Angle" units: "degrees" range: [0, 180] - - name: "MetaData/sensorAzimuthAngle" - source: variables/sensorAzimuthAngle - dimensions: ["Location"] + - name: "sensorAzimuthAngle@MetaData" + source: variables/vaz longName: "Sensor Azimuth Angle" units: "degrees" range: [-180, 180] - - name: "MetaData/sensorChannelNumber" - source: variables/sensorChannelNumber - dimensions: ["Channel"] - longName: "Channel number" - units: "" - - - name: "ObsValue/brightnessTemperature" - source: variables/brightnessTemperature + - name: "brightnessTemperature@ObsValue" coordinates: "longitude latitude Channel" - dimensions: ["Location", "Channel"] + source: variables/radiance longName: "Brightness Temperature" units: "K" range: [120, 500] diff --git a/test/testinput/bufr_ncep_1bamua.yaml b/test/testinput/bufr_ncep_1bamua.yaml new file mode 100755 index 000000000..21a1c5e9f --- /dev/null +++ b/test/testinput/bufr_ncep_1bamua.yaml @@ -0,0 +1,177 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/gdas.t12z.1bamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLAT" + + longitude: + query: "*/CLON" + + satelliteIdentifier: + query: "*/SAID" + + satelliteInstrument: + query: "*/SIID" + + fieldOfViewNumber: + query: "*/FOVN" + + landOrSeaQualifier: + query: "*/LSQL" + + heightOfLandSurface: + query: "*/HOLS" + + heightOfStation: + query: "*/HMSL" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorChannelNumber: + query: "*/BRITCSTC/CHNM" + + # ObsValue + # Note: BUFR Dump contains Antenna Temperature for all normal-feed AMSUAs except NOAA-15/16 + # NOAA-15/16 contain Brightness Temperature + antennaTemperature: + query: "*/BRITCSTC/TMBR" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c +# _206: noaa-15 + _209: noaa-18 + _223: noaa-19 + + ioda: + backend: netcdf + obsdataout: "./testrun/gdas.t12z.1bamua.{splits/satId}.tm00.nc" + + dimensions: + - name: Channel + path: "*/BRITCSTC" + + globals: + - name: "platformCommonName" + type: string + value: "AMSU-A" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-023 PROC AMSU-A 1B Tb" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteInstrument" + source: variables/satelliteInstrument + longName: "Satellite Instrument" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + - name: "MetaData/landOrSeaQualifier" + source: variables/landOrSeaQualifier + longName: "Land/Sea Qualifier" + + - name: "MetaData/heightOfLandSurface" + source: variables/heightOfLandSurface + longName: "Height of Land Surface" + units: "m" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/antennaTemperature" + coordinates: "longitude latitude Channel" + source: variables/antennaTemperature + longName: "Antenna Temperature" + units: "K" + range: [100, 500] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/test/testinput/bufr_ncep_1bamua_n15.yaml b/test/testinput/bufr_ncep_1bamua_n15.yaml new file mode 100755 index 000000000..30ebae522 --- /dev/null +++ b/test/testinput/bufr_ncep_1bamua_n15.yaml @@ -0,0 +1,172 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/gdas.t12z.1bamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLAT" + + longitude: + query: "*/CLON" + + satelliteIdentifier: + query: "*/SAID" + + satelliteInstrument: + query: "*/SIID" + + fieldOfViewNumber: + query: "*/FOVN" + + landOrSeaQualifier: + query: "*/LSQL" + + heightOfLandSurface: + query: "*/HOLS" + + heightOfStation: + query: "*/HMSL" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorChannelNumber: + query: "*/BRITCSTC/CHNM" + + # ObsValue + # Note: BUFR Dump contains Antenna Temperature for all normal-feed AMSUAs except NOAA-15/16 + # NOAA-15/16 contain Brightness Temperature + brightnessTemperature: + query: "*/BRITCSTC/TMBR" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _206: noaa-15 + + ioda: + backend: netcdf + obsdataout: "./testrun/gdas.t12z.1bamua.{splits/satId}.tm00.nc" + + dimensions: + - name: Channel + path: "*/BRITCSTC" + + globals: + - name: "platformCommonName" + type: string + value: "AMSU-A" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-023 PROC AMSU-A 1B Tb" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteInstrument" + source: variables/satelliteInstrument + longName: "Satellite Instrument" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + - name: "MetaData/landOrSeaQualifier" + source: variables/landOrSeaQualifier + longName: "Land/Sea Qualifier" + + - name: "MetaData/heightOfLandSurface" + source: variables/heightOfLandSurface + longName: "Height of Land Surface" + units: "m" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/brightnessTemperature" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemperature + longName: "Brightness Temperature" + units: "K" + range: [100, 500] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/test/testinput/bufr_ncep_1bmhs.yaml b/test/testinput/bufr_ncep_1bmhs.yaml new file mode 100755 index 000000000..63fa6a0c2 --- /dev/null +++ b/test/testinput/bufr_ncep_1bmhs.yaml @@ -0,0 +1,175 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/gdas.t12z.1bmhs.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLAT" + + longitude: + query: "*/CLON" + + satelliteIdentifier: + query: "*/SAID" + + satelliteInstrument: + query: "*/SIID" + + fieldOfViewNumber: + query: "*/FOVN" + + landOrSeaQualifier: + query: "*/LSQL" + + heightOfLandSurface: + query: "*/HOLS" + + heightOfStation: + query: "*/HMSL" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorChannelNumber: + query: "*/BRITCSTC/CHNM" + + # ObsValue + antennaTemperature: + query: "*/BRITCSTC/TMBR" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c + _209: noaa-18 + _223: noaa-19 + + ioda: + backend: netcdf + obsdataout: "./testrun/gdas.t12z.1bmhs.{splits/satId}.tm00.nc" + + dimensions: + - name: Channel + path: "*/BRITCSTC" + + globals: + - name: "platformCommonName" + type: string + value: "MHS" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-027 PROCESSED MHS Tb (NOAA-18-19, METOP-1,2,3)" + + variables: + + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "SatelliteIdentifier" + + - name: "MetaData/satelliteInstrument" + source: variables/satelliteInstrument + longName: "Satellite Instrument" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + - name: "MetaData/landOrSeaQualifier" + source: variables/landOrSeaQualifier + longName: "Land/Sea Qualifier" + + - name: "MetaData/heightOfLandSurface" + source: variables/heightOfLandSurface + longName: "Height of Land Surface" + units: "m" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/antennaTemperature" + coordinates: "longitude latitude Channel" + source: variables/antennaTemperature + longName: "Antenna Temperature" + units: "K" + range: [100, 500] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/test/testinput/bufr_ncep_adpsfc.yaml b/test/testinput/bufr_ncep_adpsfc.yaml new file mode 100644 index 000000000..571cd5fba --- /dev/null +++ b/test/testinput/bufr_ncep_adpsfc.yaml @@ -0,0 +1,296 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# # +# # This software is licensed under the terms of the Apache Licence Version 2.0 +# # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# + +observations: + - obs space: + name: bufr + + obsdatain: "testinput/gdas.t12z.adpsfc.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR[1]" + month: "*/MNTH[1]" + day: "*/DAYS[1]" + hour: "*/HOUR[1]" + minute: "*/MINU[1]" + latitude: + query: "[*/CLAT, */CLATH]" + longitude: + query: "[*/CLON, */CLONH]" + stationIdentification: + query: "*/RPID" + stationElevation: + query: "[*/SELV, */HSMSL]" + waterTemperatureMethod: + query: "*/MSST" + + # ObsValue + pressure: + query: "*/PRES" + type: float + pressureReducedToMeanSeaLevel: + query: "*/PMSL" + type: float + altimeterSetting: + query: "*/ALSE" + type: float + airTemperature: + query: "*/TMDB" + dewpointTemperature: + query: "*/TMDP" + windDirection: + query: "*/WDIR" + type: float + windSpeed: + query: "*/WSPD" + type: float + + # ObsValue - ocean + # DBSS (depth below water surface) not found + waterTemperature: + query: "*/SST1" + heightOfWaves: + query: "*/WAVSQ1/HOWV" + + # ObsValue - cloud, visibility, gust wind, min/max temperature, weather + # CEILING (cloud ceiling) not found + cloudCoverTotal: + query: "*/TOCC" + type: float + transforms: + - scale: 0.01 + cloudAmountDescription: + query: "[*/CLDSQ1/CLAM, */MTRCLD/CLAM, NC000100/CLAM, NC000101/CLAM, NC000102/CLAM]" + heightAboveSurfaceOfBaseOfLowestCloud: + query: "*/HBLCS" + heightOfBaseOfCloud: + query: "[*/CLDSQ1/HOCB, */MTRCLD/HOCB, NC000100/HOCB, NC000101/HOCB, NC000102/HOCB]" + type: float + verticalSignificanceSurfaceObservations: + query: "[*/CLDSQ1/VSSO, */MTRCLD/VSSO, NC000100/VSSO[1], NC000101/VSSO[1], NC000102/VSSO[1]]" + verticalVisibility: + query: "[*/RPSEC3/VTVI, */VTVI]" + type: float + horizontalVisibility: + query: "*/HOVI" + type: float + minimumTemperature: + query: "[*/TMPSQ3/MITM, */MTTPSQ/MITM, NC000100/BSYEXTM/MITM[1], NC000101/BSYEXTM/MITM[1], NC000102/BSYEXTM/MITM[1]]" + maximumTemperature: + query: "[*/TMPSQ3/MXTM, */MTTPSQ/MXTM, */BSYEXTM/MXTM]" + maximumWindGustSpeed: + query: "[*/WNDSQ2/MXGS, */MTGUST/MXGS, */BSYWND2/MXGS]" + presentWeather: + query: "[*/PPWSQ1/PRWE, */MTRPRW/PRWE, */PWEATHER/PRWE]" + + # QualityMarker + airTemperatureQM: + query: "*/QMAT" + dewpointTemperatureQM: + query: "*/QMDD" + pressureQM: + query: "*/QMPR" + windQM: + query: "*/QMWN" + + + ioda: + backend: netcdf + obsdataout: "testrun/gdas.t12z.adpsfc.tm00.nc" + + dimensions: + - name: CloudSequence + path: "*/CLDSQ1" + - name: MaxMinTemperatureSequence + path: "*/MTTPSQ" + - name: SynopticWindSequence + path: "*/BSYWND2" + - name: PresentWeatherSequence + path: "*/MTRPRW" + + variables: + + # MetaData + - name: "MetaData/dateTime" + coordinates: "longitude latitude" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + coordinates: "longitude latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + coordinates: "longitude latitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/stationIdentification" + coordinates: "longitude latitude" + source: variables/stationIdentification + longName: "Station Identification" + + - name: "MetaData/stationElevation" + coordinates: "longitude latitude" + source: variables/stationElevation + longName: "Elevation of Observing Location" + units: "m" + + - name: "MetaData/waterTemperatureMethod" + coordinates: "longitude latitude" + source: variables/waterTemperatureMethod + longName: "Method of Water Temperature Measurement" + + # ObsValue + - name: "ObsValue/altimeterSetting" + coordinates: "longitude latitude" + source: variables/altimeterSetting + longName: "Altimeter Setting" + units: "Pa" + + - name: "ObsValue/pressure" + coordinates: "longitude latitude" + source: variables/pressure + longName: "Pressure" + units: "Pa" + + - name: "ObsValue/pressureReducedToMeanSeaLevel" + coordinates: "longitude latitude" + source: variables/pressureReducedToMeanSeaLevel + longName: "Mean Sea-Level Pressure" + units: "Pa" + + - name: "ObsValue/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperature + longName: "Air Temperature" + units: "K" + + - name: "ObsValue/dewpointTemperature" + coordinates: "longitude latitude" + source: variables/dewpointTemperature + longName: "Dewpoint Temperature" + units: "K" + + - name: "ObsValue/windDirection" + coordinates: "longitude latitude" + source: variables/windDirection + longName: "Wind Direction" + units: "degree" + + - name: "ObsValue/windSpeed" + coordinates: "longitude latitude" + source: variables/windSpeed + longName: "Wind Speed" + units: "m s-1" + + # ObsValue - ocean + - name: "ObsValue/waterTemperature" + coordinates: "longitude latitude" + source: variables/waterTemperature + longName: "Water Temperature" + units: "K" + + - name: "ObsValue/heightOfWaves" + coordinates: "longitude latitude" + source: variables/heightOfWaves + longName: "Height of Waves" + units: "m" + + # ObsValue - cloud, visibility, gust wind, min/max temperature, weather + - name: "ObsValue/cloudCoverTotal" + coordinates: "longitude latitude" + source: variables/cloudCoverTotal + longName: "Total Cloud Coverage" + units: "1" + + - name: "ObsValue/cloudAmountDescription" + coordinates: "longitude latitude" + source: variables/cloudAmountDescription + longName: "Description of Cloud Amount" + + - name: "ObsValue/heightAboveSurfaceOfBaseOfLowestCloud" + coordinates: "longitude latitude" + source: variables/heightAboveSurfaceOfBaseOfLowestCloud + longName: "Height Above Surface of Base of Lowest Cloud Seen" + + - name: "ObsValue/heightOfBaseOfCloud" + coordinates: "longitude latitude" + source: variables/heightOfBaseOfCloud + longName: "Cloud Base Altitude" + units: "m" + + - name: "ObsValue/verticalSignificanceSurfaceObservations" + coordinates: "longitude latitude" + source: variables/verticalSignificanceSurfaceObservations + longName: "Description of Vertical Significance (Surface Observations)" + + - name: "ObsValue/horizontalVisibility" + coordinates: "longitude latitude" + source: variables/horizontalVisibility + longName: "Horizontal Visibility" + units: "m" + + - name: "ObsValue/verticalVisibility" + coordinates: "longitude latitude" + source: variables/verticalVisibility + longName: "Vertical Visibility" + units: "m" + + - name: "ObsValue/minimumTemperature" + coordinates: "longitude latitude" + source: variables/minimumTemperature + longName: "Minimum Temperature at Height and Over Period Specified" + units: "K" + + - name: "ObsValue/maximumTemperature" + coordinates: "longitude latitude" + source: variables/maximumTemperature + longName: "Maximum Temperature at Height and Over Period Specified" + units: "K" + + - name: "ObsValue/maximumWindGustSpeed" + coordinates: "longitude latitude" + source: variables/maximumWindGustSpeed + longName: "Maximum Wind Gust Speed" + units: "m s-1" + + - name: "ObsValue/presentWeather" + coordinates: "longitude latitude" + source: variables/presentWeather + longName: "Description of Present Weather" + + # QualityMarker + - name: "QualityMarker/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureQM + longName: "Quality Indicator for Atmospheric Temperature" + + - name: "QualityMarker/dewpointTemperature" + coordinates: "longitude latitude" + source: variables/dewpointTemperatureQM + longName: "Quality Indicator for Dewpoint Temperature" + + - name: "QualityMarker/pressure" + coordinates: "longitude latitude" + source: variables/pressureQM + longName: "Quality Indicator for Pressure" + + - name: "QualityMarker/windDirection" + coordinates: "longitude latitude" + source: variables/windQM + longName: "Quality Indicator for Wind Direction" + diff --git a/test/testinput/bufr_ncep_adpupa_minu.yaml b/test/testinput/bufr_ncep_adpupa_minu.yaml new file mode 100644 index 000000000..802627d28 --- /dev/null +++ b/test/testinput/bufr_ncep_adpupa_minu.yaml @@ -0,0 +1,55 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + + obsdatain: "./testinput/New/gdas.t12z.adpupa.tm00.bufr_d" + + exports: + variables: + timestamp: + datetime: + year: "[*/YEAR, */RCPTIM/RCYR[1], */RCYR[1]]" + month: "[*/MNTH, */RCPTIM/RCMO[1], */RCMO[1]]" + day: "[*/DAYS, */RCPTIM/RCDY[1], */RCDY[1]]" + hour: "[*/HOUR, */RCPTIM/RCHR[1], */RCHR[1]]" + minute: "[*/MINU, */RCPTIM/RCMI[1], */RCMI[1]]" + + latitude: + query: "*/CLAT" + longitude: + query: "*/CLON" + + + ioda: + backend: netcdf + obsdataout: "./testrun/bufr_ncep_adpupa_minu.nc" + + + variables: + + - name: "MetaData/dateTime" + coordinates: "longitude latitude" + source: variables/timestamp + dimensions: ["Location"] + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + coordinates: "longitude latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/longitude" + coordinates: "longitude latitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + diff --git a/test/testinput/bufr_ncep_esamua.yaml b/test/testinput/bufr_ncep_esamua.yaml new file mode 100755 index 000000000..c43a1f565 --- /dev/null +++ b/test/testinput/bufr_ncep_esamua.yaml @@ -0,0 +1,174 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/gdas.t12z.esamua.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteIdentifier: + query: "*/SAID" + + satelliteAntennaCorrectionsVersionNumber: + query: "*/SACV" + + orbitNumber: + query: "*/ORBN" + + scanLineNumber: + query: "*/SLNM" + + fieldOfViewNumber: + query: "*/FOVN" + + heightOfStation: + query: "*/SELV" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorChannelNumber: + query: "*/ATCHV/INCN" + + # ObsValue + brightnessTemperature: + query: "*/ATCHV/TMBRST" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c +# _206: noaa-15 + _209: noaa-18 + _223: noaa-19 + + ioda: + backend: netcdf + obsdataout: "./testrun/gdas.t12z.esamua.{splits/satId}.tm00.nc" + + dimensions: + - name: Channel + path: "*/ATCHV" + + globals: + - name: "platformCommonName" + type: string + value: "AMSUA" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-033 RARS(EARS,AP,SA) AMSU-A 1C Tb DATA)" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteAntennaCorrectionsVersionNumber" + source: variables/satelliteAntennaCorrectionsVersionNumber + longName: "Satellite Antenna Corrections Version Number" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field Of View Number" + + - name: "MetaData/orbitNumber" + source: variables/orbitNumber + longName: "Orbit Number" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/brightnessTemperature" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemperature + longName: "Brightness Temperature" + units: "K" + range: [100, 500] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/test/testinput/bufr_ncep_esmhs.yaml b/test/testinput/bufr_ncep_esmhs.yaml new file mode 100755 index 000000000..6ca3b5b3d --- /dev/null +++ b/test/testinput/bufr_ncep_esmhs.yaml @@ -0,0 +1,173 @@ +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/gdas.t12z.esmhs.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteIdentifier: + query: "*/SAID" + + satelliteAntennaCorrectionsVersionNumber: + query: "*/SACV" + + orbitNumber: + query: "*/ORBN" + + scanLineNumber: + query: "*/SLNM" + + fieldOfViewNumber: + query: "*/FOVN" + + heightOfStation: + query: "*/SELV" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + sensorChannelNumber: + query: "*/ATCHV/INCN" + + # ObsValue + brightnessTemperature: + query: "*/ATCHV/TMBRST" + + splits: + satId: + category: + variable: satelliteIdentifier + map: + _3: metop-b + _4: metop-a + _5: metop-c + _209: noaa-18 + _223: noaa-19 + + ioda: + backend: netcdf + obsdataout: "./testrun/gdas.t12z.esmhs.{splits/satId}.tm00.nc" + + dimensions: + - name: Channel + path: "*/ATCHV" + + globals: + - name: "platformCommonName" + type: string + value: "MHS" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-036 RARS(EARS,AP,SA) MHS 1C Tb DATA (N18-19, METOP-1,2,3)" + + variables: + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/satelliteAntennaCorrectionsVersionNumber" + source: variables/satelliteAntennaCorrectionsVersionNumber + longName: "Satellite Antenna Corrections Version Number" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + - name: "MetaData/orbitNumber" + source: variables/orbitNumber + longName: "Orbit Number" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number" + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degrees" + range: [0, 180] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [0, 90] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [0, 360] + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/brightnessTemperature" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemperature + longName: "Brightness Temperature" + units: "K" + range: [100, 500] + chunks: [1000, 15] + compressionLevel: 4 diff --git a/test/testinput/bufr_ncep_mtiasi.yaml b/test/testinput/bufr_ncep_mtiasi.yaml new file mode 100755 index 000000000..038a17652 --- /dev/null +++ b/test/testinput/bufr_ncep_mtiasi.yaml @@ -0,0 +1,206 @@ +# (C) Copyright 2021 UCAR +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/gdas.t12z.mtiasi.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + + latitude: + query: "*/CLATH" + + longitude: + query: "*/CLONH" + + satelliteId: + query: "*/SAID" + + sensorId: + query: "*/SIID[1]" + + scanLineNumber: + query: "*/SLNM" + + gqisFlagQual: + query: "*/QGFQ" + + fieldOfViewNumber: + query: "*/FOVN" + + solarZenithAngle: + query: "*/SOZA" + + solarAzimuthAngle: + query: "*/SOLAZI" + + sensorZenithAngle: + query: "*/SAZA" + + sensorAzimuthAngle: + query: "*/BEARAZ" + + heightOfStation: + query: "*/SELV" + + startChannel: + query: "*/IASIL1CB/STCH" + + endChannel: + query: "*/IASIL1CB/ENCH" + + channelScaleFactor: + query: "*/IASIL1CB/CHSF" + + sensorChannelNumber: + query: "*/IASICHN/CHNM" + + # ObsValue + scaledSpectralRadiance: + query: "*/IASICHN/SCRA" + + fractionOfClearPixelsInFov: + query: "*/IASIL1CS/FCPH" + type: float + transforms: + - scale: 0.01 + + splits: + satId: + category: + variable: satelliteId + map: + _3: metop-b + _4: metop-a + _5: metop-c + + ioda: + backend: netcdf + obsdataout: "./testrun/gdas.t12z.mtiasi.{splits/satId}.tm00.nc" + + dimensions: + - name: Channel + path: "*/IASICHN" + - name: Cluster + path: "*/IASIL1CS" + - name: Band + path: "*/IASIL1CB" + + globals: + - name: "platformCommonName" + type: string + value: "IASI" + + - name: "platformLongDescription" + type: string + value: "MTYP 021-027 PROCESSED MHS Tb" + + variables: + + # MetaData + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [ -90, 90 ] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + type: float + units: "degree_east" + range: [ -180, 180 ] + + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteId + longName: "Satellite Identifier" + + - name: "MetaData/instrumentIdentifier" + source: variables/sensorId + longName: "Satellite Instrument Identifier" + + - name: "MetaData/scanLineNumber" + source: variables/scanLineNumber + longName: "Scan Line Number" + + - name: "MetaData/qualityFlags" + source: variables/gqisFlagQual + longName: "Individual IASI-System Quality Flag" + + - name: "MetaData/fieldOfViewNumber" + source: variables/fieldOfViewNumber + longName: "Field of View Number" + + - name: "MetaData/solarZenithAngle" + source: variables/solarZenithAngle + longName: "Solar Zenith Angle" + units: "degree" + range: [ 0, 180 ] + + - name: "MetaData/solarAzimuthAngle" + source: variables/solarAzimuthAngle + longName: "Solar Azimuth Angle" + units: "degree" + range: [ 0, 360 ] + + - name: "MetaData/sensorZenithAngle" + source: variables/sensorZenithAngle + longName: "Sensor Zenith Angle" + units: "degree" + range: [ 0, 90 ] + + - name: "MetaData/sensorAzimuthAngle" + source: variables/sensorAzimuthAngle + longName: "Sensor Azimuth Angle" + units: "degree" + range: [ 0, 360 ] + + - name: "MetaData/heightOfStation" + source: variables/heightOfStation + longName: "Altitude of Satellite" + units: "m" + + - name: "MetaData/startChannel" + source: variables/startChannel + longName: "Start Channel" + + - name: "MetaData/endChannel" + source: variables/endChannel + longName: "End Channel" + + - name: "MetaData/channelScaleFactor" + source: variables/channelScaleFactor + longName: "Channel Scale Factor" + + - name: "MetaData/sensorChannelNumber" + source: variables/sensorChannelNumber + longName: "Sensor Channel Number" + + # ObsValue + - name: "ObsValue/scaledSpectralRadiance" + source: variables/scaledSpectralRadiance + longName: "Scaled IASI Spectral Radiance" + units: "W m-2 sr-1 m" + + - name: "MetaData/fractionOfClearPixelsInFov" + source: variables/fractionOfClearPixelsInFov + longName: "Fraction of Clear Pixels in a Field of View" + units: "1" + range: [0, 1] diff --git a/test/testinput/bufr_ncep_prepbufr_adpsfc.yaml b/test/testinput/bufr_ncep_prepbufr_adpsfc.yaml new file mode 100644 index 000000000..5ba6f514f --- /dev/null +++ b/test/testinput/bufr_ncep_prepbufr_adpsfc.yaml @@ -0,0 +1,427 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + + obsdatain: "./testinput/gdas.t12z.adpsfc.prepbufr" + + exports: + variables: + # MetaData + obsTimeMinusCycleTime: + query: "*/DHR" + prepbufrDataLevelCategory: + query: "*/CAT" + prepbufrReportType: + query: "*/TYP" + dumpReportType: + query: "*/T29" + longitude: + query: "*/XOB" + latitude: + query: "*/YOB" + stationIdentification: + query: "*/SID" + stationElevation: + query: "*/ELV" + type: float + waterTemperatureMethod: + query: "*/SST_INFO/MSST" + + # ObsValue + heightOfObservation: + query: "*/Z___INFO/Z__EVENT/ZOB" + type: float + pressure: + query: "*/P___INFO/P__EVENT/POB" + transforms: + - scale: 100 + pressureReducedToMeanSeaLevel: + query: "*/PMSL_SEQ/PMO" + transforms: + - scale: 100 + airTemperature: + query: "*/T___INFO/T__EVENT/TOB" + transforms: + - offset: 273.15 + dewpointTemperature: + query: "*/Q___INFO/TDO" + transforms: + - offset: 273.15 + specificHumidity: + type: float + query: "*/Q___INFO/Q__EVENT/QOB" + transforms: + - scale: 0.000001 + windEastward: + query: "*/W___INFO/W__EVENT/UOB" + windNorthward: + query: "*/W___INFO/W__EVENT/VOB" + + # ObsValue - ocean + waterTemperature: + query: "*/SST_INFO/SSTEVENT/SST1" + heightOfWaves: + query: "*/WAVE_SEQ/HOWV" + type: float + depthBelowWaterSurface: + query: "*/SST_INFO/DBSS_SEQ/DBSS" + type: float + # ObsValue - cloud, cloud ceiling, visibility, gust wind, min/max temperature, weather + # note: cloud ceiling is a derivative of HOCB, the height of cloud base + cloudCoverTotal: + query: "*/CLOU2SEQ/TOCC" + type: float + transforms: + - scale: 0.01 + cloudAmountDescription: + query: "*/CLOUDSEQ/CLAM" + cloudCeiling: + query: "*/CLOU3SEQ/CEILING" + type: float + heightAboveSurfaceOfBaseOfLowestCloud: + query: "*/CLOU2SEQ/HBLCS" + heightOfBaseOfCloud: + query: "*/CLOUDSEQ/HOCB" + type: float + verticalSignificanceSurfaceObservations: + query: "*/CLOUDSEQ/VSSO" + verticalVisibility: + query: "*/VISB1SEQ/VTVI_SEQ/VTVI" + type: float + horizontalVisibility: + query: "*/VISB1SEQ/HOVI" + type: float + minimumTemperature: + query: "*/TMXMNSEQ/MITM" + maximumTemperature: + query: "*/TMXMNSEQ/MXTM" + maximumWindGustSpeed: + query: "*/GUST1SEQ/MXGS" + presentWeather: + query: "*/PREWXSEQ/PRWE" + + # QualityMarker + heightQualityMarker: + query: "*/Z___INFO/Z__EVENT/ZQM" + pressureQualityMarker: + query: "*/P___INFO/P__EVENT/PQM" + pressureReducedToMeanSeaLevelQualityMarker: + query: "*/PMSL_SEQ/PMQ" + airTemperatureQualityMarker: + query: "*/T___INFO/T__EVENT/TQM" + specificHumidityQualityMarker: + query: "*/Q___INFO/Q__EVENT/QQM" + waterTemperatureQualityMarker: + query: "*/SST_INFO/SSTEVENT/SSTQM" + windEastwardQualityMarker: + query: "*/W___INFO/W__EVENT/WQM" + windNorthwardQualityMarker: + query: "*/W___INFO/W__EVENT/WQM" + + # ObsError + pressureError: + query: "*/P___INFO/P__BACKG/POE" + transforms: + - scale: 100 + airTemperatureError: + query: "*/T___INFO/T__BACKG/TOE" + relativeHumidityError: + query: "*/Q___INFO/Q__BACKG/QOE" + transforms: + - scale: 0.1 + waterTemperatureError: + query: "*/SST_INFO/SSTBACKG/SSTOE" + windSpeedError: + query: "*/W___INFO/W__BACKG/WOE" + + ioda: + backend: netcdf + obsdataout: "testrun/gdas.t12z.adpsfc.prepbufr.nc" + + dimensions: + - name: CloudSequence + path: "*/CLOUDSEQ" + - name: MaxMinTemperatureSequence + path: "*/TMXMNSEQ" + - name: PresentWeatherSequence + path: "*/PREWXSEQ" + - name: HeightEvent + path: "*/Z___INFO/Z__EVENT" + - name: PressureEvent + path: "*/P___INFO/P__EVENT" + - name: TemperatureEvent + path: "*/T___INFO/T__EVENT" + - name: HumidityEvent + path: "*/Q___INFO/Q__EVENT" + - name: WaterTemperatureEvent + path: "*/SST_INFO/SSTEVENT" + - name: WindEvent + path: "*/W___INFO/W__EVENT" + + variables: + + # MetaData + - name: "MetaData/obsTimeMinusCycleTime" + coordinates: "longitude latitude" + source: variables/obsTimeMinusCycleTime + longName: "Observation Time Minus Cycle Time" + units: "Hour" + + - name: "MetaData/prepbufrDataLevelCategory" + coordinates: "longitude latitude" + source: variables/prepbufrDataLevelCategory + longName: "Prepbufr Data Level Category" + + - name: "MetaData/prepbufrReportType" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "Prepbufr Report Type" + + - name: "MetaData/dumpReportType" + coordinates: "longitude latitude" + source: variables/dumpReportType + longName: "Data Dump Report Type" + + - name: "MetaData/latitude" + coordinates: "longitude latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + coordinates: "longitude latitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [0, 360] + + - name: "MetaData/stationIdentification" + coordinates: "longitude latitude" + source: variables/stationIdentification + longName: "Station Identification" + + - name: "MetaData/stationElevation" + coordinates: "longitude latitude" + source: variables/stationElevation + longName: "Elevation of Observing Location" + units: "m" + + - name: "MetaData/waterTemperatureMethod" + coordinates: "longitude latitude" + source: variables/waterTemperatureMethod + longName: "Method of Water Temperature Measurement" + + # ObsValue + - name: "ObsValue/heightOfObservation" + coordinates: "longitude latitude" + source: variables/heightOfObservation + longName: "Height of Observation (Station)" + units: "m" + + - name: "ObsValue/pressure" + coordinates: "longitude latitude" + source: variables/pressure + longName: "Pressure" + units: "Pa" + + - name: "ObsValue/pressureReducedToMeanSeaLevel" + coordinates: "longitude latitude" + source: variables/pressureReducedToMeanSeaLevel + longName: "Mean Sea-Level Pressure" + units: "Pa" + + - name: "ObsValue/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperature + longName: "Temperature" + units: "K" + + - name: "ObsValue/dewpointTemperature" + coordinates: "longitude latitude" + source: variables/dewpointTemperature + longName: "Dewpoint Temperature" + units: "K" + + - name: "ObsValue/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidity + longName: "Specific Humidity" + units: "kg kg-1" + + - name: "ObsValue/windEastward" + coordinates: "longitude latitude" + source: variables/windEastward + longName: "Eastward Wind" + units: "m s-1" + + - name: "ObsValue/windNorthward" + coordinates: "longitude latitude" + source: variables/windNorthward + longName: "Northward Wind" + units: "m s-1" + + # ObsValue - ocean + - name: "ObsValue/waterTemperature" + coordinates: "longitude latitude" + source: variables/waterTemperature + longName: "Water Temperature" + units: "K" + + - name: "ObsValue/heightOfWaves" + coordinates: "longitude latitude" + source: variables/heightOfWaves + longName: "Height of Waves" + units: "m" + + - name: "ObsValue/depthBelowWaterSurface" + coordinates: "longitude latitude" + source: variables/depthBelowWaterSurface + longName: "Depth Below Water Surface" + units: "m" + + # Observation - cloud, visibility, gust wind, min/max temperature + - name: "ObsValue/cloudCoverTotal" + coordinates: "longitude latitude" + source: variables/cloudCoverTotal + longName: "Total Cloud Coverage" + units: "1" + + - name: "ObsValue/cloudAmountDescription" + coordinates: "longitude latitude" + source: variables/cloudAmountDescription + longName: "Description of Cloud Amount" + + - name: "ObsValue/cloudCeiling" + coordinates: "longitude latitude" + source: variables/cloudCeiling + longName: "Cloud Ceiling" + units: "m" + + - name: "ObsValue/heightAboveSurfaceOfBaseOfLowestCloud" + coordinates: "longitude latitude" + source: variables/heightAboveSurfaceOfBaseOfLowestCloud + longName: "Height above Surface of Base of Lowest Cloud Seen" + + - name: "ObsValue/heightOfBaseOfCloud" + coordinates: "longitude latitude" + source: variables/heightOfBaseOfCloud + longName: "Height of Base of Cloud" + units: "m" + + - name: "ObsValue/verticalSignificanceSurfaceObservations" + coordinates: "longitude latitude" + source: variables/verticalSignificanceSurfaceObservations + longName: "Description of Vertical Significance (Surface Observations)" + + - name: "ObsValue/horizontalVisibility" + coordinates: "longitude latitude" + source: variables/horizontalVisibility + longName: "Horizontal Visibility" + units: "m" + + - name: "ObsValue/verticalVisibility" + coordinates: "longitude latitude" + source: variables/verticalVisibility + longName: "Vertical Visibility" + units: "m" + + - name: "ObsValue/minimumTemperature" + coordinates: "longitude latitude" + source: variables/minimumTemperature + longName: "Minimum Temperature at Height and Over Period Specified" + units: "K" + + - name: "ObsValue/maximumTemperature" + coordinates: "longitude latitude" + source: variables/maximumTemperature + longName: "Maximum Temperature at Height and Over Period Specified" + units: "K" + + - name: "ObsValue/maximumWindGustSpeed" + coordinates: "longitude latitude" + source: variables/maximumWindGustSpeed + longName: "Maximum Wind Gust Speed" + units: "m s-1" + + - name: "ObsValue/presentWeather" + coordinates: "longitude latitude" + source: variables/presentWeather + longName: "Description of Present Weather" + + # QualityMarker + - name: "QualityMarker/height" + coordinates: "longitude latitude" + source: variables/heightQualityMarker + longName: "Height Quality Marker" + + - name: "QualityMarker/pressure" + coordinates: "longitude latitude" + source: variables/pressureQualityMarker + longName: "Pressure Quality Marker" + + - name: "QualityMarker/pressureReducedToMeanSeaLevel" + coordinates: "longitude latitude" + source: variables/pressureReducedToMeanSeaLevelQualityMarker + longName: "Mean Sea Level Pressure Quality Marker" + + - name: "QualityMarker/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureQualityMarker + longName: "Temperature Quality Marker" + + - name: "QualityMarker/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidityQualityMarker + longName: "Specific Humidity Quality Marker" + + - name: "QualityMarker/waterTemperature" + coordinates: "longitude latitude" + source: variables/waterTemperatureQualityMarker + longName: "Water Temperature Quality Marker" + + - name: "QualityMarker/windNorthward" + coordinates: "longitude latitude" + source: variables/windNorthwardQualityMarker + longName: "U, V-Component of Wind Quality Marker" + + - name: "QualityMarker/windEastward" + coordinates: "longitude latitude" + source: variables/windEastwardQualityMarker + longName: "U, V-Component of Wind Quality Marker" + + # ObsError + - name: "ObsError/pressure" + coordinates: "longitude latitude" + source: variables/pressureError + longName: "Pressure Error" + units: "Pa" + + - name: "ObsError/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureError + longName: "Temperature Error" + units: "K" + + - name: "ObsError/relativeHumidity" + coordinates: "longitude latitude" + source: variables/relativeHumidityError + longName: "Relative Humidity Error" + units: "1" + + - name: "ObsError/waterTemperature" + coordinates: "longitude latitude" + source: variables/waterTemperatureError + longName: "Water Temperature Obs Error" + units: "K" + + - name: "ObsError/windSpeed" + coordinates: "longitude latitude" + source: variables/windSpeedError + longName: "East and Northward wind error" + units: "m s-1" diff --git a/test/testinput/bufr_ncep_prepbufr_adpupa.yaml b/test/testinput/bufr_ncep_prepbufr_adpupa.yaml new file mode 100755 index 000000000..9c8b65e91 --- /dev/null +++ b/test/testinput/bufr_ncep_prepbufr_adpupa.yaml @@ -0,0 +1,427 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + + obsdatain: "./testinput/ADPUPA.prepbufr" + + exports: + variables: + stationIdentification: + query: "*/SID" + group_by: prepbufrDataLvlCat + longitude: + query: "*/XOB" + latitude: + query: "*/YOB" + obsTimeMinusCycleTime: + query: "*/DHR" + stationElevation: + query: "*/ELV" + group_by: prepbufrDataLvlCat + prepbufrReportType: + query: "*/TYP" + group_by: prepbufrDataLvlCat + dumpReportType: + query: "*/T29" + group_by: prepbufrDataLvlCat + + prepbufrDataLvlCat: + query: "*/PRSLEVEL/CAT" + group_by: prepbufrDataLvlCat + + methodofTemperatureSeaSurfaceMeasurement: + query: "*/SST_INFO/MSST" + + presentWeather: + query: "*/PREWXSEQ/PRWE" + + verticalSignificanceSurfaceObservations: + query: "*/CLOUDSEQ/VSSO" + cloudAmount: + query: "*/CLOUDSEQ/CLAM" + cloudType: + query: "*/CLOUDSEQ/CLTP" + heightOfBaseOfCloud: + query: "*/CLOUDSEQ/HOCB" + + cloudCoverTotal: + query: "*/CLOU2SEQ/TOCC" + heightAboveSurfaceofBaseofLowestCloud: + query: "*/CLOU2SEQ/HBLCS" + + dewpointTemperature: + query: "*/PRSLEVEL/Q___INFO/TDO" + group_by: prepbufrDataLvlCat + transforms: + - offset: 273.15 + + virtualTemperature: + query: "*/PRSLEVEL/T___INFO/TVO" + group_by: prepbufrDataLvlCat + transforms: + - offset: 273.15 + + pressure: + query: "*/PRSLEVEL/P___INFO/P__EVENT/POB" + group_by: prepbufrDataLvlCat + transforms: + - scale: 100 + pressureQualityMarker: + query: "*/PRSLEVEL/P___INFO/P__EVENT/PQM" + group_by: prepbufrDataLvlCat + + specificHumidity: + query: "*/PRSLEVEL/Q___INFO/Q__EVENT/QOB" + group_by: prepbufrDataLvlCat + transforms: + - scale: 0.000001 + specificHumidityQualityMarker: + query: "*/PRSLEVEL/Q___INFO/Q__EVENT/QQM" + group_by: prepbufrDataLvlCat + + airTemperature: + query: "*/PRSLEVEL/T___INFO/T__EVENT/TOB" + group_by: prepbufrDataLvlCat + transforms: + - offset: 273.15 + airTemperatureQualityMarker: + query: "*/PRSLEVEL/T___INFO/T__EVENT/TQM" + group_by: prepbufrDataLvlCat + + heightOfObservation: + query: "*/PRSLEVEL/Z___INFO/Z__EVENT/ZOB" + group_by: prepbufrDataLvlCat + heightQualityMarker: + query: "*/PRSLEVEL/Z___INFO/Z__EVENT/ZQM" + group_by: prepbufrDataLvlCat + + windEastward: + query: "*/PRSLEVEL/W___INFO/W__EVENT/UOB" + group_by: prepbufrDataLvlCat + windNorthward: + query: "*/PRSLEVEL/W___INFO/W__EVENT/VOB" + group_by: prepbufrDataLvlCat + windQualityMarker: + query: "*/PRSLEVEL/W___INFO/W__EVENT/WQM" + group_by: prepbufrDataLvlCat + + pressureError: + query: "*/PRSLEVEL/P___INFO/P__BACKG/POE" + group_by: prepbufrDataLvlCat + transforms: + - scale: 100 + relativeHumidityError: + query: "*/PRSLEVEL/Q___INFO/Q__BACKG/QOE" + group_by: prepbufrDataLvlCat + airTemperatureError: + query: "*/PRSLEVEL/T___INFO/T__BACKG/TOE" + group_by: prepbufrDataLvlCat + transforms: + - offset: 273.15 + windError: + query: "*/PRSLEVEL/W___INFO/W__BACKG/WOE" + group_by: prepbufrDataLvlCat + + lonProfileLevel: + query: "*/PRSLEVEL/DRFTINFO/XDR" + group_by: prepbufrDataLvlCat + latProfileLevel: + query: "*/PRSLEVEL/DRFTINFO/YDR" + group_by: prepbufrDataLvlCat + timeCycleProfileLevel: + query: "*/PRSLEVEL/DRFTINFO/HRDR" + group_by: prepbufrDataLvlCat + + seaSurfaceTemperature: + query: "*/SST_INFO/SSTEVENT/SST1" + seaSurfaceTemperatureQualityMarker: + query: "*/SST_INFO/SSTEVENT/SSTQM" + + seaSurfaceTemperatureError: + query: "*/SST_INFO/SSTBACKG/SSTOE" + + depthBelowSeaSurface: + query: "*/SST_INFO/DBSS_SEQ/DBSS" + + + ioda: + backend: netcdf + obsdataout: "./testrun/bufr_ncep_prepbufr_adpupa.nc" + + + dimensions: + - name: Level + path: "*/PRSLEVEL" + - name: cloudseq_Dim + path: "*/CLOUDSEQ" + - name: pevent_Dim + path: "*/PRSLEVEL/P___INFO/P__EVENT" + - name: qevent_Dim + path: "*/PRSLEVEL/Q___INFO/Q__EVENT" + - name: tevent_Dim + path: "*/PRSLEVEL/T___INFO/T__EVENT" + - name: zevent_Dim + path: "*/PRSLEVEL/Z___INFO/Z__EVENT" + - name: wevent_Dim + path: "*/PRSLEVEL/W___INFO/W__EVENT" + + + variables: + + - name: "MetaData/stationIdentification" + coordinates: "longitude latitude" + source: variables/stationIdentification + longName: "Station ID" + units: "" + + - name: "MetaData/longitude" + coordinates: "longitude latitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [0, 360] + + - name: "MetaData/latitude" + coordinates: "longitude latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/obsTimeMinusCycleTime" + coordinates: "longitude latitude" + source: variables/obsTimeMinusCycleTime + longName: "Observation Time Minus Cycle Time" + units: "Hours" + + - name: "MetaData/stationElevation" + coordinates: "longitude latitude" + source: variables/stationElevation + longName: "Height of Station" + units: "Meter" + + - name: "MetaData/prepbufrReportType" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "Prepbufr Report Type" + units: "" + + - name: "MetaData/dumpReportType" + coordinates: "longitude latitude" + source: variables/dumpReportType + longName: "Data Dump Report Type" + units: "" + + - name: "MetaData/prepbufrDataLvlCat" + coordinates: "longitude latitude" + source: variables/prepbufrDataLvlCat + longName: "Prepbufr Data Level Category" + units: "" + + - name: "MetaData/methodofTemperatureSeaSurfaceMeasurement" + coordinates: "longitude latitude" + source: variables/methodofTemperatureSeaSurfaceMeasurement + longName: "Method of Sea Surface Measurement" + units: "" + + - name: "ObsValue/presentWeather" + coordinates: "longitude latitude" + source: variables/presentWeather + longName: "Present Weather" + units: "" + + - name: "QualityMarker/verticalSignificanceSurfaceObservations" + coordinates: "longitude latitude" + source: variables/verticalSignificanceSurfaceObservations + longName: "Vertical Significance (Surface Observations)" + units: "" + + - name: "ObsValue/cloudAmount" + coordinates: "longitude latitude" + source: variables/cloudAmount + longName: "Cloud Amount" + units: "" + + - name: "MetaData/cloudType" + coordinates: "longitude latitude" + source: variables/cloudType + longName: "Cloud Type" + units: "" + + - name: "ObsValue/heightOfBaseOfCloud" + coordinates: "longitude latitude" + source: variables/heightOfBaseOfCloud + longName: "Height of Base of Cloud" + units: "Meter" + + - name: "ObsValue/cloudCoverTotal" + coordinates: "longitude latitude" + source: variables/cloudCoverTotal + longName: "Cloud Cover" + units: "Percent" + + - name: "ObsValue/heightAboveSurfaceofBaseofLowestCloud" + coordinates: "longitude latitude" + source: variables/heightAboveSurfaceofBaseofLowestCloud + longName: "Height above Surface of Base of Lowest Cloud" + units: "" + + - name: "ObsValue/dewpointTemperature" + coordinates: "longitude latitude" + source: variables/dewpointTemperature + longName: "Dew Point" + units: "Kelvin" +# range: [193, 325] + + - name: "ObsValue/virtualTemperature" + coordinates: "longitude latitude" + source: variables/virtualTemperature + longName: "Virtual Temperature Non-Q Controlled" + units: "Kelvin" +# range: [193, 325] + + - name: "ObsValue/pressure" + coordinates: "longitude latitude" + source: variables/pressure + longName: "Pressure" + units: "Pa" +# range: [20000, 110000] + + - name: "QualityMarker/pressure" + coordinates: "longitude latitude" + source: variables/pressureQualityMarker + longName: "Pressure Quality Marker" + units: "" + + - name: "ObsValue/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidity + longName: "Specific Humidity" + units: "Kilogram Kilogram-1" + + - name: "QualityMarker/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidityQualityMarker + longName: "Specific Humidity Quality Marker" + units: "" + + - name: "ObsValue/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperature + longName: "Temperature" + units: "Kelvin" +# range: [193, 325] + + - name: "QualityMarker/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureQualityMarker + longName: "Temperature Quality Marker" + units: "" + + - name: "ObsValue/heightOfObservation" + coordinates: "longitude latitude" + source: variables/heightOfObservation + longName: "Height of Observation" + units: "Meter" + + - name: "QualityMarker/height" + coordinates: "longitude latitude" + source: variables/heightQualityMarker + longName: "Height Quality Marker" + units: "" + + - name: "ObsValue/windEastward" + coordinates: "longitude latitude" + source: variables/windEastward + longName: "Eastward Wind" + units: "Meter Second-1" +# range: [-50, 50] + + - name: "ObsValue/windNorthward" + coordinates: "longitude latitude" + source: variables/windNorthward + longName: "Northward Wind" + units: "Meter Second-1" +# range: [-50, 50] + + - name: "QualityMarker/wind" + coordinates: "longitude latitude" + source: variables/windQualityMarker + longName: "U, V-Component of Wind Quality Marker" + units: "" + + - name: "ObsError/pressure" + coordinates: "longitude latitude" + source: variables/pressureError + longName: "Pressure Error" + units: "Pa" + + - name: "ObsError/relativeHumidity" + coordinates: "longitude latitude" + source: variables/relativeHumidityError + longName: "Relative Humidity Error" + units: "Percent" +# units: "Percent divided by 10" + + - name: "ObsError/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureError + longName: "Temperature Error" + units: "Kelvin" + + - name: "ObsError/wind" + coordinates: "longitude latitude" + source: variables/windError + longName: "East and Northward wind error" + units: "Meter Second-1" + + - name: "MetaData/lonProfileLevel" + coordinates: "longitude latitude" + source: variables/lonProfileLevel + longName: "Longitude Profile Level" + units: "degrees_east" + range: [0, 360] + + - name: "MetaData/latProfileLevel" + coordinates: "longitude latitude" + source: variables/latProfileLevel + longName: "Latitude Profile Level" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/timeCycleProfileLevel" + coordinates: "longitude latitude" + source: variables/timeCycleProfileLevel + longName: "Time Cycle Profile Level" + units: "Hours" + + - name: "ObsValue/seaSurfaceTemperature" + coordinates: "longitude latitude" + source: variables/seaSurfaceTemperature + longName: "Sea Surface Temperature" + units: "Kelvin" + + - name: "QualityMarker/seaSurfaceTemperature" + coordinates: "longitude latitude" + source: variables/seaSurfaceTemperatureQualityMarker + longName: "Sea Surface Temperature Quality Marker" + units: "" + + - name: "ObsError/seaSurfaceTemperature" + coordinates: "longitude latitude" + source: variables/seaSurfaceTemperatureError + longName: "Sea Surface Temperature Obs Error" + units: "Kelvin" + + - name: "ObsValue/depthBelowSeaSurface" + coordinates: "longitude latitude" + source: variables/depthBelowSeaSurface + longName: "Depth Below Sea Surface" + units: "Meter" +# range: [-200, 0] + diff --git a/test/testinput/bufr_ncep_satwind_avhrr.yaml b/test/testinput/bufr_ncep_satwind_avhrr.yaml new file mode 100755 index 000000000..ce89cffa8 --- /dev/null +++ b/test/testinput/bufr_ncep_satwind_avhrr.yaml @@ -0,0 +1,161 @@ +#(C) Copyright 2021 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# SAID, CLAT, CLON, YEAR, MNTH, DAYS, HOUR, MINU, SECO, SWCM, SAZA, OGCE, SCCF, SWQM +# HAMD, PRLC, WDIR, WSPD +# GNAP, PCCF + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/gdas.t18z.satwnd_avhrr.tm00.bufr_d" + + exports: + variables: + satelliteIdentifier: + query: "*/SAID" + + latitude: + query: "*/CLAT" + + longitude: + query: "*/CLON" + + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + + windComputationMethod: + query: "*/SWCM" + + satelliteZenithAngle: + query: "*/SAZA" + + dataProviderOrigin: + query: "*/OGCE" + + sensorCentralFrequency: + query: "*/SCCF" + + qualityFlags: + query: "*/SWQM" + + windHeightAssignMethod: + query: "*/HAMD" + + pressure: + query: "*/PRLC" + + windDirection: + query: "*/WDIR" + + windSpeed: + query: "*/WSPD" + + windSpeedGeneratingApplication: + query: "*/GQCPRMS/GNAP[1]" + + windDirectionGeneratingApplication: + query: "*/GQCPRMS/GNAP[2]" + + windSpeedPercentConfidence: + query: "*/GQCPRMS/PCCF[1]" + + windDirectionPercentConfidence: + query: "*/GQCPRMS/PCCF[2]" + + ioda: + backend: netcdf + obsdataout: "./testrun/gdas.t18z.satwnd_avhrr.tm00.nc" + dimensions: + - name: Confidence + paths: + - "*/GQCPRMS" + + variables: + - name: "MetaData/satelliteIdentifier" + source: variables/satelliteIdentifier + longName: "Satellite Identifier" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude (Coarse Accuracy)" + units: "degree_north" + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude (Coarse Accuracy)" + units: "degree_east" + + - name: "MetaData/dateTime" + coordinates: "longitude latitude" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/windComputationMethod" + source: variables/windComputationMethod + longName: "Satellite Derived Wind Computation Method" + + - name: "MetaData/satelliteZenithAngle" + source: variables/satelliteZenithAngle + longName: "Satellite Zenith Angle" + units: "degree" + + - name: "MetaData/dataProviderOrigin" + source: variables/dataProviderOrigin + longName: "Identification of Originating/Generating Center" + + - name: "MetaData/sensorCentralFrequency" + source: variables/sensorCentralFrequency + longName: "Satellite Channel Center Frequency" + units: "Hz" + + - name: "MetaData/qualityFlags" + source: variables/qualityFlags + longName: "SDMEDIT Satellite Wind Quality Mark" + + - name: "MetaData/windHeightAssignMethod" + source: variables/windHeightAssignMethod + longName: "Wind Height Assignment Method" + + - name: "MetaData/pressure" + source: variables/pressure + longName: "Pressure" + units: "Pa" + + - name: "ObsValue/windDirection" + source: variables/windDirection + longName: "Wind Direction" + units: "degree" + + - name: "ObsValue/windSpeed" + source: variables/windSpeed + longName: "Wind Speed" + units: "m s-1" + + - name: "MetaData/windSpeedGeneratingApplication" + source: variables/windSpeedGeneratingApplication + longName: "Wind Speed Generating Application" + + - name: "MetaData/windDirectionGeneratingApplication" + source: variables/windDirectionGeneratingApplication + longName: "Wind Direction Generating Application" + + - name: "MetaData/windSpeedPercentConfidence" + source: variables/windSpeedPercentConfidence + longName: "Wind Speed Per Cent Confidence" + units: "percent" + + - name: "MetaData/windDirectionPercentConfidence" + source: variables/windDirectionPercentConfidence + longName: "Wind Direction Per Cent Confidence" + units: "percent" + diff --git a/test/testinput/bufr_seviri.yaml b/test/testinput/bufr_ncep_sevcsr.yaml similarity index 63% rename from test/testinput/bufr_seviri.yaml rename to test/testinput/bufr_ncep_sevcsr.yaml index 69c1239c9..e7eb600d1 100644 --- a/test/testinput/bufr_seviri.yaml +++ b/test/testinput/bufr_ncep_sevcsr.yaml @@ -1,4 +1,5 @@ -# (C) Copyright 2021 UCAR +# (C) Copyright 2021-2022 UCAR + # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -7,52 +8,44 @@ observations: - obs space: name: bufr obsdatain: "./testinput/gdas.t00z.sevcsr.tm00.bufr_d" - mnemonicSets: - - mnemonics: [YEAR, MNTH, DAYS, HOUR, MINU, CLATH, CLONH] - - mnemonics: [SAID, SAZA, SOZA] - - mnemonics: [CLDMNT, TMBRST, SDTB] - channels: 1-12 - - mnemonics: [SEQNUM] exports: variables: timestamp: datetime: - year: YEAR - month: MNTH - day: DAYS - hour: HOUR - minute: MINU + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" latitude: - mnemonic: CLATH + query: "*/CLATH" longitude: - mnemonic: CLONH + query: "*/CLONH" satelliteIdentifier: - mnemonic: SAID + query: "*/SAID" solarZenithAngle: - mnemonic: SOZA + query: "*/SOZA" sensorZenithAngle: - mnemonic: SAZA + query: "*/SAZA" cloudAmount: - mnemonic: CLDMNT + query: "*/RPSEQ7/CLDMNT" + type: float transforms: - scale: 0.01 brightnessTemperature: - mnemonic: TMBRST -# brightnessTemperatureStdDev: -# mnemonic: SDTB - observationSequenceNum: - mnemonic: SEQNUM + query: "*/RPSEQ7/TMBRST" + brightnessTemperatureStandardDeviation: + query: "*/RPSEQ7/SDTB" ioda: backend: netcdf obsdataout: "./testrun/gdas.t00z.sevcsr.tm00.nc" dimensions: - - name: "Location" - size: variables/latitude.nrows - name: "Channel" - size: variables/brightnessTemperature.ncols + paths: + - "*/RPSEQ7" globals: - name: "platformCommonName" @@ -66,40 +59,33 @@ observations: variables: - name: "MetaData/satelliteIdentifier" source: variables/satelliteIdentifier - dimensions: ["Location"] longName: "Satellite identifier" - units: "" - name: "MetaData/latitude" source: variables/latitude - dimensions: ["Location"] longName: "Latitude" units: "degrees_north" range: [-90, 90] - name: "MetaData/longitude" source: variables/longitude - dimensions: ["Location"] longName: "Longitude" units: "degrees_east" range: [-180, 180] - name: "MetaData/dateTime" source: variables/timestamp - dimensions: ["Location"] longName: "dateTime" units: "seconds since 1970-01-01T00:00:00Z" - name: "MetaData/solarZenithAngle" source: variables/solarZenithAngle - dimensions: ["Location"] longName: "Solar zenith angle" units: "degrees" range: [0, 180] - name: "MetaData/sensorZenithAngle" source: variables/sensorZenithAngle - dimensions: ["Location"] longName: "Sensor zenith angle" units: "degrees" range: [0, 90] @@ -113,25 +99,22 @@ observations: - name: "ObsValue/cloudAmount" coordinates: "longitude latitude Channel" source: variables/cloudAmount - dimensions: ["Location", "Channel"] - longName: "cloud fraction" + longName: "Cloud amount in segment" units: "1" range: [0, 1] - chunks: [1000, 12] + chunks: [1000, 15] - name: "ObsValue/brightnessTemperature" - source: variables/brightnessTemperature coordinates: "longitude latitude Channel" - dimensions: ["Location", "Channel"] + source: variables/brightnessTemperature longName: "Brightness temperature" units: "K" range: [150, 350] chunks: [1000, 12] -# - name: "ObsValue/brightnessTemperatureStdDev" -# source: variables/brightnessTemperatureStdDev -# coordinates: "longitude latitude Channel" -# dimensions: ["Location", "Channel"] -# longName: "Brightness temperature standard deviation" -# units: "K" -# chunks: [1000, 15] + - name: "ObsValue/brightnessTemperatureStandardDeviation" + coordinates: "longitude latitude Channel" + source: variables/brightnessTemperatureStandardDeviation + longName: "Brightness temperature standard deviation" + units: "K" + chunks: [1000, 15] diff --git a/test/testinput/bufr_ncep_snow_adpsfc.yaml b/test/testinput/bufr_ncep_snow_adpsfc.yaml new file mode 100644 index 000000000..1efe28aad --- /dev/null +++ b/test/testinput/bufr_ncep_snow_adpsfc.yaml @@ -0,0 +1,104 @@ +# (C) Copyright 2021-2022 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + + obsdatain: "./testinput/gdas.t06z.adpsfc.tm00.bufr_d" + + exports: + variables: + timestamp: + datetime: + year: "*/YEAR[1]" + month: "*/MNTH[1]" + day: "*/DAYS[1]" + hour: "*/HOUR[1]" + minute: "*/MINU[1]" + + longitude: + query: "[*/CLON, */CLONH]" + + latitude: + query: "[*/CLAT, */CLATH]" + + stationPressure: + query: "*/PRES" + + stationElevation: + query: "[*/SELV, */HSMSL]" + + stationIdentification: + query: "*/RPID" + + airTemperature: + query: "*/TMDB" + + dewpointTemperature: + query: "*/TMDP" + + totalSnowDepth: + query: "[*/SNWSQ1/TOSD, */MTRMSC/TOSD, */STGDSNDM/TOSD]" + + + ioda: + backend: netcdf + obsdataout: "./testrun/gdas.t06z.adpsfc_snow.tm00.nc" + + variables: + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "dateTime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/stationIdentification" + source: variables/stationIdentification + longName: "Report Number" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + + - name: "MetaData/stationElevation" + coordinates: "longitude latitude" + source: variables/stationElevation + longName: "Height of Station" + units: "m" + + - name: "ObsValue/stationPressure" + coordinates: "longitude latitude" + source: variables/stationPressure + longName: "Station Pressure" + units: "Pa" + range: [90000, 105000] + + - name: "ObsValue/airTemperature" + source: variables/airTemperature + longName: "Temperature of Air" + units: "K" + range: [230, 315] + + - name: "ObsValue/dewpointTemperature" + coordinates: "longitude latitude" + source: variables/dewpointTemperature + longName: "Dewpoint Temperature" + units: "K" + range: [200, 315] + + - name: "ObsValue/totalSnowDepth" + coordinates: "longitude latitude" + source: variables/totalSnowDepth + longName: "Total Snow Depth" + units: "m" + range: [0, 100] diff --git a/test/testinput/bufr_read_2_dim_blocks.yaml b/test/testinput/bufr_read_2_dim_blocks.yaml index 0e19a85ce..caf12882a 100644 --- a/test/testinput/bufr_read_2_dim_blocks.yaml +++ b/test/testinput/bufr_read_2_dim_blocks.yaml @@ -1,4 +1,4 @@ -# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# (C) Copyright 2021 NOAA/NWS/NCEP/EMC # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -7,77 +7,105 @@ observations: - obs space: name: bufr obsdatain: "./testinput/bufr_read_2_dim_blocks.bufr" - mnemonicSets: - - mnemonics: [YEAR, MNTH, DAYS, HOUR, MINU, CLAT, CLON] - - mnemonics: [OGCE, GNAP, PCCF] - channels: 1-12 exports: + variables: - timestamp: - datetime: - year: YEAR - month: MNTH - day: DAYS - hour: HOUR - minute: MINU - longitude: - mnemonic: CLON + latitude: - mnemonic: CLAT - dataProviderOrigin: - mnemonic: OGCE - generatingApplication: - mnemonic: GNAP - windPercentConfidence: - mnemonic: PCCF + query: "*/CLAT" + + longitude: + query: "*/CLON" + + originatingGeneratingCenter: + query: "*/OGCE" + + pressure: + query: "*/QCPRMS/GNAP[1]" + + windDirection: + query: "*/QCPRMS/GNAP[2]" + + windSpeed: + query: "*/QCPRMS/GNAP[3]" + + coldClusterTemperature: + query: "*/QCPRMS/GNAP[4]" + + percentConfidencePressure: + query: "*/QCPRMS/PCCF[1]" + + percentConfidenceWindDirection: + query: "*/QCPRMS/PCCF[2]" + + percentConfidenceWindSpeed: + query: "*/QCPRMS/PCCF[3]" + + percentConfidenceColdClusterTemperature: + query: "*/QCPRMS/PCCF[4]" ioda: backend: netcdf obsdataout: "./testrun/bufr_read_2_dim_blocks.nc" dimensions: - - name: "Location" - size: variables/windPercentConfidence.nrows - name: "Confidence" - size: variables/windPercentConfidence.ncols + paths: + - "*/QCPRMS" variables: - - name: "MetaData/dateTime" - source: variables/timestamp - dimensions: [ "Location" ] - longName: "dateTime" - units: "seconds since 1970-01-01T00:00:00Z" - - name: "MetaData/latitude" source: variables/latitude - dimensions: ["Location"] - longName: "Latitude" - units: "degrees_north" - range: [-90, 90] + longName: "Latitude (Coarse Accuracy)" + units: "degree_north" - name: "MetaData/longitude" source: variables/longitude - dimensions: ["Location"] - longName: "Longitude" - units: "degrees_east" - range: [-180, 180] + longName: "Longitude (Coarse Accuracy)" + units: "degree_east" - name: "MetaData/dataProviderOrigin" - source: variables/dataProviderOrigin - dimensions: [ "Location" ] - longName: "Originating/Generating Center" - units: "" - - - name: "MetaData/generatingApplication" - source: variables/generatingApplication - dimensions: [ "Location" ] - longName: "Generating Application" - units: "" - - - name: "ObsValue/windPercentConfidence" - source: variables/windPercentConfidence + source: variables/originatingGeneratingCenter + longName: "Originating Generating Center" + + - name: "MetaData/pressureGeneratingApplication" + source: variables/pressure + longName: "Generating Application for Pressure" + + - name: "MetaData/windDirectionGeneratingApplication" + source: variables/windDirection + longName: "Generating Application for Wind Direction" + + - name: "MetaData/windSpeedGeneratingApplication" + source: variables/windSpeed + longName: "Generating Application for Wind Speed" + + - name: "MetaData/coldClusterTemperatureGeneratingApplication" + source: variables/coldClusterTemperature + longName: "Generating Application for Cold Cluster Temperature" + + - name: "MetaData/pressurePercentConfidence" + coordinates: "longitude latitude" + source: variables/percentConfidencePressure + longName: "Percent Confidence of Pressure" + units: "percent" + + - name: "MetaData/windDirectionPercentConfidence" coordinates: "longitude latitude" - dimensions: [ "Location", "Confidence"] - longName: "Percent Confidence" + source: variables/percentConfidenceWindDirection + longName: "Percent Confidence of Wind Direction" units: "percent" + + - name: "MetaData/windSpeedPercentConfidence" + coordinates: "longitude latitude" + source: variables/percentConfidenceWindSpeed + longName: "Percent Confidence of Wind Speed" + units: "percent" + + - name: "MetaData/coldClusterTemperaturePercentConfidence" + coordinates: "longitude latitude" + source: variables/percentConfidenceColdClusterTemperature + longName: "Percent Confidence of Clod Cluster Temperature" + units: "percent" + diff --git a/test/testinput/bufr_sfcshp.bufr b/test/testinput/bufr_sfcshp.bufr new file mode 100644 index 000000000..7144de9bb --- /dev/null +++ b/test/testinput/bufr_sfcshp.bufr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21167dc5d3a6e694287b7659def2a3cf39106df4fc4a52d1920f3f572c47072b +size 196936 diff --git a/test/testinput/bufr_sfcshp.yaml b/test/testinput/bufr_sfcshp.yaml new file mode 100644 index 000000000..8ff9b4f88 --- /dev/null +++ b/test/testinput/bufr_sfcshp.yaml @@ -0,0 +1,99 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/bufr_sfcshp.bufr" + + exports: + group_by_variable: depth + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "" + + stationID: + query: "*/RPID" + + longitude: + query: "*/CLONH" + + latitude: + query: "*/CLATH" + + sst: + query: "*/SST0" + + t-profile: + query: "*/IDMSMDBS/BBYSTSL/SST1" + + s-profile: + query: "*/IDMSMDBS/BBYSTSL/SALN" + + depth: + query: "*/IDMSMDBS/BBYSTSL/DBSS" + + ioda: + backend: netcdf + obsdataout: "./testrun/bufr_sfcshp.nc" + + globals: + - name: "dataType@MetaData" + type: string + value: "Moored Buoy" + + dimensions: + - name: Levels + paths: + - "*/IDMSMDBS/BBYSTSL" + + variables: + - name: "datetime@MetaData" + source: variables/timestamp + longName: "Datetime" + units: "datetime" + + - name: "stationIdentification@MetaData" + source: variables/stationID + longName: "Report Identifier" + units: "string" + + - name: "latitude@MetaData" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "longitude@MetaData" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + + - name: "seaSurfaceTemperature@ObsValue" + coordinates: "longitude latitude" + source: variables/sst + longName: "Sea Surface Temperature" + units: "K" + range: [250, 320] + + - name: "depthBelowWaterSurface@MetaData" + coordinates: "longitude latitude Levels" + source: variables/depth + longName: "Water Depth Below Surface" + units: "m" + + - name: "waterTemperature@ObsValue" + coordinates: "longitude latitude Levels" + source: variables/t-profile + longName: "Sea Water Temperature" + units: "K" + range: [250, 320] diff --git a/test/testinput/bufr_simple_groupby.bufr b/test/testinput/bufr_simple_groupby.bufr new file mode 100644 index 000000000..b1a574efd --- /dev/null +++ b/test/testinput/bufr_simple_groupby.bufr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4106b129a61c0e42ae3c754caf24b2c012becca6ab29c6a7d34834e782e9e89b +size 159872 diff --git a/test/testinput/bufr_simple_groupby.yaml b/test/testinput/bufr_simple_groupby.yaml new file mode 100644 index 000000000..b6bdb563c --- /dev/null +++ b/test/testinput/bufr_simple_groupby.yaml @@ -0,0 +1,67 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + obsdatain: "./testinput/bufr_simple_groupby.bufr" + + exports: + group_by_variable: depth + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + longitude: + query: "*/CLON" + latitude: + query: "*/CLAT" + depth: + query: "*/DTSCUR/DBSS" + temp: + query: "*/DTSCUR/STMP" + saln: + query: "*/DTSCUR/SALN" + ioda: + backend: netcdf + obsdataout: "./testrun/bufr_simple_groupby.nc" + + variables: + + - name: "datetime@MetaData" + source: variables/timestamp + longName: "Datetime" + units: "datetime" + + - name: "latitude@MetaData" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "longitude@MetaData" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [-180, 180] + + - name: "depth@MetaData" + source: variables/depth + longName: "Depth below sea surface" + units: "m" + + - name: "sea_water_temperature@ObsValue" + source: variables/temp + longName: "Temperature at depth" + units: "deg K" + + - name: "sea_water_salinity@ObsValue" + source: variables/saln + longName: "Salinity at depth" + units: "PSU" diff --git a/test/testinput/bufr_specific_subsets_by_query.yaml b/test/testinput/bufr_specific_subsets_by_query.yaml new file mode 100644 index 000000000..a2023f3d1 --- /dev/null +++ b/test/testinput/bufr_specific_subsets_by_query.yaml @@ -0,0 +1,48 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# # # +# # # This software is licensed under the terms of the Apache Licence Version 2.0 +# # # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +observations: + - obs space: + name: bufr + + obsdatain: "./testinput/gdas.t12z.aircft.tm00.bufr_d" + + exports: + #MetaData + variables: + timestamp: + datetime: + year: "[NC004001/YEAR, NC004002/YEAR, NC004003/YEAR, NC004006/YEAR, NC004009/YEAR, NC004010/YEAR, NC004011/YEAR]" + month: "[NC004001/MNTH, NC004002/MNTH, NC004003/MNTH, NC004006/MNTH, NC004009/MNTH, NC004010/MNTH, NC004011/MNTH]" + day: "[NC004001/DAYS, NC004002/DAYS, NC004003/DAYS, NC004006/DAYS, NC004009/DAYS, NC004010/DAYS, NC004011/DAYS]" + hour: "[NC004001/HOUR, NC004002/HOUR, NC004003/HOUR, NC004006/HOUR, NC004009/HOUR, NC004010/HOUR, NC004011/HOUR]" + minute: "[NC004001/MINU, NC004002/MINU, NC004003/MINU, NC004006/MINU, NC004009/MINU, NC004010/MINU, NC004011/MINU]" + latitude: + query: "[NC004001/CLAT, NC004002/CLAT, NC004003/CLAT, NC004006/CLATH, NC004009/CLATH, NC004010/CLATH, NC004011/CLATH]" + longitude: + query: "[NC004001/CLON, NC004002/CLON, NC004003/CLON, NC004006/CLONH, NC004009/CLONH, NC004010/CLONH, NC004011/CLONH]" + + ioda: + backend: netcdf + obsdataout: "./testrun/bufr_specifying_subsets.nc" + + #MetaData + variables: + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] diff --git a/test/testinput/bufr_specifying_subsets.yaml b/test/testinput/bufr_specifying_subsets.yaml new file mode 100644 index 000000000..3e45a1acc --- /dev/null +++ b/test/testinput/bufr_specifying_subsets.yaml @@ -0,0 +1,57 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# # # +# # # This software is licensed under the terms of the Apache Licence Version 2.0 +# # # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +observations: + - obs space: + name: bufr + + obsdatain: "./testinput/gdas.t12z.aircft.tm00.bufr_d" + + exports: + subsets: + - NC004001 + - NC004002 + - NC004003 + - NC004006 + - NC004009 + - NC004010 + - NC004011 + + #MetaData + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + latitude: + query: "[*/CLATH, */CLAT]" + longitude: + query: "[*/CLONH, */CLON]" + + ioda: + backend: netcdf + obsdataout: "./testrun/bufr_specifying_subsets.nc" + + #MetaData + variables: + - name: "MetaData/dateTime" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] diff --git a/test/testinput/bufr_splitting.yaml b/test/testinput/bufr_splitting.yaml index edf050d85..c87b0d5c7 100644 --- a/test/testinput/bufr_splitting.yaml +++ b/test/testinput/bufr_splitting.yaml @@ -7,78 +7,65 @@ observations: - obs space: name: bufr obsdatain: "./testinput/gdas.t18z.1bmhs.tm00.bufr_d" - mnemonicSets: - - mnemonics: [SAID, FOVN, YEAR, MNTH, DAYS, HOUR, MINU, SECO, CLAT, CLON, CLATH, CLONH, HOLS] - - mnemonics: [SAZA, SOZA, BEARAZ, SOLAZI] - - mnemonics: [TMBR] - channels : 1-5 exports: + variables: + timestamp: + datetime: + year: "*/YEAR" + month: "*/MNTH" + day: "*/DAYS" + hour: "*/HOUR" + minute: "*/MINU" + second: "*/SECO" + longitude: + query: "*/CLON" + latitude: + query: "*/CLAT" + radiance: + query: "[*/BRITCSTC/TMBR, */BRIT/TMBR]" + splits: hour: - category: - mnemonic: HOUR + category: + variable: timestamp_hour minute: category: - mnemonic: MINU + variable: timestamp_minute map: # Optional _5: five #can't use integers as keys so underscore _6: six _7: seven - variables: - timestamp: - datetime: - year: YEAR - month: MNTH - day: DAYS - hour: HOUR - minute: MINU - second: SECO - longitude: - mnemonic: CLON - transforms: - - offset: 0 - latitude: - mnemonic: CLAT - brightnessTemperature: - mnemonic: TMBR - ioda: backend: netcdf obsdataout: "./testrun/gdas.t18z.1bmhs.tm00.{splits/hour}.{splits/minute}.split.nc" dimensions: - - name: "Location" - size: variables/brightnessTemperature.nrows - name: "Channel" - size: variables/brightnessTemperature.ncols + path: "*/BRITCSTC" variables: - name: "MetaData/dateTime" source: variables/timestamp - dimensions: [ "Location" ] longName: "dateTime" units: "seconds since 1970-01-01T00:00:00Z" - name: "MetaData/latitude" source: variables/latitude - dimensions: ["Location"] longName: "Latitude" units: "degrees_north" range: [-90, 90] - name: "MetaData/longitude" source: variables/longitude - dimensions: ["Location"] longName: "Longitude" units: "degrees_east" range: [-180, 180] - - name: "ObsValue/brightnessTemperature" - source: variables/brightnessTemperature + - name: "radiance@ObsValue" coordinates: "longitude latitude Channel" - dimensions: ["Location", "Channel"] + source: variables/radiance longName: "Radiance" units: "K" range: [120, 500] diff --git a/test/testinput/gdas.t00z.sevcsr.tm00.bufr_d b/test/testinput/gdas.t00z.sevcsr.tm00.bufr_d index 70ee9b2c9..81a5011df 100644 --- a/test/testinput/gdas.t00z.sevcsr.tm00.bufr_d +++ b/test/testinput/gdas.t00z.sevcsr.tm00.bufr_d @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8977151480d41ac15a127354e8393b276e3853b8fa47193e50aad45cbc0de37a -size 499917 +oid sha256:b7b7a0c92d6e3757bb1745b2257036618dbea8dc000cfdaa8fc7deb9e161bd08 +size 254384 diff --git a/test/testinput/gdas.t12z.1bamua.tm00.bufr_d b/test/testinput/gdas.t12z.1bamua.tm00.bufr_d new file mode 100644 index 000000000..704e36553 --- /dev/null +++ b/test/testinput/gdas.t12z.1bamua.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c19f3611a457d35465d7ec99764f0dc4100d9736e0becd5479824af96c9cdfd4 +size 192440 diff --git a/test/testinput/gdas.t12z.1bmhs.tm00.bufr_d b/test/testinput/gdas.t12z.1bmhs.tm00.bufr_d new file mode 100644 index 000000000..588726688 --- /dev/null +++ b/test/testinput/gdas.t12z.1bmhs.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d491163081b0184d20f57815207e20ce701768164f55e0865f7a5d9222bfd639 +size 332264 diff --git a/test/testinput/gdas.t12z.adpsfc.prepbufr b/test/testinput/gdas.t12z.adpsfc.prepbufr new file mode 100644 index 000000000..83cc56d27 --- /dev/null +++ b/test/testinput/gdas.t12z.adpsfc.prepbufr @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab6f14528cd939e27b9556958a2d0629ae482d3f26ba67b18ddd7e196b0f4a5f +size 199648 diff --git a/test/testinput/gdas.t12z.adpsfc.tm00.bufr_d b/test/testinput/gdas.t12z.adpsfc.tm00.bufr_d new file mode 100644 index 000000000..b8440c0d2 --- /dev/null +++ b/test/testinput/gdas.t12z.adpsfc.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:342c674293ca16293193ed958bc2ed9af54b58e4df9506e7ee011de0bd5ffa82 +size 837416 diff --git a/test/testinput/gdas.t12z.adpupa.tm00.bufr_d b/test/testinput/gdas.t12z.adpupa.tm00.bufr_d new file mode 100644 index 000000000..f1a001963 --- /dev/null +++ b/test/testinput/gdas.t12z.adpupa.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b520bf510c44a218816ffd89313b31ba09bd7b66db1364444634e3841897409d +size 1616312 diff --git a/test/testinput/gdas.t12z.aircft.tm00.bufr_d b/test/testinput/gdas.t12z.aircft.tm00.bufr_d new file mode 100644 index 000000000..704f6fa51 --- /dev/null +++ b/test/testinput/gdas.t12z.aircft.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41b1ee8552c02f8015a32fe3eea1708fa3fd39c8bf02bd5cd4a7a42d50c2adf9 +size 123088 diff --git a/test/testinput/gdas.t12z.esamua.tm00.bufr_d b/test/testinput/gdas.t12z.esamua.tm00.bufr_d new file mode 100644 index 000000000..c841465b5 --- /dev/null +++ b/test/testinput/gdas.t12z.esamua.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03237cf364e8a28a3fa67ba67ab78e60e63e1e0981d4c60d415f76337c421e52 +size 96808 diff --git a/test/testinput/gdas.t12z.esmhs.tm00.bufr_d b/test/testinput/gdas.t12z.esmhs.tm00.bufr_d new file mode 100644 index 000000000..2b9b6d91d --- /dev/null +++ b/test/testinput/gdas.t12z.esmhs.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52b436a4ac2e227696cbc3ffd780ddda53150bfdc90eccd1c258e5f7e3c071ea +size 197160 diff --git a/test/testinput/gdas.t12z.mtiasi.tm00.bufr_d b/test/testinput/gdas.t12z.mtiasi.tm00.bufr_d new file mode 100644 index 000000000..a1461b890 --- /dev/null +++ b/test/testinput/gdas.t12z.mtiasi.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96429ba439a551dfcdbb3ae4e39261a6868e3ba20a8b3f62783f8de88be14584 +size 837304 diff --git a/test/testinput/gdas.t18z.satwnd_avhrr.tm00.bufr_d b/test/testinput/gdas.t18z.satwnd_avhrr.tm00.bufr_d new file mode 100644 index 000000000..47fef057e --- /dev/null +++ b/test/testinput/gdas.t18z.satwnd_avhrr.tm00.bufr_d @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f47375e24820464a6e4afbc082b9fd75eb7f7387ab6666ba9cfce02c2ba1d5 +size 1561008 diff --git a/test/testinput/modis_aod.hdf b/test/testinput/modis_aod.hdf new file mode 100644 index 000000000..e8d6e5a7f Binary files /dev/null and b/test/testinput/modis_aod.hdf differ diff --git a/test/testinput/modis_aod.nc b/test/testinput/modis_aod.nc deleted file mode 100644 index b6a51d97d..000000000 --- a/test/testinput/modis_aod.nc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4a0854d2e218f00b92ba6e7474c549f9e1850e1804c85f006ca10712f1bb47c8 -size 56829344 diff --git a/test/testinput/nsidc_l4_icec.nc b/test/testinput/nsidc_l4_icec.nc new file mode 100644 index 000000000..2dc569cd3 --- /dev/null +++ b/test/testinput/nsidc_l4_icec.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ced9bfeb4a1aacf8bbf928d11e18de603d056ab0c8f136f5c52a8f663d9ab792 +size 109892 diff --git a/test/testinput/pace_radiance_L1B.nc b/test/testinput/pace_radiance_L1B.nc new file mode 100644 index 000000000..6cdcd25bb --- /dev/null +++ b/test/testinput/pace_radiance_L1B.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b817a978812c10b8bf9d78f8e8f5ce70560e84903f963ab7314293df913a9d81 +size 178698 diff --git a/test/testinput/satbias_converter.yaml b/test/testinput/satbias_converter.yaml index 652f6653b..8151658b6 100644 --- a/test/testinput/satbias_converter.yaml +++ b/test/testinput/satbias_converter.yaml @@ -202,7 +202,19 @@ output: predictors: *default_preds - sensor: gmi_gpm output file: satbias_gmi_gpm.nc4 - predictors: *default_preds + predictors: + - constant + - zenith_angle + - cosine_of_latitude_times_orbit_node + - lapse_rate_order_2 + - lapse_rate + - cloud_liquid_water_order_2 + - cloud_liquid_water + - emissivity + - scan_angle_order_4 + - scan_angle_order_3 + - scan_angle_order_2 + - scan_angle - sensor: saphir_meghat output file: satbias_saphir_meghat.nc4 predictors: *default_preds diff --git a/test/testinput/satbias_converter_gmi_gpm.yaml b/test/testinput/satbias_converter_gmi_gpm.yaml new file mode 100644 index 000000000..0fa78eb1f --- /dev/null +++ b/test/testinput/satbias_converter_gmi_gpm.yaml @@ -0,0 +1,18 @@ +input coeff file: testinput/satbias_crtm_in +input err file: testinput/satbias_crtm_pc +output: +- sensor: gmi_gpm + predictors: + - constant + - zenith_angle + - cosine_of_latitude_times_orbit_node + - lapse_rate_order_2 + - lapse_rate + - cloud_liquid_water_order_2 + - cloud_liquid_water + - emissivity + - scan_angle_order_4 + - scan_angle_order_3 + - scan_angle_order_2 + - scan_angle + output file: testrun/satbias_gmi_gpm.nc4 diff --git a/test/testoutput/2020100106_metars_small.nc b/test/testoutput/2020100106_metars_small.nc index 10f10eb9b..a855888cd 100644 --- a/test/testoutput/2020100106_metars_small.nc +++ b/test/testoutput/2020100106_metars_small.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4314c0264f7c8164cc40441bf4a52456621611291a46740cd6f71a87ea9123f5 -size 18742 +oid sha256:22d1a4434d3ef77dfa3beb4d9aacb8262b17c83d9f209a51cee72c2f70162872 +size 27572 diff --git a/test/testoutput/2021081612_sonde_small.nc b/test/testoutput/2021081612_sonde_small.nc new file mode 100644 index 000000000..5ac982935 --- /dev/null +++ b/test/testoutput/2021081612_sonde_small.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d450724d0f06e7a80c9e557be321e2719cbc007c18f45b6d6cff63a8d8e64b +size 70380 diff --git a/test/testoutput/SWOT_L2_ADT.nc b/test/testoutput/SWOT_L2_ADT.nc new file mode 100644 index 000000000..4f7340e64 --- /dev/null +++ b/test/testoutput/SWOT_L2_ADT.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f701f5bf947b87dffd6385909d9248a1220993bb6f354a21cc10440e271b3620 +size 298133 diff --git a/test/testoutput/adpupa_prepbufr.nc b/test/testoutput/adpupa_prepbufr.nc new file mode 100644 index 000000000..4808938c0 --- /dev/null +++ b/test/testoutput/adpupa_prepbufr.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2a0727a471571d9e9a09dd80e1f7125a999d137a875d7e9a62e2489c1084dcd +size 409232 diff --git a/test/testoutput/adpupa_prepbufr_group_by.nc b/test/testoutput/adpupa_prepbufr_group_by.nc new file mode 100644 index 000000000..50374d37c --- /dev/null +++ b/test/testoutput/adpupa_prepbufr_group_by.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c31f373858d01bf89403ee97613719fb3fb184a481b6edd0c68ff03578eb551 +size 477389 diff --git a/test/testoutput/aircraft_prepbufr_acftprofiles.nc b/test/testoutput/aircraft_prepbufr_acftprofiles.nc new file mode 100644 index 000000000..678968025 --- /dev/null +++ b/test/testoutput/aircraft_prepbufr_acftprofiles.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66486f8d8abe5350794db0ddab4b2e0cd10f4a1c031e326e76311ada41bea2c2 +size 217400 diff --git a/test/testoutput/amdar_wmo_multi2.nc b/test/testoutput/amdar_wmo_multi2.nc index 6a4cff897..022b0d44f 100644 --- a/test/testoutput/amdar_wmo_multi2.nc +++ b/test/testoutput/amdar_wmo_multi2.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5a461fc873ef0825dce0e481e52b15c48d21870c6a21ff04cbb8e720d4c27645 -size 142201 +oid sha256:27f4d5448dc3ec99c37dbf67106e9084791ab01340dd5f48aedb574acd37e7b9 +size 146615 diff --git a/test/testoutput/amsr2_icec_l2p.nc b/test/testoutput/amsr2_icec_l2p.nc new file mode 100644 index 000000000..5ac443805 --- /dev/null +++ b/test/testoutput/amsr2_icec_l2p.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36c2d2b5e4f2719aef5749f6e1b5383c5431b4cc0ecab3d87e2f527f2d76fac2 +size 16951 diff --git a/test/testoutput/amsua_metop-b_obs_2020110112.nc4 b/test/testoutput/amsua_metop-b_obs_2020110112.nc4 new file mode 100644 index 000000000..ceb55678f --- /dev/null +++ b/test/testoutput/amsua_metop-b_obs_2020110112.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:812b4a72f97befcf5e8dcbf11297c4c36cafee5b8839e54a5aa89b7bc01930ab +size 126289 diff --git a/test/testoutput/amsua_n18_obs_2020110112.nc4 b/test/testoutput/amsua_n18_obs_2020110112.nc4 new file mode 100644 index 000000000..22abeca6b --- /dev/null +++ b/test/testoutput/amsua_n18_obs_2020110112.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34ba505d99286afae2d3e073cd1c98a3abed83cfdc36ac5dab2f61d57165c6b3 +size 129269 diff --git a/test/testoutput/bufr_empty_fields.nc b/test/testoutput/bufr_empty_fields.nc new file mode 100644 index 000000000..5d657f386 --- /dev/null +++ b/test/testoutput/bufr_empty_fields.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dae902e3517280891dbb4fbdf39f7d47ec72c066daedaac438f1520fa9de17f +size 24615 diff --git a/test/testoutput/bufr_ncep_adpupa_minu.nc b/test/testoutput/bufr_ncep_adpupa_minu.nc new file mode 100644 index 000000000..fe7319e10 --- /dev/null +++ b/test/testoutput/bufr_ncep_adpupa_minu.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b40fbfb1a02c4bbf9ae29058a52960769380c36bbae920536a921f0eaa8f2dfc +size 25718 diff --git a/test/testoutput/bufr_ncep_prepbufr_adpupa.nc b/test/testoutput/bufr_ncep_prepbufr_adpupa.nc new file mode 100644 index 000000000..f508657ec --- /dev/null +++ b/test/testoutput/bufr_ncep_prepbufr_adpupa.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ad4c3cbae71f6a161e9d10283dc2cbb18cc08cdf73f83dfa731bc856b9f2f77 +size 1609832 diff --git a/test/testoutput/bufr_read_2_dim_blocks.nc b/test/testoutput/bufr_read_2_dim_blocks.nc index 0980db381..a8d59bf31 100644 --- a/test/testoutput/bufr_read_2_dim_blocks.nc +++ b/test/testoutput/bufr_read_2_dim_blocks.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ae61ef6a25e4a0f682a064b1ea1eaf4ca233e892ff88778b174e58183f53deb7 -size 160404 +oid sha256:d04f07940271c4f098c9b8dea1da0e0c8dc71480bd86f83d91e4949d80ad0cbb +size 227558 diff --git a/test/testoutput/bufr_sfcshp.nc b/test/testoutput/bufr_sfcshp.nc new file mode 100644 index 000000000..a22589020 --- /dev/null +++ b/test/testoutput/bufr_sfcshp.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b22b5a6a471fe7f40a45f70f82368798e57482dee2a9f72a0dfa7ed0453ae38 +size 484030 diff --git a/test/testoutput/bufr_simple_groupby.nc b/test/testoutput/bufr_simple_groupby.nc new file mode 100644 index 000000000..cb27ee167 --- /dev/null +++ b/test/testoutput/bufr_simple_groupby.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9450b5569481b60f7408568e41174b33926e7777ad43ebb338eea5025bb5bcb8 +size 48992 diff --git a/test/testoutput/bufr_specifying_subsets.nc b/test/testoutput/bufr_specifying_subsets.nc new file mode 100644 index 000000000..ecb61585f --- /dev/null +++ b/test/testoutput/bufr_specifying_subsets.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f3b86786b014bf39de2abaa815688ee982a32aadebe50149ea016acadf5646f +size 24062 diff --git a/test/testoutput/buoy_wmo_multi2.nc b/test/testoutput/buoy_wmo_multi2.nc index 515e563b0..2f608b947 100644 --- a/test/testoutput/buoy_wmo_multi2.nc +++ b/test/testoutput/buoy_wmo_multi2.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:11aba523924556f753c9e0104adc2615f7095acd57dcd49f35f4c6da6938a78a -size 31455 +oid sha256:ca24658dd52d6bd16010582d4219987af9b6bd010192871a4dd6395bf9524e01 +size 31453 diff --git a/test/testoutput/gdas.t00z.sevcsr.tm00.nc b/test/testoutput/gdas.t00z.sevcsr.tm00.nc index 97a9aeeb8..4cb60869a 100644 --- a/test/testoutput/gdas.t00z.sevcsr.tm00.nc +++ b/test/testoutput/gdas.t00z.sevcsr.tm00.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:eccb8a1979989132422bd72e96594f30b608a15d059317dbb137a8fec0d8f7e8 -size 241049 +oid sha256:8f323a107a1d9758c0222249aafe80f00eef792ced712786074b4381766cab75 +size 56554 diff --git a/test/testoutput/gdas.t06z.adpsfc_snow.tm00.nc b/test/testoutput/gdas.t06z.adpsfc_snow.tm00.nc index 309dcb543..2f92a645d 100644 --- a/test/testoutput/gdas.t06z.adpsfc_snow.tm00.nc +++ b/test/testoutput/gdas.t06z.adpsfc_snow.tm00.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1c39880462c073e30fd1acc7a351b5df3b782b87ae348598f89c9224eefc52e4 -size 771429 +oid sha256:7d28e3a6cefe1d5a9cc07e84d6c9db61f27ade0648600083d793b41c777a5264 +size 3986529 diff --git a/test/testoutput/gdas.t12z.1bamua.metop-c.tm00.nc b/test/testoutput/gdas.t12z.1bamua.metop-c.tm00.nc new file mode 100644 index 000000000..b58fed637 --- /dev/null +++ b/test/testoutput/gdas.t12z.1bamua.metop-c.tm00.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0d644d49291e31ff7b63f90e4e35edcea50cf3f8bcb384f3ebaaf552af44d3c +size 101448 diff --git a/test/testoutput/gdas.t12z.1bamua.noaa-15.tm00.nc b/test/testoutput/gdas.t12z.1bamua.noaa-15.tm00.nc new file mode 100644 index 000000000..92b65a378 --- /dev/null +++ b/test/testoutput/gdas.t12z.1bamua.noaa-15.tm00.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:301b039f7c7c3289f4d18e61c53d7b93918f0034eac23289b3c5dd0dd15767a5 +size 88731 diff --git a/test/testoutput/gdas.t12z.1bmhs.metop-b.tm00.nc b/test/testoutput/gdas.t12z.1bmhs.metop-b.tm00.nc new file mode 100644 index 000000000..8bd59bbe6 --- /dev/null +++ b/test/testoutput/gdas.t12z.1bmhs.metop-b.tm00.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a179d4371fe9af7779d20ec66ba35202d2e2e4d175c7a2402f9d1c82d01b677f +size 158092 diff --git a/test/testoutput/gdas.t12z.adpsfc.prepbufr.nc b/test/testoutput/gdas.t12z.adpsfc.prepbufr.nc new file mode 100644 index 000000000..b2769670e --- /dev/null +++ b/test/testoutput/gdas.t12z.adpsfc.prepbufr.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af465f2d17058f406a51242dcb3c6ba506532d6643061a7c1e54654e5b66cdf7 +size 284915 diff --git a/test/testoutput/gdas.t12z.adpsfc.tm00.nc b/test/testoutput/gdas.t12z.adpsfc.tm00.nc new file mode 100644 index 000000000..9db35373a --- /dev/null +++ b/test/testoutput/gdas.t12z.adpsfc.tm00.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f15b99c2ba2c41fea253e56aa4a289696966453a265c634dd0a8cd40d34a54eb +size 328237 diff --git a/test/testoutput/gdas.t12z.esamua.noaa-18.tm00.nc b/test/testoutput/gdas.t12z.esamua.noaa-18.tm00.nc new file mode 100644 index 000000000..9219b4b53 --- /dev/null +++ b/test/testoutput/gdas.t12z.esamua.noaa-18.tm00.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22ea0481d12249440b7041c4d95d3786f055d2ceb4fcd0ecfc73d98217877c20 +size 72442 diff --git a/test/testoutput/gdas.t12z.esmhs.noaa-19.tm00.nc b/test/testoutput/gdas.t12z.esmhs.noaa-19.tm00.nc new file mode 100644 index 000000000..1ac9f1efc --- /dev/null +++ b/test/testoutput/gdas.t12z.esmhs.noaa-19.tm00.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a806f9d58bca1684a96a846b76aa18b1d44d01941591412aec28a3d6d94fd3b +size 158064 diff --git a/test/testoutput/gdas.t12z.mtiasi.metop-c.tm00.nc b/test/testoutput/gdas.t12z.mtiasi.metop-c.tm00.nc new file mode 100644 index 000000000..d9f111e3b --- /dev/null +++ b/test/testoutput/gdas.t12z.mtiasi.metop-c.tm00.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ab3e5c11041d73a2d56748f6e72b57df3cf4a2a67ff80287dac40a012846735 +size 545926 diff --git a/test/testoutput/gdas.t18z.1bmhs.tm00.15.7.filter_split.nc b/test/testoutput/gdas.t18z.1bmhs.tm00.15.7.filter_split.nc index 18b4a672c..0d3db7f22 100644 --- a/test/testoutput/gdas.t18z.1bmhs.tm00.15.7.filter_split.nc +++ b/test/testoutput/gdas.t18z.1bmhs.tm00.15.7.filter_split.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:27449e450a13d8041440dc33402561c28fbdeb0154e6a38d15795209afb83d4e -size 57851 +oid sha256:fb2278b9c7b894e243efd7c3705c59ec0b29e20dfafe6c516c33ed6438b83ab5 +size 57916 diff --git a/test/testoutput/gdas.t18z.1bmhs.tm00.15.seven.split.nc b/test/testoutput/gdas.t18z.1bmhs.tm00.15.seven.split.nc index cfe9f7ee2..4b6787fb1 100644 --- a/test/testoutput/gdas.t18z.1bmhs.tm00.15.seven.split.nc +++ b/test/testoutput/gdas.t18z.1bmhs.tm00.15.seven.split.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f4398180126b1b335f4e53a3a48b55bfeac3c239db26e16572427eb99b690fd5 -size 69208 +oid sha256:cb80523052bef9bcd45f1c33823e013de8ad20c49f6db9d7e3606fef262ef0c1 +size 69271 diff --git a/test/testoutput/gdas.t18z.1bmhs.tm00.filtering.nc b/test/testoutput/gdas.t18z.1bmhs.tm00.filtering.nc index e4a4458a3..23cbb369e 100644 --- a/test/testoutput/gdas.t18z.1bmhs.tm00.filtering.nc +++ b/test/testoutput/gdas.t18z.1bmhs.tm00.filtering.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:884416e6e3481bd2567e879464e47c7fdca358b9e1d0d92837370644b0caa15a -size 103704 +oid sha256:dd17602a8ce81bf471912b8b60cf8e59c2afccd78369de6302c39154f6c6b76d +size 103769 diff --git a/test/testoutput/gdas.t18z.1bmhs.tm00.nc b/test/testoutput/gdas.t18z.1bmhs.tm00.nc index de3dcbb01..41e8fa886 100644 --- a/test/testoutput/gdas.t18z.1bmhs.tm00.nc +++ b/test/testoutput/gdas.t18z.1bmhs.tm00.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b5c1d942ec9f3d4c734a0770d0f502f0bda74c45c9b31ff8517742edf4f17329 -size 799977 +oid sha256:b8dde16bfd27b73a4777f48ddbaa76b86c32d82994c57091bf2f6eb4269192a1 +size 798784 diff --git a/test/testoutput/gdas.t18z.satwnd_avhrr.tm00.nc b/test/testoutput/gdas.t18z.satwnd_avhrr.tm00.nc new file mode 100644 index 000000000..2ba5ef90c --- /dev/null +++ b/test/testoutput/gdas.t18z.satwnd_avhrr.tm00.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bcdae0bb6086ffc572a9fcca8c1d709d40e4780cc07b55e6bf85547dc87c97d +size 300447 diff --git a/test/testoutput/iasi_metop-b_obs_2020110112.nc4 b/test/testoutput/iasi_metop-b_obs_2020110112.nc4 new file mode 100644 index 000000000..248e89491 --- /dev/null +++ b/test/testoutput/iasi_metop-b_obs_2020110112.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:119a6aaccda6a9dce42ac5adf928f6ea719925a205885678aa9511a0df8cd9cf +size 3911239 diff --git a/test/testoutput/iasi_metop-b_obs_2022021912.nc4 b/test/testoutput/iasi_metop-b_obs_2022021912.nc4 new file mode 100644 index 000000000..d7d3c4f0d --- /dev/null +++ b/test/testoutput/iasi_metop-b_obs_2022021912.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0cb3c3fba2fc310b7185b7738a937a7fbf719c185d33ac4d692d9005aca648d +size 3235339 diff --git a/test/testoutput/mhs_metop-b_obs_2020110112.nc4 b/test/testoutput/mhs_metop-b_obs_2020110112.nc4 new file mode 100644 index 000000000..2e9422145 --- /dev/null +++ b/test/testoutput/mhs_metop-b_obs_2020110112.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d5b23169f28aa6d2b5a025f02be089b2c75a57ac66e8fbf243f22731d1cac44 +size 1182581 diff --git a/test/testoutput/mhs_n19_obs_2020110112.nc4 b/test/testoutput/mhs_n19_obs_2020110112.nc4 new file mode 100644 index 000000000..8cd2f1573 --- /dev/null +++ b/test/testoutput/mhs_n19_obs_2020110112.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f2080ced0d528f44be98d66382688e6ff42129c8446abd7e61ef203e898bf3 +size 606573 diff --git a/test/testoutput/mopitt_co.nc b/test/testoutput/mopitt_co.nc index 1390a3b32..33cccafeb 100644 --- a/test/testoutput/mopitt_co.nc +++ b/test/testoutput/mopitt_co.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:26fbf37673b15ccb4d4d9a3e64136cb3134bd34cb0b01366375c06b60a1e4edb -size 209576 +oid sha256:5a5703deff298e440fb7f4349a4d8585335d16b3d9b9e0e141f4927ac3f42ab4 +size 171846 diff --git a/test/testoutput/nsidc_l4_icec.nc b/test/testoutput/nsidc_l4_icec.nc new file mode 100644 index 000000000..3960e6353 --- /dev/null +++ b/test/testoutput/nsidc_l4_icec.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:401d3b02f5287dd6eb7a8d2904770385fece53f499720183e414a5cfa3035695 +size 133049 diff --git a/test/testoutput/pace_radiance_L1B.nc b/test/testoutput/pace_radiance_L1B.nc new file mode 100644 index 000000000..2ba8e349d --- /dev/null +++ b/test/testoutput/pace_radiance_L1B.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca2da8d7caae002d9e6e3f4d571341430daf063ff5aec6aa0b4d242932731205 +size 92888 diff --git a/test/testoutput/satbias_gmi_gpm.nc4 b/test/testoutput/satbias_gmi_gpm.nc4 new file mode 100644 index 000000000..730bf3129 --- /dev/null +++ b/test/testoutput/satbias_gmi_gpm.nc4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbde42fcf6cb016c12b12a798672888c71a9397b39540bfa503022fa3d11131c +size 9806 diff --git a/test/testoutput/ship_wmo_multi2.nc b/test/testoutput/ship_wmo_multi2.nc index daec4cb48..b929fd3f9 100644 --- a/test/testoutput/ship_wmo_multi2.nc +++ b/test/testoutput/ship_wmo_multi2.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:397dfc682868e72c2bf5ea9304428a207bce98c88fa3d511f592e40fa7e10300 -size 132435 +oid sha256:afffe0c1e257550feed5c4d61508b73455054c8a02242deef5ab8e2be9636f9f +size 139331 diff --git a/test/testoutput/sonde_wmo_multi.nc b/test/testoutput/sonde_wmo_multi.nc index 440d5f6b5..a017fcf4f 100644 --- a/test/testoutput/sonde_wmo_multi.nc +++ b/test/testoutput/sonde_wmo_multi.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:20f491f8ee4056e83470e06a8043f77cc8516b7afbd60b6d7ed6d83651aaa27a -size 340266 +oid sha256:f44ea5a22aed43389b02a26900d22baa5aa164a77004db34ea3d07f9faf5b4ad +size 405222 diff --git a/test/testoutput/sst_ostia.nc b/test/testoutput/sst_ostia.nc index cc0b62c0b..878dc0317 100644 --- a/test/testoutput/sst_ostia.nc +++ b/test/testoutput/sst_ostia.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8dd373cb95217395787e4821f957e4a5fd7fea8be351cc78fe0167600dd04f88 -size 468061 +oid sha256:0ea0643adb5427b5ad951ba8ec17db74dd7f671fdbfa23ec281b574f4b322500 +size 89386 diff --git a/test/testoutput/synop_wmo_multi2.nc b/test/testoutput/synop_wmo_multi2.nc index 80213513c..3dd156efb 100644 --- a/test/testoutput/synop_wmo_multi2.nc +++ b/test/testoutput/synop_wmo_multi2.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1392534f99d02b7822f0ed5bfd3f8fccd9da41914e3b10d58f40dee7b2e2b2a3 -size 131487 +oid sha256:a2f2b360e707c8bd8ca0d4f6f65c6ca5b73382baaacf73b148a25dc31bebfc52 +size 104167 diff --git a/test/testoutput/tropomi_no2.nc b/test/testoutput/tropomi_no2.nc deleted file mode 100644 index 2d4f1f2d4..000000000 --- a/test/testoutput/tropomi_no2.nc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f04e576a00685f8828643903b85527b68b8a54eacd1c7a1072e0ca242b69a7c6 -size 226603 diff --git a/test/testoutput/tropomi_no2_total.nc b/test/testoutput/tropomi_no2_total.nc new file mode 100644 index 000000000..1ffff379e --- /dev/null +++ b/test/testoutput/tropomi_no2_total.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:235c758180174ff347c9a47b489edd8a8ae84841a17e05aa91e0f4c13145ae26 +size 104443 diff --git a/test/testoutput/tropomi_no2_tropo.nc b/test/testoutput/tropomi_no2_tropo.nc new file mode 100644 index 000000000..cc9c2507c --- /dev/null +++ b/test/testoutput/tropomi_no2_tropo.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e3a30f2acb6a0ecb37314c6850a0dc1e45290c7fad7ae7b9c4971712e47ff0 +size 96238 diff --git a/test/testoutput/wmo_raob_double.nc4 b/test/testoutput/wmo_raob_double.nc4 deleted file mode 100644 index 61effcec7..000000000 --- a/test/testoutput/wmo_raob_double.nc4 +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eea9a73d8a2d05253aaa1c26255f2ba7c099d27f1c9ed75a06ccca24fffc320e -size 185980 diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 0d66fb9b8..f02da5406 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -11,3 +11,4 @@ set_targets_deps( "${programs}" add_subdirectory( fortran ) add_subdirectory( run_satwnds ) +add_subdirectory( bufr ) diff --git a/tools/bufr/CMakeLists.txt b/tools/bufr/CMakeLists.txt new file mode 100644 index 000000000..9c110d879 --- /dev/null +++ b/tools/bufr/CMakeLists.txt @@ -0,0 +1,16 @@ + + +list(APPEND _bufr_deps + eckit + bufr::bufr_d) + +list(APPEND _srcs + print_queries.cpp + ../../src/bufr/BufrParser/Query/DataProvider.h + ../../src/bufr/BufrParser/Query/DataProvider.cpp + ../../src/bufr/BufrParser/Query/SubsetTable.h + ../../src/bufr/BufrParser/Query/SubsetTable.cpp) + +ecbuild_add_executable( TARGET print_queries.x + SOURCES ${_srcs} + LIBS ${_bufr_deps}) diff --git a/tools/bufr/print_queries.cpp b/tools/bufr/print_queries.cpp new file mode 100644 index 000000000..0dd25d675 --- /dev/null +++ b/tools/bufr/print_queries.cpp @@ -0,0 +1,356 @@ + +#include +#include +#include +#include +#include +#include +#include + + +#include "../../src/bufr/BufrParser/Query/DataProvider.h" +#include "../../src/bufr/BufrParser/Query/SubsetTable.h" + +#include "bufr_interface.h" + + +std::set getSubsets(int fileUnit) +{ + static const int SubsetLen = 9; + int iddate; + + std::set subsets; + + char subset[SubsetLen]; + while (ireadmg_f(fileUnit, subset, &iddate, SubsetLen) == 0) + { + auto str_subset = std::string(subset); + str_subset.erase( + remove_if(str_subset.begin(), str_subset.end(), isspace), str_subset.end()); + subsets.insert(str_subset); + } + + return subsets; +} + + +std::vector> +getDimPaths(const std::vector& queryData) +{ + std::map> dimPathMap; + for (auto& query : queryData) + { + std::stringstream pathStream; + pathStream << "*"; + for (size_t idx=1; idx <= query.dimIdxs.back(); idx++) + { + pathStream << "/" << query.pathComponents[idx]; + } + + dimPathMap[pathStream.str()] = + std::make_pair(query.dimIdxs.size(), + pathStream.str()); + } + + std::vector> result; + for (auto& dimPath : dimPathMap) + { + result.push_back(dimPath.second); + } + + return result; +} + + +std::vector getQueries(int fileUnit, + const std::string& subset, + Ingester::bufr::DataProvider& dataProvider) +{ + static const int SubsetLen = 9; + + int iddate; + int bufrLoc; + int il, im; // throw away + char current_subset[9]; + bool subsetFound = false; + + std::vector queryData; + + while (ireadmg_f(fileUnit, current_subset, &iddate, SubsetLen) == 0) + { + auto msg_subset = std::string(current_subset); + msg_subset.erase( + remove_if(msg_subset.begin(), msg_subset.end(), isspace), msg_subset.end()); + + status_f(fileUnit, &bufrLoc, &il, &im); + dataProvider.updateData(bufrLoc); + + if (msg_subset == subset) + { + while (ireadsb_f(fileUnit) == 0) + { + status_f(fileUnit, &bufrLoc, &il, &im); + dataProvider.updateData(bufrLoc); + queryData = Ingester::bufr::SubsetTable(dataProvider).allQueryData(); + subsetFound = true; + } + } + + if (subsetFound) break; + } + + return queryData; +} + + +void printHelp() +{ + std::cout << "Description: " << std::endl; + std::cout << " Lists all the queries possible on a BUFR file per subset." << std::endl; + std::cout << "Arguments: " << std::endl; + std::cout << " -h (Optional) Print out the help message." << std::endl; + std::cout << " -s (Optional) Print paths only for this subset." << std::endl; + std::cout << " input_file Path to the BUFR file." << std::endl; + std::cout << " output_file (Optional) Save the output. " << std::endl; + std::cout << "Examples: " << std::endl; + std::cout << " ./print_queries.x ../data/bufr_satwnd_old_format.bufr" << std::endl; + std::cout << " ./print_queries.x -s NC005066 ../data/bufr_satwnd_old_format.bufr" << std::endl; +} + +std::string dimStyledStr(int dims) +{ + std::ostringstream ostr; + ostr << dims << "d"; + + return ostr.str(); +} + +std::string typeStyledStr(const Ingester::bufr::TypeInfo& info) +{ + std::string typeStr; + + if (info.isString()) + { + typeStr = "string"; + } + else if (info.isInteger()) + { + if (info.isSigned()) + { + if (info.is64Bit()) + { + typeStr = "int64 "; + } + else + { + typeStr = "int "; + } + } + else + { + if (info.is64Bit()) + { + typeStr = "uint64"; + } + else + { + typeStr = "uint "; + } + } + } + else + { + if (info.is64Bit()) + { + typeStr = "double"; + } + else + { + typeStr = "float "; + } + } + + return typeStr; +} + +void printDimPaths(std::vector> dimPaths) +{ + for (auto& dimPath : dimPaths) + { + std::cout << " " << dimPath.first << "d " << dimPath.second << std::endl; + } +} + +void printQueryList(const std::vector& queries) +{ + for (auto query : queries) + { + std::ostringstream ostr; + ostr << dimStyledStr(query.dimIdxs.size()) << " "; + ostr << typeStyledStr(query.typeInfo) << " "; + ostr << query.pathComponents[0]; + for (size_t pathIdx = 1; pathIdx < query.pathComponents.size(); pathIdx++) + { + if (std::find(query.dimIdxs.begin(), query.dimIdxs.end(), pathIdx) != query.dimIdxs.end()) + { + ostr << "/" << query.pathComponents[pathIdx]; + } + else + { + ostr << "/" << query.pathComponents[pathIdx]; + } + } + + if (query.requiresIdx) + { + ostr << "[" << query.idx << "]"; + } + + std::cout << " " << ostr.str() << std::endl; + } +} + +void printQueries(const std::string& filePath, + const std::string& subset, + const std::string& tablePath) +{ + const static int FileUnit = 12; + const static int FileUnitTable1 = 13; + const static int FileUnitTable2 = 14; + + open_f(FileUnit, filePath.c_str()); + + if (tablePath.empty()) + { + openbf_f(FileUnit, "IN", FileUnit); + } + else + { + openbf_f(FileUnit, "SEC3", FileUnit); + mtinfo_f(tablePath.c_str(), FileUnitTable1, FileUnitTable2); + } + + auto dataProvider = Ingester::bufr::DataProvider(FileUnit); + if (!subset.empty()) + { + auto queries = getQueries(FileUnit, subset.c_str(), dataProvider); + std::cout << subset << std::endl; + std::cout << " Dimensioning Sub-paths: " << std::endl; + printDimPaths(getDimPaths(queries)); + std::cout << std::endl; + std::cout << " Queries: " << std::endl; + printQueryList(queries); + std::cout << std::endl; + } + else + { + auto subsets = getSubsets(FileUnit); + + if (subsets.empty()) + { + std::cerr << "No BUFR subsets found in " << filePath << std::endl; + exit(1); + } + + std::cout << "Available subsets: " << std::endl; + for (auto subset : subsets) + { + std::cout << subset << std::endl; + } + std::cout << "Total number of subsets found: " << subsets.size() << std::endl << std::endl; + + + for (auto subset : subsets) + { + closbf_f(FileUnit); + close_f(FileUnit); + + open_f(FileUnit, filePath.c_str()); + + if (tablePath.empty()) + { + openbf_f(FileUnit, "IN", FileUnit); + } + else + { + openbf_f(FileUnit, "SEC3", FileUnit); + mtinfo_f(tablePath.c_str(), FileUnitTable1, FileUnitTable2); + } + + auto queries = getQueries(FileUnit, subset.c_str(), dataProvider); + + std::cout << subset << std::endl; + std::cout << " Dimensioning Sub-paths: " << std::endl; + printDimPaths(getDimPaths(queries)); + std::cout << std::endl; + std::cout << " Queries: " << std::endl; + printQueryList(queries); + std::cout << std::endl; + } + } + + closbf_f(FileUnit); + close_f(FileUnit); +} + + +int main(int argc, char** argv) +{ + std::string inputFile = ""; + std::string tablePath = ""; + std::string subset = ""; + + int idx = 1; + while (idx < argc) + { + std::string arg = argv[idx]; + if (arg.substr(0,2) == "-s") + { + if (arg.size() == 2) + { + subset = argv[idx+1]; + idx = idx + 2; + } + else + { + subset = arg.substr(2, arg.size()); + idx++; + } + } + else if (arg == "-h") + { + printHelp(); + exit(0); + } + else if (arg == "-t") + { + tablePath = std::string(argv[idx + 1]); + idx = idx + 2; + } + else + { + inputFile = arg; + idx++; + } + } + + if (inputFile.empty()) + { + printHelp(); + std::cerr << "Error: no input file specified" << std::endl; + exit(1); + } + + try + { + printQueries(inputFile, subset, tablePath); + } + catch (const std::exception &e) + { + throw; + } + + return 0; +} + + diff --git a/tools/fortran/CMakeLists.txt b/tools/fortran/CMakeLists.txt index f1a07a98b..7d4925886 100644 --- a/tools/fortran/CMakeLists.txt +++ b/tools/fortran/CMakeLists.txt @@ -2,9 +2,9 @@ if( iodaconv_bufr_ENABLED ) ecbuild_add_executable( TARGET pb_decode SOURCES pb_decode.f90 - LIBS bufr::bufr_4) + LIBS bufr::bufr_d) ecbuild_add_executable( TARGET pb_decode_events SOURCES pb_decode_events.f90 - LIBS bufr::bufr_4) + LIBS bufr::bufr_d) endif()