diff --git a/.env b/.env index dca75cc..c3e700d 100644 --- a/.env +++ b/.env @@ -1,5 +1,6 @@ COMET_API_KEY= COMET_REST_API_KEY=Wnh3NkQ1aWVNb21XNk1wclM2R28zdDJjQQ== +HDF5_DIR=${CONDA_PREFIX} LD_LIBRARY_PATH=${CONDA_PREFIX}/lib/ GMT_LIBRARY_PATH=${CONDA_PREFIX}/lib/ JUPYTER_CONFIG_DIR=${PWD}/.jupyter diff --git a/.github/main.workflow b/.github/main.workflow deleted file mode 100644 index 889a06a..0000000 --- a/.github/main.workflow +++ /dev/null @@ -1,15 +0,0 @@ -workflow "Build and Test DeepBedMap" { - resolves = ["Run Tests"] - on = "push" -} - -action "Build DeepBedMap App" { - uses = "actions/docker/cli@86ff551d26008267bb89ac11198ba7f1d807b699" - args = "build --file Dockerfile --tag weiji14/deepbedmap --target app ." -} - -action "Run Tests" { - uses = "actions/docker/cli@86ff551d26008267bb89ac11198ba7f1d807b699" - args = "run weiji14/deepbedmap python -m pytest --verbose --disable-warnings --nbval test_ipynb.ipynb" - needs = ["Build DeepBedMap App"] -} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..c3a3576 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,11 @@ +on: push +name: Build and Test DeepBedMap +jobs: + buildDeepBedMapApp: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@master + - name: Build DeepBedMap App + run: docker build --file Dockerfile --tag weiji14/deepbedmap --target app . + - name: Run Tests + run: docker run weiji14/deepbedmap python -m pytest --verbose --disable-warnings --nbval test_ipynb.ipynb diff --git a/Dockerfile b/Dockerfile index f91ee2d..757419e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,7 @@ RUN conda env update -n base -f environment.yml && \ # Install dependencies in Pipfile.lock using pipenv COPY Pipfile* ${HOME}/ RUN conda activate base && \ + export HDF5_DIR=${CONDA_PREFIX} && \ export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib && \ pipenv install --python ${CONDA_PREFIX}/bin/python --dev --deploy && \ rm --recursive ${HOME}/.cache/pip* && \ diff --git a/Pipfile b/Pipfile index 2bf789f..6446332 100644 --- a/Pipfile +++ b/Pipfile @@ -18,7 +18,7 @@ livelossplot = "==0.4.1" matplotlib = "==3.1.1" netcdf4 = "==1.4.1" numpy = "==1.17.0rc2" -onnx_chainer = "==1.4.1" +onnx_chainer = "==1.5.0" optuna = "==0.13.0" pandas = "==0.25.0" pygmt = {editable = true, ref = "0.0.1a0-40-g29220bd", git = "https://github.com/weiji14/pygmt.git"} @@ -35,7 +35,7 @@ nbval = "==0.9.2" pytest = "==5.0.1" [requires] -python_version = "3.6" +python_version = "3.7" [pipenv] allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock index ece35a5..f3d4e90 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "1a76f415863c6333c7d7d47c0d3868276cf972105bd4f4b9368ef3c3cf95a6c2" + "sha256": "5857fce0ea85523b7e6eb5f1c59a2d9af8c8c771a2ce93a640419bce5e3bf52c" }, "pipfile-spec": 6, "requires": { - "python_version": "3.6" + "python_version": "3.7" }, "sources": [ { @@ -148,11 +148,11 @@ }, "cmd2": { "hashes": [ - "sha256:2361cb0448e2bf94a9cd9bd630f80f53032d62c3afda82fc8ead270409e39466", - "sha256:a57d80a0de678a927bd774e3c13e396fd45418547bc9e9b587a9c3e8cb43227e" + "sha256:4b78379d53aff811d1deac720bbe71661769822a5fb2d830cd730656d180fb3d", + "sha256:afebd649dda771b1de893e8df0a5f6085da5f0dcf7f63e18b247b3284f9365ce" ], "markers": "python_version >= '3.0'", - "version": "==0.9.14" + "version": "==0.9.16" }, "colorama": { "hashes": [ @@ -332,17 +332,17 @@ }, "ipykernel": { "hashes": [ - "sha256:346189536b88859937b5f4848a6fd85d1ad0729f01724a411de5cae9b618819c", - "sha256:f0e962052718068ad3b1d8bcc703794660858f58803c3798628817f492a8769c" + "sha256:167c3ef08450f5e060b76c749905acb0e0fbef9365899377a4a1eae728864383", + "sha256:b503913e0b4cce7ed2de965457dfb2edd633e8234161a60e23f2fe2161345d12" ], - "version": "==5.1.1" + "version": "==5.1.2" }, "ipython": { "hashes": [ - "sha256:11067ab11d98b1e6c7f0993506f7a5f8a91af420f7e82be6575fcb7a6ca372a0", - "sha256:60bc55c2c1d287161191cc2469e73c116d9b634cff25fe214a43cba7cec94c79" + "sha256:1d3a1692921e932751bc1a1f7bb96dc38671eeefdc66ed33ee4cbc57e92a410e", + "sha256:537cd0176ff6abd06ef3e23f2d0c4c2c8a4d9277b7451544c6cbf56d1c79a83d" ], - "version": "==7.6.1" + "version": "==7.7.0" }, "ipython-genutils": { "hashes": [ @@ -353,10 +353,10 @@ }, "jedi": { "hashes": [ - "sha256:53c850f1a7d3cfcd306cc513e2450a54bdf5cacd7604b74e42dd1f0758eaaf36", - "sha256:e07457174ef7cb2342ff94fa56484fe41cec7ef69b0059f01d3f812379cb6f7c" + "sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27", + "sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e" ], - "version": "==0.14.1" + "version": "==0.15.1" }, "jinja2": { "hashes": [ @@ -382,10 +382,10 @@ }, "jsonschema": { "hashes": [ - "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", - "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" + "sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", + "sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" ], - "version": "==3.0.1" + "version": "==3.0.2" }, "jupyter-client": { "hashes": [ @@ -466,9 +466,9 @@ }, "mako": { "hashes": [ - "sha256:f5a642d8c5699269ab62a68b296ff990767eb120f51e2e8f3d6afb16bdb57f4b" + "sha256:a36919599a9b7dc5d86a7a8988f23a9a3a3d083070023bab23d64f7f1d1e0a4b" ], - "version": "==1.0.14" + "version": "==1.1.0" }, "markupsafe": { "hashes": [ @@ -533,10 +533,10 @@ }, "nbconvert": { "hashes": [ - "sha256:138381baa41d83584459b5cfecfc38c800ccf1f37d9ddd0bd440783346a4c39c", - "sha256:4a978548d8383f6b2cfca4a3b0543afb77bc7cb5a96e8b424337ab58c12da9bc" + "sha256:427a468ec26e7d68a529b95f578d5cbf018cb4c1f889e897681c2b6d11897695", + "sha256:48d3c342057a2cf21e8df820d49ff27ab9f25fc72b8f15606bd47967333b2709" ], - "version": "==5.5.0" + "version": "==5.6.0" }, "nbformat": { "hashes": [ @@ -646,27 +646,31 @@ }, "onnx": { "hashes": [ - "sha256:05e169c86a8e8e0af065265ee29845a8272ef3600e13b80e5cbb784a6b6c4ad0", - "sha256:1e024323293b06071db6aadaae922e2521c6b6dad63d86348d61e2150433172a", - "sha256:21acafa512cea1b51c0891a80eb41a498ccc7994241f790a878e90369e3d6f44", - "sha256:43ec60046a335df667d691cf1e1afbb376620c69a334c8b3b0574ebdbb5f67ce", - "sha256:50f3cc33358685428ff7ab23042c2eef1d73fb58c38c7cc9b0a1b22af9086ee9", - "sha256:93efb54934d877b55903c460050fb275bd6376095a6a365f7061859df6f5c7ef", - "sha256:a17ecbfab19bf5eb88819f11161241c22ec94fdc74a10a50d67896ffb95a62c5", - "sha256:a818458e9cb0186136dd6edfe403a39472458f100c33fc2fa5784080ae5e4738", - "sha256:b0fb57c853420c44f099919e044b527dfeff2ac59d9f8fdc2ae19979b3bc5e28", - "sha256:b93d2cb902978942ae0f98aabe5972cd511bbd1511df1a7bb98fca6d3a63e23e", - "sha256:cfe22d41e80c0095d11ffa848fa09b0a424eae7841d1605edaf276b6926cfc4c", - "sha256:e52522212247c5573ce6ba736ca29342e3899e1042ad65ac3c132d4c97d60351" + "sha256:144c6812b5650e3148af607798f37009e3fda80d53222ce07fcdca94e31e1634", + "sha256:1a584a4ef62a6db178c257fffb06a9d8e61b41c0a80bfd8bcd8a253d72c4b0b4", + "sha256:352ce8ab166e3d7760bb02eff3b9b0670fbc4123229b547b25b428ff87f3af6f", + "sha256:37ace5a7a0c47d2a79edd83861eff17f6e463eafe96c8d930624bc51ca1023f6", + "sha256:3c87b805046b47b6e8f8b58aec00aa3fbc54923fecc1ec83f9fe20d9568a49f6", + "sha256:4c24e30c97b46034c932114f0276b26ac5f7bc1a98638beff988ae91dcd6ede2", + "sha256:660bdaabe27d523d9b2714ae8ce0b45a64dcb324da666e9f64cae1960600d739", + "sha256:8343fec593bb9e84aa41b666c86b00d1c6932bbb6be2ae02e646b4a8396f74e0", + "sha256:9223e3a06fbfac670966beb3206907b3a4c7eb6a9e15517979e08235049ee4c8", + "sha256:9fc6b80aa641428adacce445a479e92cfb5c418166b7af4ac68daac0d9900317", + "sha256:b01f506c85094bf41ddab83f926530d42d5c4a9ab72e56e78b41e2df7b2f87e2", + "sha256:c732490c07acece6dfa1152f74f0df3d0e625564c7f3c5e39be44e87f83c12e6", + "sha256:d5a6d8f922734704951367048307abbc41414b6a781f239193898b492c6f01e7", + "sha256:e0226ed768836de0ac5bbfd773da832c47368fe21c0bf0cea724fbe4cbaa7acd", + "sha256:f4ab108ed69819926118c54e0db85b831b66ec8ec28eb1931eef7c5bcaf82452", + "sha256:f6202b2b5e353f98cba793cc52d449e578cc242968e89a4c6cfc73fa2591e926" ], - "version": "==1.4.1" + "version": "==1.5.0" }, "onnx-chainer": { "hashes": [ - "sha256:8dff6aee68db4acd8af1153c11872d0ed42e59af1352f5c48f51b9d49e7e1b2f" + "sha256:3a92d77770cabfd912f43fe9e47ca25df2fecba1f510c75afc357a3b6657af9f" ], "index": "pypi", - "version": "==1.4.1" + "version": "==1.5.0" }, "optuna": { "hashes": [ @@ -677,10 +681,10 @@ }, "packaging": { "hashes": [ - "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", - "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" + "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", + "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" ], - "version": "==19.0" + "version": "==19.1" }, "pandas": { "hashes": [ @@ -717,10 +721,10 @@ }, "pbr": { "hashes": [ - "sha256:0ca44dc9fd3b04a22297c2a91082d8df2894862e8f4c86a49dac69eae9e85ca0", - "sha256:4aed6c1b1fa5020def0f22aed663d87b81bb3235f112490b07d2643d7a98c5b5" + "sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc", + "sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf" ], - "version": "==5.4.1" + "version": "==5.4.2" }, "pexpect": { "hashes": [ @@ -854,10 +858,10 @@ }, "pyparsing": { "hashes": [ - "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", - "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "version": "==2.4.0" + "version": "==2.4.2" }, "pyperclip": { "hashes": [ @@ -892,9 +896,9 @@ }, "pyrsistent": { "hashes": [ - "sha256:50cffebc87ca91b9d4be2dcc2e479272bcb466b5a0487b6c271f7ddea6917e14" + "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" ], - "version": "==0.15.3" + "version": "==0.15.4" }, "python-dateutil": { "hashes": [ @@ -913,10 +917,10 @@ }, "pytz": { "hashes": [ - "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", - "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" + "sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", + "sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7" ], - "version": "==2019.1" + "version": "==2019.2" }, "pywavelets": { "hashes": [ @@ -951,49 +955,51 @@ }, "pyyaml": { "hashes": [ - "sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", - "sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", - "sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", - "sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", - "sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", - "sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", - "sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", - "sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", - "sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", - "sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", - "sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" - ], - "version": "==5.1.1" + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "version": "==5.1.2" }, "pyzmq": { "hashes": [ - "sha256:00dd015159eaeb1c0731ad49310e1f5d839c9a35a15e4f3267f5052233fad99b", - "sha256:03913b6beb8e7b417b9910b0ee1fd5d62e9626d218faefbe879d70714ceab1a2", - "sha256:13f17386df81d5e6efb9a4faea341d8de22cdc82e49a326dded26e33f42a3112", - "sha256:16c6281d96885db1e15f7047ddc1a8f48ff4ea35d31ca709f4d2eb39f246d356", - "sha256:17efab4a804e31f58361631256d660214204046f9e2b962738b171b9ad674ea7", - "sha256:2b79919ddeff3d3c96aa6087c21d294c8db1c01f6bfeee73324944683685f419", - "sha256:2f832e4711657bb8d16ea1feba860f676ec5f14fb9fe3b449b5953a60e89edae", - "sha256:31a11d37ac73107363b47e14c94547dbfc6a550029c3fe0530be443199026fc2", - "sha256:33a3e928e6c3138c675e1d6702dd11f6b7050177d7aab3fc322db6e1d2274490", - "sha256:34a38195a6d3a9646cbcdaf8eb245b4d935c7a57f7e1b3af467814bc1a92467e", - "sha256:42900054f1500acef6df7428edf806abbf641bf92eb9ceded24aa863397c3bae", - "sha256:4ccc7f3c63aa9d744dadb62c49eda2d0e7de55649b80c45d7c684d70161a69af", - "sha256:5b220c37c346e6575db8c88a940c1fc234f99ce8e0068c408919bb8896c4b6d2", - "sha256:6074848da5c8b44a1ca40adf75cf65aa92bc80f635e8249aa8f37a69b2b9b6f5", - "sha256:61a4155964bd4a14ef95bf46cb1651bcf8dcbbed8c0108e9c974c1fcbb57788f", - "sha256:62b5774688326600c52f587f7a033ca6b6284bef4c8b1b5fda32480897759eac", - "sha256:65a9ffa4f9f085d696f16fd7541f34b3c357d25fe99c90e3bce2ea59c3b5b4b6", - "sha256:76a077d2c30f8adc5e919a55985a784b96aeca69b53c1ea6fd5723d3ae2e6f53", - "sha256:8e5b4c51557071d6379d6dc1f54f35e9f6a137f5e84e102efb869c8d3c13c8ff", - "sha256:917f73e07cc04f0678a96d93e7bb8b1adcccdde9ccfe202e622814f4d1d1ecfd", - "sha256:91c75d3c4c357f9643e739db9e79ab9681b2f6ae8ec5678d6ef2ea0d01532596", - "sha256:923dd91618b100bb4c92ab9ed7b65825a595b8524a094ce03c7cb2aaae7d353b", - "sha256:9849054e0355e2bc7f4668766a25517ba76095031c9ff5e39ae8949cee5bb024", - "sha256:c9d453933f0e3f44b9759189f2a18aa765f7f1a4345c727c18ebe8ad0d748d26", - "sha256:cb7514936277abce64c2f4c56883e5704d85ed04d98d2d432d1c6764003bb003" - ], - "version": "==18.0.2" + "sha256:01636e95a88d60118479041c6aaaaf5419c6485b7b1d37c9c4dd424b7b9f1121", + "sha256:021dba0d1436516092c624359e5da51472b11ba8edffa334218912f7e8b65467", + "sha256:0463bd941b6aead494d4035f7eebd70035293dd6caf8425993e85ad41de13fa3", + "sha256:05fd51edd81eed798fccafdd49c936b6c166ffae7b32482e4d6d6a2e196af4e6", + "sha256:1fadc8fbdf3d22753c36d4172169d184ee6654f8d6539e7af25029643363c490", + "sha256:22efa0596cf245a78a99060fe5682c4cd00c58bb7614271129215c889062db80", + "sha256:260c70b7c018905ec3659d0f04db735ac830fe27236e43b9dc0532cf7c9873ef", + "sha256:2762c45e289732d4450406cedca35a9d4d71e449131ba2f491e0bf473e3d2ff2", + "sha256:2fc6cada8dc53521c1189596f1898d45c5f68603194d3a6453d6db4b27f4e12e", + "sha256:343b9710a61f2b167673bea1974e70b5dccfe64b5ed10626798f08c1f7227e72", + "sha256:41bf96d5f554598a0632c3ec28e3026f1d6591a50f580df38eff0b8067efb9e7", + "sha256:856b2cdf7a1e2cbb84928e1e8db0ea4018709b39804103d3a409e5584f553f57", + "sha256:85b869abc894672de9aecdf032158ea8ad01e2f0c3b09ef60e3687fb79418096", + "sha256:93f44739db69234c013a16990e43db1aa0af3cf5a4b8b377d028ff24515fbeb3", + "sha256:98fa3e75ccb22c0dc99654e3dd9ff693b956861459e8c8e8734dd6247b89eb29", + "sha256:9a22c94d2e93af8bebd4fcf5fa38830f5e3b1ff0d4424e2912b07651eb1bafb4", + "sha256:a7d3f4b4bbb5d7866ae727763268b5c15797cbd7b63ea17f3b0ec1067da8994b", + "sha256:b645a49376547b3816433a7e2d2a99135c8e651e50497e7ecac3bd126e4bea16", + "sha256:cf0765822e78cf9e45451647a346d443f66792aba906bc340f4e0ac7870c169c", + "sha256:dc398e1e047efb18bfab7a8989346c6921a847feae2cad69fedf6ca12fb99e2c", + "sha256:dd5995ae2e80044e33b5077fb4bc2b0c1788ac6feaf15a6b87a00c14b4bdd682", + "sha256:e03fe5e07e70f245dc9013a9d48ae8cc4b10c33a1968039c5a3b64b5d01d083d", + "sha256:ea09a306144dff2795e48439883349819bef2c53c0ee62a3c2fae429451843bb", + "sha256:f4e37f33da282c3c319849877e34f97f0a3acec09622ec61b7333205bdd13b52", + "sha256:fa4bad0d1d173dee3e8ef3c3eb6b2bb6c723fc7a661eeecc1ecb2fa99860dd45" + ], + "version": "==18.1.0" }, "quilt": { "hashes": [ @@ -1068,24 +1074,24 @@ }, "scipy": { "hashes": [ - "sha256:03b1e0775edbe6a4c64effb05fff2ce1429b76d29d754aa5ee2d848b60033351", - "sha256:09d008237baabf52a5d4f5a6fcf9b3c03408f3f61a69c404472a16861a73917e", - "sha256:10325f0ffac2400b1ec09537b7e403419dcd25d9fee602a44e8a32119af9079e", - "sha256:1db9f964ed9c52dc5bd6127f0dd90ac89791daa690a5665cc01eae185912e1ba", - "sha256:409846be9d6bdcbd78b9e5afe2f64b2da5a923dd7c1cd0615ce589489533fdbb", - "sha256:4907040f62b91c2e170359c3d36c000af783f0fa1516a83d6c1517cde0af5340", - "sha256:6c0543f2fdd38dee631fb023c0f31c284a532d205590b393d72009c14847f5b1", - "sha256:826b9f5fbb7f908a13aa1efd4b7321e36992f5868d5d8311c7b40cf9b11ca0e7", - "sha256:a7695a378c2ce402405ea37b12c7a338a8755e081869bd6b95858893ceb617ae", - "sha256:a84c31e8409b420c3ca57fd30c7589378d6fdc8d155d866a7f8e6e80dec6fd06", - "sha256:adadeeae5500de0da2b9e8dd478520d0a9945b577b2198f2462555e68f58e7ef", - "sha256:b283a76a83fe463c9587a2c88003f800e08c3929dfbeba833b78260f9c209785", - "sha256:c19a7389ab3cd712058a8c3c9ffd8d27a57f3d84b9c91a931f542682bb3d269d", - "sha256:c3bb4bd2aca82fb498247deeac12265921fe231502a6bc6edea3ee7fe6c40a7a", - "sha256:c5ea60ece0c0c1c849025bfc541b60a6751b491b6f11dd9ef37ab5b8c9041921", - "sha256:db61a640ca20f237317d27bc658c1fc54c7581ff7f6502d112922dc285bdabee" - ], - "version": "==1.3.0" + "sha256:0baa64bf42592032f6f6445a07144e355ca876b177f47ad8d0612901c9375bef", + "sha256:243b04730d7223d2b844bda9500310eecc9eda0cba9ceaf0cde1839f8287dfa8", + "sha256:2643cfb46d97b7797d1dbdb6f3c23fe3402904e3c90e6facfe6a9b98d808c1b5", + "sha256:396eb4cdad421f846a1498299474f0a3752921229388f91f60dc3eda55a00488", + "sha256:3ae3692616975d3c10aca6d574d6b4ff95568768d4525f76222fb60f142075b9", + "sha256:435d19f80b4dcf67dc090cc04fde2c5c8a70b3372e64f6a9c58c5b806abfa5a8", + "sha256:46a5e55850cfe02332998b3aef481d33f1efee1960fe6cfee0202c7dd6fc21ab", + "sha256:75b513c462e58eeca82b22fc00f0d1875a37b12913eee9d979233349fce5c8b2", + "sha256:7ccfa44a08226825126c4ef0027aa46a38c928a10f0a8a8483c80dd9f9a0ad44", + "sha256:89dd6a6d329e3f693d1204d5562dd63af0fd7a17854ced17f9cbc37d5b853c8d", + "sha256:a81da2fe32f4eab8b60d56ad43e44d93d392da228a77e229e59b51508a00299c", + "sha256:a9d606d11eb2eec7ef893eb825017fbb6eef1e1d0b98a5b7fc11446ebeb2b9b1", + "sha256:ac37eb652248e2d7cbbfd89619dce5ecfd27d657e714ed049d82f19b162e8d45", + "sha256:cbc0611699e420774e945f6a4e2830f7ca2b3ee3483fca1aa659100049487dd5", + "sha256:d02d813ec9958ed63b390ded463163685af6025cb2e9a226ec2c477df90c6957", + "sha256:dd3b52e00f93fd1c86f2d78243dfb0d02743c94dd1d34ffea10055438e63b99d" + ], + "version": "==1.3.1" }, "send2trash": { "hashes": [ @@ -1328,24 +1334,24 @@ }, "importlib-metadata": { "hashes": [ - "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", - "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + "sha256:23d3d873e008a513952355379d93cbcab874c58f4f034ff657c7a87422fa64e8", + "sha256:80d2de76188eabfbfcf27e6a37342c2827801e59c4cc14b0371c56fed43820e3" ], - "version": "==0.18" + "version": "==0.19" }, "ipykernel": { "hashes": [ - "sha256:346189536b88859937b5f4848a6fd85d1ad0729f01724a411de5cae9b618819c", - "sha256:f0e962052718068ad3b1d8bcc703794660858f58803c3798628817f492a8769c" + "sha256:167c3ef08450f5e060b76c749905acb0e0fbef9365899377a4a1eae728864383", + "sha256:b503913e0b4cce7ed2de965457dfb2edd633e8234161a60e23f2fe2161345d12" ], - "version": "==5.1.1" + "version": "==5.1.2" }, "ipython": { "hashes": [ - "sha256:11067ab11d98b1e6c7f0993506f7a5f8a91af420f7e82be6575fcb7a6ca372a0", - "sha256:60bc55c2c1d287161191cc2469e73c116d9b634cff25fe214a43cba7cec94c79" + "sha256:1d3a1692921e932751bc1a1f7bb96dc38671eeefdc66ed33ee4cbc57e92a410e", + "sha256:537cd0176ff6abd06ef3e23f2d0c4c2c8a4d9277b7451544c6cbf56d1c79a83d" ], - "version": "==7.6.1" + "version": "==7.7.0" }, "ipython-genutils": { "hashes": [ @@ -1356,17 +1362,17 @@ }, "jedi": { "hashes": [ - "sha256:53c850f1a7d3cfcd306cc513e2450a54bdf5cacd7604b74e42dd1f0758eaaf36", - "sha256:e07457174ef7cb2342ff94fa56484fe41cec7ef69b0059f01d3f812379cb6f7c" + "sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27", + "sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e" ], - "version": "==0.14.1" + "version": "==0.15.1" }, "jsonschema": { "hashes": [ - "sha256:0c0a81564f181de3212efa2d17de1910f8732fa1b71c42266d983cd74304e20d", - "sha256:a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" + "sha256:5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f", + "sha256:8d4a2b7b6c2237e0199c8ea1a6d3e05bf118e289ae2b9d7ba444182a2959560d" ], - "version": "==3.0.1" + "version": "==3.0.2" }, "jupyter-client": { "hashes": [ @@ -1406,10 +1412,10 @@ }, "packaging": { "hashes": [ - "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", - "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" + "sha256:a7ac867b97fdc07ee80a8058fe4435ccd274ecc3b0ed61d852d7d53055528cf9", + "sha256:c491ca87294da7cc01902edbe30a5bc6c4c28172b5138ab4e4aa1b9d7bfaeafe" ], - "version": "==19.0" + "version": "==19.1" }, "parse": { "hashes": [ @@ -1485,16 +1491,16 @@ }, "pyparsing": { "hashes": [ - "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", - "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", + "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" ], - "version": "==2.4.0" + "version": "==2.4.2" }, "pyrsistent": { "hashes": [ - "sha256:50cffebc87ca91b9d4be2dcc2e479272bcb466b5a0487b6c271f7ddea6917e14" + "sha256:34b47fa169d6006b32e99d4b3c4031f155e6e68ebcc107d6454852e8e0ee6533" ], - "version": "==0.15.3" + "version": "==0.15.4" }, "pytest": { "hashes": [ @@ -1513,33 +1519,33 @@ }, "pyzmq": { "hashes": [ - "sha256:00dd015159eaeb1c0731ad49310e1f5d839c9a35a15e4f3267f5052233fad99b", - "sha256:03913b6beb8e7b417b9910b0ee1fd5d62e9626d218faefbe879d70714ceab1a2", - "sha256:13f17386df81d5e6efb9a4faea341d8de22cdc82e49a326dded26e33f42a3112", - "sha256:16c6281d96885db1e15f7047ddc1a8f48ff4ea35d31ca709f4d2eb39f246d356", - "sha256:17efab4a804e31f58361631256d660214204046f9e2b962738b171b9ad674ea7", - "sha256:2b79919ddeff3d3c96aa6087c21d294c8db1c01f6bfeee73324944683685f419", - "sha256:2f832e4711657bb8d16ea1feba860f676ec5f14fb9fe3b449b5953a60e89edae", - "sha256:31a11d37ac73107363b47e14c94547dbfc6a550029c3fe0530be443199026fc2", - "sha256:33a3e928e6c3138c675e1d6702dd11f6b7050177d7aab3fc322db6e1d2274490", - "sha256:34a38195a6d3a9646cbcdaf8eb245b4d935c7a57f7e1b3af467814bc1a92467e", - "sha256:42900054f1500acef6df7428edf806abbf641bf92eb9ceded24aa863397c3bae", - "sha256:4ccc7f3c63aa9d744dadb62c49eda2d0e7de55649b80c45d7c684d70161a69af", - "sha256:5b220c37c346e6575db8c88a940c1fc234f99ce8e0068c408919bb8896c4b6d2", - "sha256:6074848da5c8b44a1ca40adf75cf65aa92bc80f635e8249aa8f37a69b2b9b6f5", - "sha256:61a4155964bd4a14ef95bf46cb1651bcf8dcbbed8c0108e9c974c1fcbb57788f", - "sha256:62b5774688326600c52f587f7a033ca6b6284bef4c8b1b5fda32480897759eac", - "sha256:65a9ffa4f9f085d696f16fd7541f34b3c357d25fe99c90e3bce2ea59c3b5b4b6", - "sha256:76a077d2c30f8adc5e919a55985a784b96aeca69b53c1ea6fd5723d3ae2e6f53", - "sha256:8e5b4c51557071d6379d6dc1f54f35e9f6a137f5e84e102efb869c8d3c13c8ff", - "sha256:917f73e07cc04f0678a96d93e7bb8b1adcccdde9ccfe202e622814f4d1d1ecfd", - "sha256:91c75d3c4c357f9643e739db9e79ab9681b2f6ae8ec5678d6ef2ea0d01532596", - "sha256:923dd91618b100bb4c92ab9ed7b65825a595b8524a094ce03c7cb2aaae7d353b", - "sha256:9849054e0355e2bc7f4668766a25517ba76095031c9ff5e39ae8949cee5bb024", - "sha256:c9d453933f0e3f44b9759189f2a18aa765f7f1a4345c727c18ebe8ad0d748d26", - "sha256:cb7514936277abce64c2f4c56883e5704d85ed04d98d2d432d1c6764003bb003" - ], - "version": "==18.0.2" + "sha256:01636e95a88d60118479041c6aaaaf5419c6485b7b1d37c9c4dd424b7b9f1121", + "sha256:021dba0d1436516092c624359e5da51472b11ba8edffa334218912f7e8b65467", + "sha256:0463bd941b6aead494d4035f7eebd70035293dd6caf8425993e85ad41de13fa3", + "sha256:05fd51edd81eed798fccafdd49c936b6c166ffae7b32482e4d6d6a2e196af4e6", + "sha256:1fadc8fbdf3d22753c36d4172169d184ee6654f8d6539e7af25029643363c490", + "sha256:22efa0596cf245a78a99060fe5682c4cd00c58bb7614271129215c889062db80", + "sha256:260c70b7c018905ec3659d0f04db735ac830fe27236e43b9dc0532cf7c9873ef", + "sha256:2762c45e289732d4450406cedca35a9d4d71e449131ba2f491e0bf473e3d2ff2", + "sha256:2fc6cada8dc53521c1189596f1898d45c5f68603194d3a6453d6db4b27f4e12e", + "sha256:343b9710a61f2b167673bea1974e70b5dccfe64b5ed10626798f08c1f7227e72", + "sha256:41bf96d5f554598a0632c3ec28e3026f1d6591a50f580df38eff0b8067efb9e7", + "sha256:856b2cdf7a1e2cbb84928e1e8db0ea4018709b39804103d3a409e5584f553f57", + "sha256:85b869abc894672de9aecdf032158ea8ad01e2f0c3b09ef60e3687fb79418096", + "sha256:93f44739db69234c013a16990e43db1aa0af3cf5a4b8b377d028ff24515fbeb3", + "sha256:98fa3e75ccb22c0dc99654e3dd9ff693b956861459e8c8e8734dd6247b89eb29", + "sha256:9a22c94d2e93af8bebd4fcf5fa38830f5e3b1ff0d4424e2912b07651eb1bafb4", + "sha256:a7d3f4b4bbb5d7866ae727763268b5c15797cbd7b63ea17f3b0ec1067da8994b", + "sha256:b645a49376547b3816433a7e2d2a99135c8e651e50497e7ecac3bd126e4bea16", + "sha256:cf0765822e78cf9e45451647a346d443f66792aba906bc340f4e0ac7870c169c", + "sha256:dc398e1e047efb18bfab7a8989346c6921a847feae2cad69fedf6ca12fb99e2c", + "sha256:dd5995ae2e80044e33b5077fb4bc2b0c1788ac6feaf15a6b87a00c14b4bdd682", + "sha256:e03fe5e07e70f245dc9013a9d48ae8cc4b10c33a1968039c5a3b64b5d01d083d", + "sha256:ea09a306144dff2795e48439883349819bef2c53c0ee62a3c2fae429451843bb", + "sha256:f4e37f33da282c3c319849877e34f97f0a3acec09622ec61b7333205bdd13b52", + "sha256:fa4bad0d1d173dee3e8ef3c3eb6b2bb6c723fc7a661eeecc1ecb2fa99860dd45" + ], + "version": "==18.1.0" }, "six": { "hashes": [ diff --git a/README.md b/README.md index 39d703a..f8ace28 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,9 @@ Also a convenient [flat file](https://en.wikipedia.org/wiki/Flat-file_database) │ ├── *.feature... (easily understandable specifications written using the Given-When-Then gherkin language) │ └── README.md (markdown information on the feature files) ├── highres/ (contains high resolution localized DEMs) - │ ├── *.grd/las/txt/csv... (input vector file containing the point-based data) - │ ├── *.json (the pdal pipeline file) - │ ├── *.tif (output raster geotiff file) + │ ├── *.txt/csv/grd/xyz... (input vector file containing the point-based bed elevation data) + │ ├── *.json (the pipeline file used to process the xyz point data) + │ ├── *.nc (output raster netcdf files) │ └── README.md (markdown information on highres data sources) ├── lowres/ (contains low resolution whole-continent DEMs) │ ├── bedmap2_bed.tif (the low resolution DEM!) @@ -35,18 +35,20 @@ Also a convenient [flat file](https://en.wikipedia.org/wiki/Flat-file_database) ├── model/ (*hidden in git, neural network model related files) │ ├── train/ (a place to store the raster tile bounds and model training data) │ └── weights/ (contains the neural network model's architecture and weights) - ├── .env (environment config file used by pipenv, supposedly) + ├── .env (environment variable config file used by pipenv) ├── .ignore (files ignored by a particular piece of software) - ├── Dockerfile (set of commands to reproduce the software stack here into a docker image) + ├── . (stuff to make the code in this repo look and run nicely e.g. linters, CI/CD config files, etc) + ├── Dockerfile (set of commands to fully reproduce the software stack here into a docker image, used by binder) ├── LICENSE.md (the license covering this repository) - ├── Pipfile (what you want, the minimal core dependencies) - ├── Pipfile.lock (what you need, all the pinned dependencies for full reproducibility) + ├── Pipfile (what you want, the summary list of core python dependencies) + ├── Pipfile.lock (what you need, all the pinned python dependencies for full reproducibility) ├── README.md (the markdown file you're reading now) ├── data_list.yml (human and machine readable list of the datasets and their metadata) - ├── data_prep.ipynb (jupyter notebook that prepares the data) - ├── environment.yml (conda packages to install, used by binder) - ├── srgan_train.ipynb (jupyter notebook that trains the Super Resolution Generative Adversarial Network model) - └── test_ipynb.ipynb (jupyter notebook that runs doctests in the other jupyter notebooks!) + ├── data_prep.ipynb/py (paired jupyter notebook/python script that prepares the data) + ├── deepbedmap.ipynb/py (paired jupyter notebook/python script that predicts an Antarctic bed digital elevation model) + ├── environment.yml (conda binary packages to install) + ├── srgan_train.ipynb/py (paired jupyter notebook/python script that trains the ESRGAN neural network model) + └── test_ipynb.ipynb/py (paired jupyter notebook/python script that runs doctests in the other jupyter notebooks!) ``` @@ -72,16 +74,19 @@ The conda virtual environment will also be created with Python and [pipenv](http cd deepbedmap conda env create -f environment.yml -Activate the conda environment first, and then use pipenv to install the necessary python libraries. -Note that `pipenv install` won't work directly (see Common problems below). -You may want to check that `which pipenv` returns something similar to ~/.conda/envs/deepbedmap/bin/pipenv. +Activate the conda environment first. conda activate deepbedmap +Then set some environment variables **before** using pipenv to install the necessary python libraries, +otherwise you may encounter some problems (see Common problems below). +You may want to ensure that `which pipenv` returns something similar to ~/.conda/envs/deepbedmap/bin/pipenv. + + export HDF5_DIR=$CONDA_PREFIX/ export LD_LIBRARY_PATH=$CONDA_PREFIX/lib/ - pipenv install --python $CONDA_PREFIX/bin/python + pipenv install --python $CONDA_PREFIX/bin/python --dev #or just - LD_LIBRARY_PATH=$CONDA_PREFIX/lib/ pipenv install --python $CONDA_PREFIX/bin/python + HDF5_DIR=$CONDA_PREFIX/ LD_LIBRARY_PATH=$CONDA_PREFIX/lib/ pipenv install --python $CONDA_PREFIX/bin/python --dev Finally, double-check that the libraries have been installed. @@ -90,16 +95,24 @@ Finally, double-check that the libraries have been installed. ### Syncing/Updating to new dependencies conda env update -f environment.yml - pipenv sync + pipenv sync --dev ### Common problems Note that the [.env](https://pipenv.readthedocs.io/en/latest/advanced/#configuration-with-environment-variables) file stores some environment variables. -However, it only works when running `pipenv shell` or `pipenv run `. -So after running `conda activate deepbedmap`, and you see an `...error while loading shared libraries: libpython3.6m.so.1.0...`, you may need to run this: +So if you run `conda activate deepbedmap` followed by some other command and get an `...error while loading shared libraries: libpython3.7m.so.1.0...`, +you may need to run `pipenv shell` or do `pipenv run ` to have those environment variables registered properly. +Or just run this first: export LD_LIBRARY_PATH=$CONDA_PREFIX/lib/ +Also, if you get a problem when using `pipenv` to install [netcdf4](https://github.com/Unidata/netcdf4-python), make sure you have done: + + export HDF5_DIR=$CONDA_PREFIX/ + +and then you can try using `pipenv install` or `pipenv sync` again. +See also this [issue](https://github.com/pydata/xarray/issues/3185#issuecomment-520693149) for more information. + ## Running jupyter lab conda activate deepbedmap diff --git a/environment.yml b/environment.yml index f7976e6..8bd4eca 100644 --- a/environment.yml +++ b/environment.yml @@ -6,8 +6,8 @@ dependencies: - hcc::cuda_driver=410.73[md5=941787b750b372f4a240287634589d24] - anaconda::cudatoolkit=10.0[md5=4388ad6015992501b042eea5197eb447] - conda-forge/label/dev::gmt=6.0.0rc3[md5=5dd95d211fff2434e792dac514063721] - - conda-forge::pip=19.2.1[md5=95e1564d40edda4b98171de82ceba8ab] - - conda-forge::python=3.6.7[md5=787937284685efaaa233453b49a9e719] + - conda-forge::pip=19.2.1[md5=f607b2d79d7c31147699e57687c227c3] + - conda-forge::python=3.7.3[md5=e9385633516bf38da68e983b4f8b9e2a] - conda-forge::libspatialindex=1.9.0[md5=ef8f25a228de3a0dc716566b8d3de593] - pip: - pipenv==2018.11.26