Make data sources readable and incorporate sha256 hash checks

README.md

@@ -28,6 +28,7 @@ Also a convenient [flat file](https://en.wikipedia.org/wiki/Flat-file_database)
     ├── model/ (*hidden in git, neural network model related files)
     │    ├── logs/ (directory for tensorboard log files)
     │    └── train/ (a place to store the model training data)
+    │        ├── W*_data.npy (miscellaneous numpy arrays)
     │        ├── X_data.npy (highres numpy arrays)
     │        └── Y_data.npy (lowres numpy arrays)
     ├── .env (environment config file used by pipenv, supposedly)
@@ -37,6 +38,7 @@ Also a convenient [flat file](https://en.wikipedia.org/wiki/Flat-file_database)
     ├── Pipfile (what you want, the minimal core dependencies)
     ├── Pipfile.lock (what you need, all the pinned dependencies for full reproducibility)
     ├── README.md (the markdown file you're reading now)
+    ├── data_list.yml (human and machine readable list of the datasets and their metadata)
     ├── data_prep.ipynb (jupyter notebook that prepares the data)
     ├── environment.yml (conda packages to install, used by binder)
     ├── srgan_train.ipynb (jupyter notebook that trains the Super Resolution Generative Adversarial Network model)

data_list.yml

@@ -0,0 +1,154 @@
+- 
+  citekey: Fretwell2013BEDMAP2
+  folder: lowres
+  location: Antarctica
+  resolution: 1000m
+  doi: 
+    dataset: "https://doi.org/10.7488/ds/1916"
+    literature: "https://doi.org/10.5194/tc-7-375-2013"
+  files: 
+    - 
+      filename: bedmap2_bed.tif
+      url: "http://data.pgc.umn.edu/elev/dem/bedmap2/bedmap2_bed.tif"
+      sha256: 28e2ca7656d61b0bc7f8f8c1db41914023e0cab1634e0ee645f38a87d894b416
+- 
+  citekey: Noh2018REMA
+  folder: misc
+  location: Antarctica
+  resolution: 200m
+  doi: 
+    dataset: "https://doi.org/10.7910/DVN/SAIK8B"
+    literature: "https://doi.org/10.1016/j.isprsjprs.2017.12.008"
+  files: 
+    - 
+      filename: REMA_200m_dem_filled.tif
+      url: "http://data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.0/200m/REMA_200m_dem_filled.tif"
+      sha256: 8ac252e40810ac5e59934879a066f496c847936771f318dab2ab4a257052d964
+- 
+  citekey: Rignot2011MEASURES
+  folder: misc
+  location: Antarctica
+  resolution: 450m
+  doi: 
+    dataset: "https://doi.org/10.5067/D7GK8F5J8M8R"
+    literature: "https://doi.org/10.1126/science.1208336"
+  files: 
+    - 
+      filename: MEaSUREs_IceFlowSpeed_450m.tif
+      url: "http://data.pgc.umn.edu/gis/packages/quantarctica/Quantarctica3/Glaciology/MEaSUREs%20Ice%20Flow%20Velocity/MEaSUREs_IceFlowSpeed_450m.tif"
+      sha256: 4a4efc3a84204c3d67887e8d7fa1186467b51e696451f2832ebbea3ca491c8a8
+- 
+  citekey: King2016Rutford
+  folder: highres
+  location: Rutford Ice Stream
+  resolution: nan
+  doi: 
+    dataset: "https://doi.org/10.5285/54757cbe-0b13-4385-8b31-4dfaa1dab55e"
+    literature: "https://doi.org/10.5194/essd-8-151-2016"
+  files: 
+    - 
+      filename: bed_WGS84_grid.txt
+      url: "http://ramadda.nerc-bas.ac.uk/repository/entry/get/Polar%20Data%20Centre/DOI/Rutford%20Ice%20Stream%20bed%20elevation%20DEM%20from%20radar%20data/bed_WGS84_grid.txt?entryid=synth%3A54757cbe-0b13-4385-8b31-4dfaa1dab55e%3AL2JlZF9XR1M4NF9ncmlkLnR4dA%3D%3D"
+      sha256: 0d3e98ca727fc1201b436170af5a63f23348aaf146a3ac6234f6c4da283e8b34
+- 
+  citekey: Bingham2018PIG
+  folder: highres
+  location: Pine Island Glacier
+  resolution: nan
+  doi: 
+    dataset: nan
+    literature: "https://doi.org/10.1038/s41467-017-01597-y"
+  files: 
+    - 
+      filename: 2007t1.txt
+      url: nan
+      sha256: 04bdbd3c8e814cbc8f0d324277e339a46cc90a8dc23434d11815a8966951e766
+    - 
+      filename: 2007tr.txt
+      url: nan
+      sha256: 3858a1e58e17b2816920e1b309534cee0391f72a6a0aa68d57777b030e70e9a3
+    - 
+      filename: 2010tr.txt
+      url: nan
+      sha256: 751ea56acc5271b3fb54893ed59e05ff485187a6fc5daaedf75946d730805b80
+    - 
+      filename: istar08.txt
+      url: nan
+      sha256: ed03c64332e8d406371c74a66f3cd21fb3f78ee498ae8408c355879bb89eb13d
+    - 
+      filename: istar18.txt
+      url: nan
+      sha256: 3e69d86f28e26810d29b0b9309090684dcb295c0dd39007fe9ee0d1285c57804
+    - 
+      filename: istar15.txt
+      url: nan
+      sha256: 59c981e8c96f73f3a5bd98be6570e101848b4f67a12d98a577292e7bcf776b17
+    - 
+      filename: istar13.txt
+      url: nan
+      sha256: f5bcf80c7ea5095e2eabf72b69a264bf36ed56af5cb67976f9428f560e5702a2
+    - 
+      filename: istar17.txt
+      url: nan
+      sha256: f51a674dc27d6e0b99d199949a706ecf96ea807883c1901fea186efc799a36e8
+    - 
+      filename: istar07.txt
+      url: nan
+      sha256: c81ec04290433f598ce4368e4aae088adeeabb546913edc44c54a5a5d7593e93
+- 
+  citekey: Shi2010CRESIS
+  folder: highres
+  location: Antarctica
+  resolution: nan
+  doi: 
+    dataset: "https://doi.org/10.5067/GDQ0CUCVTE2Q"
+    literature: "https://doi.org/10.1109/IGARSS.2010.5649518"
+  files: 
+    - 
+      filename: 2009_Antarctica_DC8.csv
+      url: "https://data.cresis.ku.edu/data/rds/2009_Antarctica_DC8/csv_good/2009_Antarctica_DC8.csv"
+      sha256: 1b9fe0faf4ef217794c2a1de9ef8cfa45f5949efdc4e925930d31c0554cf0ca2
+    - 
+      filename: 2009_Antarctica_TO.csv
+      url: "https://data.cresis.ku.edu/data/rds/2009_Antarctica_TO/csv_good/2009_Antarctica_TO.csv"
+      sha256: 7a90c5955fa881b4fb88e45ff11629e60ff9ad045c07bf4c6e3aa1f7d1a9361d
+    - 
+      filename: 2009_Antarctica_TO_Gambit.csv
+      url: "https://data.cresis.ku.edu/data/rds/2009_Antarctica_TO_Gambit/csv_good/2009_Antarctica_TO_Gambit.csv"
+      sha256: 93da613223733a4850283b700060afdb14f1002fe5613b8d78c6d3be83e34072
+    - 
+      filename: 2010_Antarctica_DC8.csv
+      url: "https://data.cresis.ku.edu/data/rds/2010_Antarctica_DC8/csv_good/2010_Antarctica_DC8.csv"
+      sha256: f725a8dbc21d31601b99ccaf9f5282ecd516f2ff966d268b4e735ea1af2014e6
+    - 
+      filename: 2011_Antarctica_DC8.csv
+      url: "https://data.cresis.ku.edu/data/rds/2011_Antarctica_DC8/csv_good/2011_Antarctica_DC8.csv"
+      sha256: 38aba2a39b0d58b72827f25cfcd667fc943f25c0024d3c52cb1b9e65e9e76163
+    - 
+      filename: 2011_Antarctica_TO.csv
+      url: "https://data.cresis.ku.edu/data/rds/2011_Antarctica_TO/csv_good/2011_Antarctica_TO.csv"
+      sha256: 4bf37750b9986ce582c9fd1f3a6ac622fc17f3b3ecb07b7a7132eb3797ee31d1
+    - 
+      filename: 2012_Antarctica_DC8.csv
+      url: "https://data.cresis.ku.edu/data/rds/2012_Antarctica_DC8/csv_good/2012_Antarctica_DC8.csv"
+      sha256: 5c6701b8c34bd57517b93e8e18f32e4579d6e2f56e4796bd7140b3e338544007
+    - 
+      filename: 2013_Antarctica_Basler.csv
+      url: "https://data.cresis.ku.edu/data/rds/2013_Antarctica_Basler/csv_good/2013_Antarctica_Basler.csv"
+      sha256: 56609027b4af04ba078ae093772916341bd1d6ab5f110de11b21294507733cc8
+    - 
+      filename: 2013_Antarctica_P3.csv
+      url: "https://data.cresis.ku.edu/data/rds/2013_Antarctica_P3/csv_good/2013_Antarctica_P3.csv"
+      sha256: 9de95030f49ce0bbf107eb72418db2845c39822872a6c9aa10f023148262f658
+    - 
+      filename: 2014_Antarctica_DC8.csv
+      url: "https://data.cresis.ku.edu/data/rds/2014_Antarctica_DC8/csv_good/2014_Antarctica_DC8.csv"
+      sha256: bd8c8674ba66508c64303725bfe45b3365467d01f69cfa8ec4258a3ced05e5bf
+    - 
+      filename: 2016_Antarctica_DC8.csv
+      url: "https://data.cresis.ku.edu/data/rds/2016_Antarctica_DC8/csv_good/2016_Antarctica_DC8.csv"
+      sha256: ec3b514dfcae265f5b8643eeb3503be8a0a6531e563faf9f12cb67f2b618a741
+    - 
+      filename: 2017_Antarctica_P3.csv
+      url: "https://data.cresis.ku.edu/data/rds/2017_Antarctica_P3/csv_good/2017_Antarctica_P3.csv"
+      sha256: 9208a64fefe2f4a6e7f08d44c0af0c35400cd814590c32b8eb02f1545bfc8bec

highres/README.md

@@ -1,18 +1,9 @@
-# High Resolution Antarctic DEMs
+# High Resolution Antarctic datasets
 
-| Filename                   | Location/Name           | Resolution | Literature Citation  |      Data Citation/Link      |
-|:---------------------------|:----------------------- | ----------:|:--------------------:|:----------------------------:|
-| 20??_Antarctica_*.csv * 12 | Mainly West Antarctica  |         ?m | [Shi2010]            | [IRMCR2v1@NSIDC][RDS@CReSIS] |
-| 2010tr.txt                 | Pine Island Glacier     |        50m | [Bingham2018]        | [PIG2010-2011]               |
-| bed_WGS84_grid.txt         | Rutford Ice Stream      |     ~125m? | [King2016]           | [Rutford2007-2009]           |
-| istar??.txt * 6            | Pine Island Glacier     |        50m | [Bingham2018]        | [PIG2013-2014]               |
+Note: This file was automatically generated from [data_list.yml](/data_list.yml) using [data_prep.ipynb](/data_prep.ipynb)
 
-[RDS@CReSIS]: https://data.cresis.ku.edu/data/rds/
-[IRMCR2v1@NSIDC]: https://doi.org/10.5067/GDQ0CUCVTE2Q
-[PIG2010-2011]: https://istar-gis-features.data.bas.ac.uk/Legacy/delores_2010_2011_DEM/
-[PIG2013-2014]: https://istar-gis-features.data.bas.ac.uk/glacial/delores_dems/
-[Rutford2007-2009]: https://doi.org/10.5285/54757cbe-0b13-4385-8b31-4dfaa1dab55e
-
-[Bingham2018]: https://doi.org/10.1038/s41467-017-01597-y
-[King2016]: https://doi.org/10.5194/essd-8-151-2016
-[Shi2010]: https://doi.org/10.1109/IGARSS.2010.5649518
+Filename|Location|Resolution|Literature Citation|Data Citation
+---|---|---|---|---
+bed_WGS84_grid.txt|Rutford Ice Stream|nan|[King2016Rutford](https://doi.org/10.5194/essd-8-151-2016)|[DOI](https://doi.org/10.5285/54757cbe-0b13-4385-8b31-4dfaa1dab55e)
+9 *.txt files|Pine Island Glacier|nan|[Bingham2018PIG](https://doi.org/10.1038/s41467-017-01597-y)|
+12 *.csv files|Antarctica|nan|[Shi2010CRESIS](https://doi.org/10.1109/IGARSS.2010.5649518)|[DOI](https://doi.org/10.5067/GDQ0CUCVTE2Q)

lowres/README.md

@@ -1,12 +1,7 @@
-# Low Resolution Antarctic DEM
+# Low Resolution Antarctic datasets
 
-| Filename           | Name                    | Resolution | Literature Citation | Data Citation/Link |
-|:-------------------|:----------------------- | ----------:|:-------------------:|:------------------:|
-| bedmap2_bed.tif    | BEDMAP2                 |      1000m | [Fretwell2013]      |  [BAS] [PGC]       |
+Note: This file was automatically generated from [data_list.yml](/data_list.yml) using [data_prep.ipynb](/data_prep.ipynb)
 
-
-
-[BAS]: https://secure.antarctica.ac.uk/data/bedmap2/
-[PGC]: https://data.pgc.umn.edu/elev/dem/bedmap2/
-
-[Fretwell2013]: https://doi.org/10.5194/tc-7-375-2013
+Filename|Location|Resolution|Literature Citation|Data Citation
+---|---|---|---|---
+bedmap2_bed.tif|Antarctica|1000m|[Fretwell2013BEDMAP2](https://doi.org/10.5194/tc-7-375-2013)|[DOI](https://doi.org/10.7488/ds/1916)

misc/README.md

@@ -1,12 +1,8 @@
-# Miscellaneous datasets (not a bed DEM)
+# Miscellaneous Antarctic datasets
 
-| No | Filename                       | Resolution | Literature Citation  |          Data Citation/Link          |
-|---:|:-------------------------------| ----------:|:--------------------:|:------------------------------------:|
-|  1 | REMA_200m_dem_filled.tif       |       200m |                      |  [REMAV1]                            |
-|  2 | MEaSUREs_IceFlowSpeed_450m.tif |       450m |    [Rignot2011]      |  [MEaSUREsV2@NSIDC] [MEaSUREsV2@PGC] |
+Note: This file was automatically generated from [data_list.yml](/data_list.yml) using [data_prep.ipynb](/data_prep.ipynb)
 
-[MEaSUREsV2@NSIDC]: https://nsidc.org/data/nsidc-0484/versions/2
-[MEaSUREsV2@PGC]: http://data.pgc.umn.edu/gis/packages/quantarctica/Quantarctica30-31updates/Glaciology/MEaSUREs%20Ice%20Flow%20Velocity/MEaSUREs_IceFlowSpeed_450m.tif
-[REMAV1]: http://data.pgc.umn.edu/elev/dem/setsm/REMA/mosaic/v1.0/
-
-[Rignot2011]: https://doi.org/10.5067/D7GK8F5J8M8R
+Filename|Location|Resolution|Literature Citation|Data Citation
+---|---|---|---|---
+REMA_200m_dem_filled.tif|Antarctica|200m|[Noh2018REMA](https://doi.org/10.1016/j.isprsjprs.2017.12.008)|[DOI](https://doi.org/10.7910/DVN/SAIK8B)
+MEaSUREs_IceFlowSpeed_450m.tif|Antarctica|450m|[Rignot2011MEASURES](https://doi.org/10.1126/science.1208336)|[DOI](https://doi.org/10.5067/D7GK8F5J8M8R)

test_ipynb.ipynb

@@ -75,7 +75,23 @@
      "output_type": "stream",
      "text": [
       "Trying:\n",
-      "    download_to_path(path=\"highres/2017_Antarctica_Basler.csv\", url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/2017_Antarctica_Basler.csv\")\n",
+      "    download_to_path(path=\"highres/2017_Antarctica_Basler.csv\",\n",
+      "                     url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/2017_Antarctica_Basler.csv\")\n",
+      "Expecting:\n",
+      "    <Response [200]>\n",
+      "ok\n",
+      "Trying:\n",
+      "    check_sha256('highres/2017_Antarctica_Basler.csv')\n",
+      "Expecting:\n",
+      "    '53cef7a0d28ff92b30367514f27e888efbc32b1bda929981b371d2e00d4c671b'\n",
+      "ok\n",
+      "Trying:\n",
+      "    os.remove(path=\"highres/2017_Antarctica_Basler.csv\")\n",
+      "Expecting nothing\n",
+      "ok\n",
+      "Trying:\n",
+      "    download_to_path(path=\"highres/2017_Antarctica_Basler.csv\",\n",
+      "                     url=\"https://data.cresis.ku.edu/data/rds/2017_Antarctica_Basler/csv_good/2017_Antarctica_Basler.csv\")\n",
       "Expecting:\n",
       "    <Response [200]>\n",
       "ok\n",
@@ -92,10 +108,11 @@
       "    data_prep\n",
       "    data_prep.get_window_bounds\n",
       "    data_prep.selective_tile\n",
-      "1 items passed all tests:\n",
+      "2 items passed all tests:\n",
+      "   3 tests in data_prep.check_sha256\n",
       "   3 tests in data_prep.download_to_path\n",
-      "3 tests in 4 items.\n",
-      "3 passed and 0 failed.\n",
+      "6 tests in 5 items.\n",
+      "6 passed and 0 failed.\n",
       "Test passed.\n"
      ]
     }