HTTP authentication

CHANGES.txt

@@ -1,13 +1,15 @@
-0.6.0 (2018-11-06)
+0.6.1 (2019-11-11)
+------------------
+- Added ability to use HTTP Authentication for TMS endpoints (#152)
+
+0.6.0 (2019-11-06)
 ------------------
 - Use sys.exectuable in place of python string (#124)
 - Correct script reference to fix bug in skynet train example (#129)
 - Add s3 requirement to rasterio (#137)
 - users can split data into more groups than train and test, for example train/test/validate, and specify the ratio for
 each split (#149)
 
-
-
 0.5.1 (2018-11-12)
 ------------------
 - Skip invalid or empty geometries which prevent segmentation rendering (#118)

docs/parameters.rst

@@ -34,6 +34,9 @@ Here is the full list of configuration parameters you can specify in a ``config.
 
  		Remote files like a `WMS endpoint <http://www.opengeospatial.org/standards/wms>`_ ``GetMap`` request. Fill out all necessary parameters except ``bbox`` which should be set as ``{bbox}``. Ex: ``'https://basemap.nationalmap.gov/arcgis/services/USGSImageryOnly/MapServer/WMSServer?SERVICE=WMS&REQUEST=GetMap&VERSION=1.1.1&LAYERS=0&STYLES=&FORMAT=image%2Fjpeg&TRANSPARENT=false&HEIGHT=256&WIDTH=256&SRS=EPSG%3A3857&BBOX={bbox}'``
 
+**http_auth**: list
+	Optional parameter to specify a username and password for restricted WMS services. For example, ``['my_username', 'my_password']``.
+
 **background_ratio**: float
 	Specify how many background (or "negative") training examples to create when there is only one class specified with the ``classes`` parameter. Label Maker will generate ``background_ratio`` times the number of images matching the one class.
 

label_maker/images.py

@@ -72,4 +72,4 @@ def class_test(value):
     image_function = get_image_function(imagery)
 
     for tile in tiles:
-        image_function(tile, imagery, tiles_dir, imagery_offset)
+        image_function(tile, imagery, tiles_dir, imagery_offset, kwargs)

label_maker/main.py

@@ -93,6 +93,10 @@ def cli():
         config['country'] = op.splitext(op.basename(config.get('geojson')))[0]
         config['bounding_box'] = get_bounds(json.load(open(config.get('geojson'), 'r')))
 
+    # Convert HTTP auth from list to tuple if it exists
+    if 'http_auth' in config.keys():
+        config['http_auth'] = tuple(config['http_auth'])
+
     if cmd == 'download':
         download_mbtiles(dest_folder=dest_folder, **config)
     elif cmd == 'labels':

label_maker/preview.py

@@ -64,7 +64,8 @@ def preview(dest_folder, number, classes, imagery, ml_type, imagery_offset=False
             if n >= number:
                 break
 
-            tile_img = image_function(tile, imagery, class_dir, imagery_offset)
+            tile_img = image_function(tile, imagery, class_dir, imagery_offset,
+                                      kwargs)
 
             if ml_type == 'object-detection':
                 img = Image.open(tile_img)

label_maker/utils.py

@@ -24,17 +24,18 @@ def class_match(ml_type, label, i):
         return np.count_nonzero(label == i)
     return None
 
-def download_tile_tms(tile, imagery, folder, *args):
+def download_tile_tms(tile, imagery, folder, kwargs):
     """Download a satellite image tile from a tms endpoint"""
     o = urlparse(imagery)
     _, image_format = op.splitext(o.path)
-    r = requests.get(url(tile.split('-'), imagery))
+    r = requests.get(url(tile.split('-'), imagery),
+                     auth=kwargs.get('http_auth'))
     tile_img = op.join(folder, '{}{}'.format(tile, image_format))
     with open(tile_img, 'wb')as w:
         w.write(r.content)
     return tile_img
 
-def get_tile_tif(tile, imagery, folder, imagery_offset):
+def get_tile_tif(tile, imagery, folder, imagery_offset, kwargs):
     """
     Read a GeoTIFF with a window corresponding to a TMS tile
 
@@ -87,7 +88,7 @@ def get_tile_tif(tile, imagery, folder, imagery_offset):
 
     return tile_img
 
-def get_tile_wms(tile, imagery, folder, imagery_offset):
+def get_tile_wms(tile, imagery, folder, imagery_offset, kwargs):
     """
     Read a WMS endpoint with query parameters corresponding to a TMS tile
 
@@ -118,7 +119,7 @@ def get_tile_wms(tile, imagery, folder, imagery_offset):
 
     # request the image with the transformed bounding box and save
     wms_url = imagery.replace('{bbox}', ','.join([str(b) for b in bbox]))
-    r = requests.get(wms_url)
+    r = requests.get(wms_url, auth=kwargs.get('http_auth'))
     tile_img = op.join(folder, '{}.{}'.format(tile, image_format))
     with open(tile_img, 'wb') as w:
         w.write(r.content)

label_maker/validate.py

@@ -27,6 +27,7 @@
     'zoom': {'type': 'integer', 'required': True},
     'classes': {'type': 'list', 'schema': class_schema, 'required': True},
     'imagery': {'type': 'string', 'required': True},
+    'http_auth': {'type': 'list', 'schema': {'type': 'string'}},
     'background_ratio': {'type': 'float'},
     'ml_type': {'allowed': ['classification', 'object-detection', 'segmentation'], 'required': True},
     'seed': {'type': 'integer'},

label_maker/version.py

@@ -1,2 +1,2 @@
 """Library verison"""
-__version__ = '0.6.0'
+__version__ = '0.6.1'

test/unit/test_utils.py

@@ -81,7 +81,7 @@ def test_get_tile_tif(self):
         if not op.isdir(tiles_dir):
             makedirs(tiles_dir)
 
-        get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, None)
+        get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, None, {})
         test_tile = Image.open('test/tiles/{}.jpg'.format(tile))
         fixture_tile = Image.open('test/fixtures/{}.jpg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)
@@ -95,7 +95,7 @@ def test_get_tile_tif_offset(self):
         if not op.isdir(tiles_dir):
             makedirs(tiles_dir)
 
-        get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, [128, 64])
+        get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, [128, 64], {})
         test_tile = Image.open('test/tiles/{}.jpg'.format(tile))
         fixture_tile = Image.open('test/fixtures/{}_offset.jpg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)
@@ -109,7 +109,7 @@ def test_get_tile_vrt(self):
         if not op.isdir(tiles_dir):
             makedirs(tiles_dir)
 
-        get_tile_tif(tile, 'test/fixtures/drone.vrt', tiles_dir, None)
+        get_tile_tif(tile, 'test/fixtures/drone.vrt', tiles_dir, None, {})
         test_tile = Image.open('test/tiles/{}.jpg'.format(tile))
         fixture_tile = Image.open('test/fixtures/{}.jpg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)
@@ -125,7 +125,7 @@ def test_get_tile_wms(self):
 
         nasa_url = 'https://gibs.earthdata.nasa.gov/wms/epsg4326/best/wms.cgi?SERVICE=WMS&REQUEST=GetMap&layers=MODIS_Aqua_CorrectedReflectance_TrueColor&version=1.3.0&crs=EPSG:4326&transparent=false&width=256&height=256&bbox={bbox}&format=image/jpeg&time=2019-03-05'
 
-        get_tile_wms(tile, nasa_url, tiles_dir, None)
+        get_tile_wms(tile, nasa_url, tiles_dir, None, {})
         test_tile = Image.open('test/tiles/{}.jpeg'.format(tile))
         fixture_tile = Image.open('test/fixtures/{}.jpeg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)