download files to specific directory 📦

.gitignore

@@ -90,3 +90,4 @@ ENV/
 
 ._.DS_Store
 .idea
+tmp

Dockerfile

@@ -2,6 +2,10 @@ FROM python:3.5
 
 ENV PYTHONUNBUFFERED 1
 
+# runtime dependencies
+RUN apt-get install -y --no-install-recommends \
+    libmagickwand-dev
+
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 

nasabot/geo/animation.py

@@ -0,0 +1,13 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def animate(file_sequence):
+    """
+
+    :param file_sequence:
+    :return:
+    """
+    logger.debug('animate sequence of files')
+    logger.debug(file_sequence)

nasabot/geo/download.py

@@ -3,21 +3,25 @@
 from contextlib import closing
 import logging
 import datetime
+import os
 from slugify import slugify
+from nasabot.geo import animation
+
+dir_path = os.getcwd()
 
 logger = logging.getLogger(__name__)
 
 
-def url2filename(url):
-    # TODO: define destination directory
-    *first, ext = url.split('.')
-    return slugify('.'.join(first)) + '.' + ext
+def makedir(filename):
+    os.makedirs(os.path.dirname(filename), exist_ok=True)
 
 
-async def download(url, session, chunk_size):
-    filename = url2filename(url)
-    logging.info('downloading %s\n  to %s', url, filename)
+async def download(filename, url, session, chunk_size):
+    logging.info('downloading %s\n ---> to %s', url, filename)
     response = await session.get(url)
+
+    makedir(filename)
+
     with closing(response), open(filename, 'wb') as file:
         while True:  # save file
             chunk = await response.content.read(chunk_size)
@@ -28,19 +32,19 @@ async def download(url, session, chunk_size):
     return filename, (response.status, tuple(response.headers.items()))
 
 
-async def bound_download(url, session, semaphore, chunk_size=1 << 15):
+async def bound_download(filename, url, session, semaphore, chunk_size=1 << 15):
     async with semaphore:  # limit number of concurrent downloads
-        return await download(url, session, chunk_size)
+        return await download(filename, url, session, chunk_size)
 
 
-async def download_animation(source, animation, num_in_parallel=4):
+async def download_animation(target, source, animation, num_in_parallel=4):
     urls = [source.get_url_by_frame(frame) for frame in animation]
 
     logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
 
     with closing(aiohttp.ClientSession()) as session:
         semaphore = asyncio.Semaphore(num_in_parallel)
-        download_tasks = (bound_download(url, session, semaphore) for url in urls)
+        download_tasks = (bound_download(target.get_filename_by_url(url), url, session, semaphore) for url in urls)
         return await asyncio.gather(*download_tasks)
 
 
@@ -73,17 +77,41 @@ def get_url_by_frame(self, frame):
         return self.pattern.format(**{**self.values, **frame})
 
 
+def uri_2_filename(uri):
+    """
+    convert uri to related filename
+    :param uri:
+    :return:
+    """
+    *first, ext = uri.split('.')
+    return slugify('.'.join(first)) + '.' + ext
+
+
+class Target:
+    def __init__(self, root_path=''):
+        self.root_path = root_path
+
+    def get_filename_by_url(self, url):
+        return os.path.join(self.root_path, uri_2_filename(url))
+
+
 if __name__ == '__main__':
     loop = asyncio.get_event_loop()
-    app = loop.run_until_complete(download_animation(GIBSSource(
-        'https://gibs.earthdata.nasa.gov/wmts/{projection}/best/{layer}/default/{date}/{resolution}/{z}/{y}/{x}.jpg',
-        layer='MODIS_Terra_CorrectedReflectance_TrueColor',
-        resolution='GoogleMapsCompatible_Level9',
-        projection='epsg3857',
-        x=2,
-        y=1,
-        z=2,
-    ), IntervalAnimation(
-        datetime.date(2017, 1, 1),
-        datetime.date(2017, 5, 8)
-    )))
+    files = loop.run_until_complete(download_animation(
+        Target(
+            os.path.join(dir_path, 'tmp')
+        ),
+        GIBSSource(
+            'https://gibs.earthdata.nasa.gov/wmts/{projection}/best/{layer}/default/{date}/{resolution}/{z}/{y}/{x}.jpg',
+            layer='MODIS_Terra_CorrectedReflectance_TrueColor',
+            resolution='GoogleMapsCompatible_Level9',
+            projection='epsg3857',
+            x=2,
+            y=1,
+            z=2,
+        ),
+        IntervalAnimation(
+            datetime.date(2017, 1, 1),
+            datetime.date(2017, 5, 8),
+        )))
+    animation.animate(files)

requirements.txt

@@ -10,3 +10,4 @@ pytest-catchlog==1.2.2
 pytest-cov==2.4.0
 pytest-mock==1.6.0
 python-slugify==1.2.4
+Wand