replace aiohttp with requests

i guess i was to optimistic to just use aiohttp instead of
requests. i need to make more effort to learn all the new shiny
stuff in python. let make things furst work and then worry about
performance.

this should fix #48 and #51

default.nix

@@ -14,8 +14,7 @@ in stdenv.mkDerivation rec {
     setuptools
     zcbuildout
     zcrecipeegg
-    chardet
-    aiohttp
+    requests
   ];  # six attrs effect ];
   buildInputs = [ python zip makeWrapper ];
   sourceRoot = ".";
@@ -31,8 +30,7 @@ in stdenv.mkDerivation rec {
     # mv six-*/six.py                    $out/pkgs/
     # mv attrs-*/src/attr                $out/pkgs/attrs
     # mv effect-*/effect                 $out/pkgs/effect
-    mv chardet-*/chardet                $out/pkgs/
-    mv aiohttp-*/aiohttp                $out/pkgs/
+    mv requests-*/requests              $out/pkgs/
 
     if [ "$IN_NIX_SHELL" != "1" ]; then
       if [ -e git-export ]; then

src/pypi2nix/deps.nix

@@ -32,11 +32,8 @@ rec {
   effectVersion = "0.10.1";
   effectHash = "6a6fd28fb44179ce01a148d4e8bdbede";
 
-  chardetVersion = "2.3.0";
-  chardetHash = "25274d664ccb5130adae08047416e1a8";
-
-  aiohttpVersion = "0.21.6";
-  aiodnsHash = "d7f63e51bc86a61d9bccca13986c3855";
+  requestsVersion = "2.10.0";
+  requestsHash = "a36f7a64600f1bfec4d55ae021d232ae";
 
 
   # --- wheels used to bootstrap python environment ---------------------------
@@ -99,14 +96,9 @@ rec {
   #   md5 = effectHash;
   # };
 
-  chardet = fetchurl {
-    url = "https://pypi.python.org/packages/7d/87/4e3a3f38b2f5c578ce44f8dc2aa053217de9f0b6d737739b0ddac38ed237/chardet-${chardetVersion}.tar.gz";
-    md5 = chardetHash;
-  };
-
-  aiohttp = fetchurl {
-    url = "https://pypi.python.org/packages/04/78/9faeb8b5b1d53e8c81c99764412c2225d982943e4261bba2c6f725e15fce/aiohttp-${aiohttpVersion}.tar.gz";
-    md5 = aiodnsHash;
+  requests = fetchurl {
+    url = "https://pypi.python.org/packages/49/6f/183063f01aae1e025cf0130772b55848750a2f3a89bfa11b385b35d7329d/requests-${requestsVersion}.tar.gz";
+    md5 = requestsHash;
   };
 
 }

src/pypi2nix/stage2.py

@@ -1,12 +1,10 @@
 """Parse metadata from .dist-info directories in a wheelhouse."""
 
-import aiohttp
-import asyncio
 import click
-import glob
 import hashlib
 import json
-import os.path 
+import os.path
+import requests
 
 from pypi2nix.utils import TO_IGNORE, safe
 
@@ -57,7 +55,7 @@ def extract_deps(metadata):
     return list(set(deps))
 
 
-def metadata(wheel):
+def process_metadata(wheel):
     """Find the actual metadata json file from several possible names.
     """
     for _file in ('metadata.json', 'pydist.json'):
@@ -77,58 +75,38 @@ def metadata(wheel):
                         'description': safe(metadata.get('summary', '')),
                     }
     raise click.ClickException(
-        "Unable to find metadata.json/pydist.json in `%s` folder." %  wheel)
+        "Unable to find metadata.json/pydist.json in `%s` folder." % wheel)
 
 
-async def fetch_metadata(session, url):
-    """Fetch page asynchronously.
+def download_file(url, filename, chunk_size=1024):
+    r = requests.get(url, stream=True)
+    r.raise_for_status()  # TODO: handle this nicer
 
-    :param session: Session of client
-    :param url: Requested url
-    """
-    async with session.get(url) as response:
-        with aiohttp.Timeout(2):
-            async with session.get(url) as response:
-                assert response.status == 200
-                return await response.json()
+    with open(filename, 'wb') as fd:
+        for chunk in r.iter_content(chunk_size):
+            fd.write(chunk)
 
 
-def fetch_all_metadata(cache_dir, packages, index=INDEX_URL):
-    """Yield JSON information obtained from PyPI index given an iterable of
-       package names.
+def process_wheel(cache_dir, wheel, index=INDEX_URL):
+    """
     """
-    loop = asyncio.get_event_loop()
-    conn = aiohttp.TCPConnector(verify_ssl=False)
-    with aiohttp.ClientSession(loop=loop, connector=conn) as session:
-        for package in packages:
-            url = "{}/{}/json".format(index, package['name'])
-            yield combine_metadata(cache_dir, session, loop, package, loop.run_until_complete(fetch_metadata(session, url)))
-    loop.close()
-
-
-async def download_file(session, url, filename, chunk_size=1024):
-    async with session.get(url) as resp:
-        with open(filename, 'wb') as f:
-            while True:
-                chunk = await resp.content.read(chunk_size)
-                if not chunk:
-                    break
-                f.write(chunk)
-
-
-def combine_metadata(cache_dir, session, loop, old, new):
-    if not new.get('releases'):
+
+    url = "{}/{}/json".format(index, wheel['name'])
+    r = requests.get(url)
+    r.raise_for_status()  # TODO: handle this nicer
+    wheel_data = r.json()
+
+    if not wheel_data.get('releases'):
         raise click.ClickException(
-            "Unable to find releases for packge {name}".format(**old))
+            "Unable to find releases for packge {name}".format(**wheel))
 
-    if not new['releases'].get(old['version']):
+    if not wheel_data['releases'].get(wheel['version']):
         raise click.ClickException(
             "Unable to find releases for package {name} of version "
-            "{version}".format(**old))
+            "{version}".format(**wheel))
 
     release = None
-    releases = new['releases'][old['version']]
-    for possible_release in releases:
+    for possible_release in wheel_data['releases'][wheel['version']]:
         for extension in EXTENSIONS:
             if possible_release['url'].endswith(extension):
                 release = dict()
@@ -142,11 +120,12 @@ def combine_metadata(cache_dir, session, loop, old, new):
                     filename = os.path.join(
                         cache_dir, possible_release['filename'])
                     if not os.path.exists(filename):
-                        loop.run_until_complete(download_file(session, possible_release['url'], filename))
+                        download_file(possible_release['url'], filename)
 
                     # calculate sha256
                     with open(filename, 'rb') as f:
-                        release['hash_value'] = hashlib.sha256(f.read()).hexdigest()
+                        hash = hashlib.sha256(f.read())
+                    release['hash_value'] = hash.hexdigest()
 
             if release:
                 break
@@ -156,22 +135,27 @@ def combine_metadata(cache_dir, session, loop, old, new):
     if not release:
         raise click.ClickException(
             "Unable to find source releases for package {name} of version "
-            "{version}".format(**old))
+            "{version}".format(**wheel))
 
-    old.update(release)
+    if release:
+        wheel.update(release)
 
-    return old
+    return wheel
 
 
-def main(wheels, cache_dir):
+def main(wheels, cache_dir, index=INDEX_URL):
     """Extract packages metadata from wheels dist-info folders.
     """
 
-    wheels_metadata = []
+    metadata = []
     for wheel in wheels:
+
         click.echo('|-> from %s' % os.path.basename(wheel))
-        wheel_metadata = metadata(wheel)
-        if wheel_metadata:
-            wheels_metadata.append(wheel_metadata)
 
-    return list(fetch_all_metadata(cache_dir, wheels_metadata))
+        wheel_metadata = process_metadata(wheel)
+        if not wheel_metadata:
+            continue
+
+        metadata.append(process_wheel(cache_dir, wheel_metadata, index))
+
+    return metadata