Added package hashing logic

Fixes #222

pypicloud/__init__.py

@@ -23,14 +23,16 @@ def to_json(value):
     return render("json", value)
 
 
-def _app_url(request, *paths, **params):
+def _app_url(request, *paths, fragment="", **params):
     """ Get the base url for the root of the app plus an optional path """
     path = "/".join(paths)
     if not path.startswith("/"):
         path = "/" + path
     if params:
         path += "?" + urlencode(params)
-    return request.application_url + path
+    if fragment:
+        fragment = "#" + fragment
+    return request.application_url + path + fragment
 
 
 def _fallback_simple(request):

pypicloud/access/base.py

@@ -46,7 +46,7 @@ def get_pwd_context(
     """ Create a passlib context for hashing passwords """
     if preferred_hash is None or preferred_hash == "sha":
         preferred_hash = "sha256_crypt" if sys_bits < 64 else "sha512_crypt"
-    if preferred_hash is "pbkdf2":
+    if preferred_hash == "pbkdf2":
         preferred_hash = "pbkdf2_sha256" if sys_bits < 64 else "pbkdf2_sha512"
 
     if preferred_hash not in SCHEMES:

pypicloud/cache/base.py

@@ -1,7 +1,8 @@
 """ Base class for all cache implementations """
 from typing import BinaryIO, Callable, Optional, List, Dict, Any, Tuple
 from datetime import datetime
-
+from io import BytesIO
+import hashlib
 import logging
 from pyramid.settings import asbool
 
@@ -17,10 +18,15 @@ class ICache(object):
 
     """ Base class for a caching database that stores package metadata """
 
-    def __init__(self, request=None, storage=None, allow_overwrite=None):
+    package_class = Package
+
+    def __init__(
+        self, request=None, storage=None, allow_overwrite=None, calculate_hashes=True
+    ):
         self.request = request
         self.storage = storage(request)
         self.allow_overwrite = allow_overwrite
+        self.calculate_hashes = calculate_hashes
 
     def new_package(self, *args, **kwargs):
         return Package(*args, **kwargs)
@@ -43,6 +49,9 @@ def configure(cls, settings):
         return {
             "storage": get_storage_impl(settings),
             "allow_overwrite": asbool(settings.get("pypi.allow_overwrite", False)),
+            "calculate_hashes": asbool(
+                settings.get("pypi.calculate_package_hashes", True)
+            ),
         }
 
     @classmethod
@@ -121,10 +130,17 @@ def upload(
         name = normalize_name(name)
         filename = posixpath.basename(filename)
         old_pkg = self.fetch(filename)
+        metadata = {"requires_python": requires_python}
         if old_pkg is not None and not self.allow_overwrite:
             raise ValueError("Package '%s' already exists!" % filename)
-        new_pkg = self.new_package(
-            name, version, filename, summary=summary, requires_python=requires_python
+        if self.calculate_hashes:
+            file_data = data.read()
+            metadata["hash_sha256"] = hashlib.sha256(file_data).hexdigest()
+            metadata["hash_md5"] = hashlib.md5(file_data).hexdigest()
+            data = BytesIO(file_data)
+
+        new_pkg = self.package_class(
+            name, version, filename, summary=summary, **metadata
         )
         self.storage.upload(new_pkg, data)
         self.save(new_pkg)

pypicloud/models.py

@@ -8,7 +8,7 @@
 from .util import normalize_name
 
 
-METADATA_FIELDS = ["requires_python", "summary"]
+METADATA_FIELDS = ["requires_python", "summary", "hash_sha256", "hash_md5"]
 
 
 @total_ordering

pypicloud/storage/base.py

@@ -1,4 +1,5 @@
 """ Base class for storage backends """
+from pyramid.request import Request
 from typing import Type, List, BinaryIO, Tuple
 from pypicloud.models import Package
 
@@ -7,7 +8,7 @@ class IStorage(object):
 
     """ Base class for a backend that stores package files """
 
-    def __init__(self, request):
+    def __init__(self, request: Request):
         self.request = request
 
     @classmethod
@@ -33,7 +34,12 @@ def get_url(self, package: Package) -> str:
             Link to the location of this package file
 
         """
-        return self.request.app_url("api", "package", package.name, package.filename)
+        fragment = ""
+        if package.data.get("hash_sha256"):
+            fragment = "sha256=" + package.data["hash_sha256"]
+        return self.request.app_url(
+            "api", "package", package.name, package.filename, fragment=fragment
+        )
 
     def download_response(self, package: Package):
         """

pypicloud/views/simple.py

@@ -127,13 +127,16 @@ def package_versions_json(context, request):
         if max_version is None or version > max_version:
             max_version = version
 
-        response["releases"].setdefault(version_str, []).append(
-            {
-                "filename": filename,
-                "url": pkg["url"],
-                "requires_python": pkg["requires_python"],
-            }
-        )
+        release = {
+            "filename": filename,
+            "url": pkg["non_hashed_url"],
+            "requires_python": pkg["requires_python"],
+        }
+        if pkg.get("hash_sha256"):
+            release["digests"] = {"md5": pkg["hash_md5"], "sha256": pkg["hash_sha256"]}
+            release["md5_digest"] = pkg["hash_md5"]
+
+        response["releases"].setdefault(version_str, []).append(release)
     if max_version is not None:
         response["urls"] = response["releases"].get(str(max_version), [])
     return response
@@ -161,9 +164,18 @@ def packages_to_dict(request, packages):
     """ Convert a list of packages to a dict used by the template """
     pkgs = {}
     for package in packages:
+        url = package.get_url(request)
+        # We could also do with a url without the sha256 fragment for the JSON api
+        non_fragment_url = url
+        if "#sha256=" in url:
+            non_fragment_url = non_fragment_url[: url.find("#sha256=")]
+
         pkgs[package.filename] = {
-            "url": package.get_url(request),
+            "url": url,
+            "non_hashed_url": non_fragment_url,
             "requires_python": package.data.get("requires_python"),
+            "hash_sha256": package.data.get("hash_sha256"),
+            "hash_md5": package.data.get("hash_md5"),
         }
     return pkgs