Django 2.2 (#7196)

* Complete removal of nose test artifacts

* Update base requirements to Django 2.2 and python 3 compatibile packages

Author: SmileyChris

.dockerignore

@@ -1,2 +1,4 @@
 .git
 .env
+**/__pycache__
+**/*.pyc

Dockerfile

@@ -23,7 +23,7 @@ RUN gulp build --production
 ########
 # Python dependencies builder
 #
-FROM python:2-stretch AS python-builder
+FROM python:3-slim AS python-builder
 
 WORKDIR /app
 ENV LANG=C.UTF-8
@@ -34,6 +34,7 @@ ENV PATH="/venv/bin:$PATH"
 COPY docker/bin/apt-install /usr/local/bin/
 RUN apt-install gettext build-essential libxml2-dev libxslt1-dev libxslt1.1
 
+RUN pip install virtualenv
 RUN virtualenv /venv
 
 COPY requirements/base.txt requirements/prod.txt ./requirements/
@@ -45,7 +46,7 @@ RUN pip install --no-cache-dir -r requirements/prod.txt
 ########
 # django app container
 #
-FROM python:2-slim-stretch AS app-base
+FROM python:3-slim AS app-base
 
 # Extra python env
 ENV PYTHONDONTWRITEBYTECODE=1

Makefile

@@ -79,7 +79,7 @@ clean:
 	git clean -f
 
 lint: .docker-build-pull
-	${DC} run test flake8 bedrock lib tests
+	${DC} run test flake8
 	${DC} run assets gulp js:lint css:lint
 
 test: .docker-build-pull

bedrock/base/cache.py

@@ -12,7 +12,7 @@ class SimpleDictCache(LocMemCache):
     def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
         key = self.make_key(key, version=version)
         self.validate_key(key)
-        with self._lock.writer():
+        with self._lock:
             if self._has_expired(key):
                 self._set(key, value, timeout)
                 return True
@@ -22,13 +22,13 @@ def get(self, key, default=None, version=None):
         key = self.make_key(key, version=version)
         self.validate_key(key)
         value = default
-        with self._lock.reader():
+        with self._lock:
             if not self._has_expired(key):
                 value = self._cache[key]
         if value is not default:
             return value
 
-        with self._lock.writer():
+        with self._lock:
             try:
                 del self._cache[key]
                 del self._expire_info[key]
@@ -39,7 +39,7 @@ def get(self, key, default=None, version=None):
     def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
         key = self.make_key(key, version=version)
         self.validate_key(key)
-        with self._lock.writer():
+        with self._lock:
             self._set(key, value, timeout)
 
     def incr(self, key, delta=1, version=None):
@@ -48,6 +48,6 @@ def incr(self, key, delta=1, version=None):
             raise ValueError("Key '%s' not found" % key)
         new_value = value + delta
         key = self.make_key(key, version=version)
-        with self._lock.writer():
+        with self._lock:
             self._cache[key] = new_value
         return new_value

bedrock/base/log_settings.py

@@ -91,7 +91,7 @@ def emit(self, record):
         cfg[key] = value
 
 # Set the level and handlers for all loggers.
-for logger in cfg['loggers'].values() + [cfg['root']]:
+for logger in list(cfg['loggers'].values()) + [cfg['root']]:
     if 'handlers' not in logger:
         logger['handlers'] = ['syslog' if use_syslog else 'console']
     if 'level' not in logger:

bedrock/base/management/commands/update_www_config.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 
 import os
 
@@ -27,7 +26,7 @@ def refresh_db_values():
 
     ConfigValue.objects.all().delete()
     count = 0
-    for name, value in values.iteritems():
+    for name, value in values.items():
         if value:
             ConfigValue.objects.create(name=name, value=value)
             count += 1

bedrock/base/middleware.py

@@ -5,30 +5,41 @@
 the locale codes.
 """
 import base64
-import urllib
+import urllib.parse
+from urllib.parse import unquote
 from warnings import warn
 
+from commonware.middleware import FrameOptionsHeader as OldFrameOptionsHeader
+from commonware.middleware import RobotsTagHeader as OldRobotsTagHeader
 from django.conf import settings
 from django.core.exceptions import MiddlewareNotUsed
-from django.http import HttpResponsePermanentRedirect, HttpResponse
-from django.utils.encoding import force_text
+from django.http import HttpResponse, HttpResponsePermanentRedirect
+from django.utils.deprecation import MiddlewareMixin
 
-from . import urlresolvers
 from lib.l10n_utils import translation
 
+from . import urlresolvers
+
 
-class LocaleURLMiddleware(object):
+class LocaleURLMiddleware:
     """
     1. Search for the locale.
     2. Save it in the request.
     3. Strip them from the URL.
     """
 
-    def __init__(self):
+    def __init__(self, get_response=None):
         if not settings.USE_I18N or not settings.USE_L10N:
             warn("USE_I18N or USE_L10N is False but LocaleURLMiddleware is "
                  "loaded. Consider removing bedrock.base.middleware."
-                 "LocaleURLMiddleware from your MIDDLEWARE_CLASSES setting.")
+                 "LocaleURLMiddleware from your MIDDLEWARE setting.")
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.process_request(request)
+        if response:
+            return response
+        return self.get_response(request)
 
     def process_request(self, request):
         prefixer = urlresolvers.Prefixer(request)
@@ -37,11 +48,11 @@ def process_request(self, request):
 
         if full_path != request.path:
             query_string = request.META.get('QUERY_STRING', '')
-            full_path = urllib.quote(full_path.encode('utf-8'))
+            full_path = urllib.parse.quote(full_path.encode('utf-8'))
 
             if query_string:
                 full_path = '?'.join(
-                    [full_path, force_text(query_string, errors='ignore')])
+                    [full_path, unquote(query_string, errors='ignore')])
 
             response = HttpResponsePermanentRedirect(full_path)
 
@@ -58,14 +69,21 @@ def process_request(self, request):
         translation.activate(prefixer.locale or settings.LANGUAGE_CODE)
 
 
-class BasicAuthMiddleware(object):
+class BasicAuthMiddleware:
     """
     Middleware to protect the entire site with a single basic-auth username and password.
     Set the BASIC_AUTH_CREDS environment variable to enable.
     """
-    def __init__(self):
+    def __init__(self, get_response=None):
         if not settings.BASIC_AUTH_CREDS:
             raise MiddlewareNotUsed
+        self.get_response = None
+
+    def __call__(self, request):
+        response = self.process_request(request)
+        if response:
+            return response
+        return self.get_response(request)
 
     def process_request(self, request):
         required_auth = settings.BASIC_AUTH_CREDS
@@ -85,3 +103,11 @@ def process_request(self, request):
             realm = settings.APP_NAME or 'bedrock-demo'
             response['WWW-Authenticate'] = 'Basic realm="{}"'.format(realm)
             return response
+
+
+class RobotsTagHeader(OldRobotsTagHeader, MiddlewareMixin):
+    pass
+
+
+class FrameOptionsHeader(OldFrameOptionsHeader, MiddlewareMixin):
+    pass

bedrock/base/migrations/0001_initial.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
 
 from django.db import migrations, models
 

bedrock/base/models.py

@@ -8,7 +8,7 @@ class ConfigValue(models.Model):
     class Meta:
         app_label = 'base'
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s=%s' % (self.name, self.value)
 
 

bedrock/base/templatetags/helpers.py

@@ -1,6 +1,5 @@
 import datetime
-import urllib
-import urlparse
+import urllib.parse
 
 from django.conf import settings
 from django.contrib.staticfiles.storage import staticfiles_storage
@@ -73,43 +72,43 @@ def urlparams(url_, hash=None, **query):
     New query params will be appended to exising parameters, except duplicate
     names, which will be replaced.
     """
-    url = urlparse.urlparse(url_)
+    url = urllib.parse.urlparse(url_)
     fragment = hash if hash is not None else url.fragment
 
     # Use dict(parse_qsl) so we don't get lists of values.
     q = url.query
-    query_dict = dict(urlparse.parse_qsl(smart_str(q))) if q else {}
+    query_dict = dict(urllib.parse.parse_qsl(smart_str(q))) if q else {}
     query_dict.update((k, v) for k, v in query.items())
 
     query_string = _urlencode([(k, v) for k, v in query_dict.items()
                                if v is not None])
-    new = urlparse.ParseResult(url.scheme, url.netloc, url.path, url.params,
-                               query_string, fragment)
+    new = urllib.parse.ParseResult(
+        url.scheme, url.netloc, url.path, url.params, query_string, fragment)
     return new.geturl()
 
 
 def _urlencode(items):
     """A Unicode-safe URLencoder."""
     try:
-        return urllib.urlencode(items)
+        return urllib.parse.urlencode(items)
     except UnicodeEncodeError:
-        return urllib.urlencode([(k, smart_str(v)) for k, v in items])
+        return urllib.parse.urlencode([(k, smart_str(v)) for k, v in items])
 
 
 @library.filter
 def mailtoencode(txt):
     """Url encode a string using %20 for spaces."""
-    if isinstance(txt, unicode):
+    if isinstance(txt, str):
         txt = txt.encode('utf-8')
-    return urllib.quote(txt)
+    return urllib.parse.quote(txt)
 
 
 @library.filter
 def urlencode(txt):
     """Url encode a string using + for spaces."""
-    if isinstance(txt, unicode):
+    if isinstance(txt, str):
         txt = txt.encode('utf-8')
-    return urllib.quote_plus(txt)
+    return urllib.parse.quote_plus(txt)
 
 
 @library.global_function

bedrock/base/tests/test_accepted_locales.py

@@ -82,9 +82,9 @@ def test_dev_languages(self):
         # simulate the successful result of the DEV_LANGUAGES list
         # comprehension defined in settings.
         settings.DEV_LANGUAGES = ['en-US', 'fr']
-        assert settings.LANGUAGE_URL_MAP == {'en-us': 'en-US', 'fr': 'fr'}, \
-               ('DEV is True, but DEV_LANGUAGES are not used to define the '
-                'allowed locales.')
+        assert settings.LANGUAGE_URL_MAP == {'en-us': 'en-US', 'fr': 'fr'}, (
+            'DEV is True, but DEV_LANGUAGES are not used to define the '
+            'allowed locales.')
 
     def test_prod_languages(self):
         """Test the accepted locales on prod instances.

bedrock/base/tests/test_middleware.py

@@ -1,3 +1,5 @@
+from urllib.parse import urlencode
+
 from django.test import TestCase, RequestFactory
 from django.test.utils import override_settings
 
@@ -15,7 +17,7 @@ def test_redirects_to_correct_language(self):
         """Should redirect to lang prefixed url."""
         path = '/the/dude/'
         req = self.rf.get(path, HTTP_ACCEPT_LANGUAGE='de')
-        resp = LocaleURLMiddleware().process_request(req)
+        resp = self.middleware.process_request(req)
         self.assertEqual(resp['Location'], '/de' + path)
 
     @override_settings(DEV_LANGUAGES=('es', 'fr'),
@@ -24,17 +26,18 @@ def test_redirects_to_default_language(self):
         """Should redirect to default lang if not in settings."""
         path = '/the/dude/'
         req = self.rf.get(path, HTTP_ACCEPT_LANGUAGE='de')
-        resp = LocaleURLMiddleware().process_request(req)
+        resp = self.middleware.process_request(req)
         self.assertEqual(resp['Location'], '/en-US' + path)
 
     @override_settings(DEV_LANGUAGES=('de', 'fr'))
     def test_redirects_to_correct_language_despite_unicode_errors(self):
         """Should redirect to lang prefixed url, stripping invalid chars."""
         path = '/the/dude/'
-        corrupt_querystring = '?a\xa4\x91b\xa4\x91i\xc0de=s'
+        corrupt_querystring = '?' + urlencode(
+            {b'a\xa4\x91b\xa4\x91i\xc0de': 's'})
         corrected_querystring = '?abide=s'
         req = self.rf.get(path + corrupt_querystring,
                           HTTP_ACCEPT_LANGUAGE='de')
-        resp = LocaleURLMiddleware().process_request(req)
+        resp = self.middleware.process_request(req)
         self.assertEqual(resp['Location'],
                          '/de' + path + corrected_querystring)