Black all the Python code (mozilla-services#1005)

* Black all the Python code

Fixes #998

* ignore .cache/

* something for black to notice (test)

* undo black breakage

* documenting use of black

Author: Peter Bengtsson

.circleci/config.yml

@@ -58,6 +58,11 @@ jobs:
           command: |
             make test
 
+      - run:
+          name: Run lint check
+          command: |
+            docker-compose run test-ci lintcheck
+
       - run:
           name: Linting check for frontend
           command: |

setup.cfg .flake8

@@ -1,13 +1,15 @@
 [flake8]
+max-line-length = 88
 exclude=
     # ignore the migrations since they are created faulty by default
     tecken/*/migrations/*,
     # No use in checking the Node modules
-    node_modules/*/*/*,
+    frontend/node_modules/*/*/*,
     # No need to traverse our git directory
     .git,
     # There's no value in checking cache directories
     __pycache__,
     # Sphinx docs is best left as is
     docs/conf.py,
     hack.*py,
+    tecken/wsgi.py,

.gitignore

@@ -27,3 +27,4 @@ test-upload-inbox/
 upload-inbox/
 miniodata/
 .pytest_cache/
+.cache/

Dockerfile

@@ -14,7 +14,6 @@ WORKDIR /app
 RUN bin/build_frontend.sh
 
 FROM python:3.6-slim@sha256:5a96684a1729acd0680b39a3c24ef33f36bb6951873f77cde1e227b059a0f881
-MAINTAINER Peter Bengtsson <[email protected]>
 
 ENV PYTHONUNBUFFERED=1 \
     PYTHONPATH=/app/ \
@@ -33,13 +32,13 @@ RUN mkdir /app && \
 # install a few essentials and clean apt caches afterwards
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        apt-transport-https build-essential curl git libpq-dev \
-        gettext libffi-dev jed
+    apt-transport-https build-essential curl git libpq-dev \
+    gettext libffi-dev jed
 
 # Install dump_syms
 RUN DEBIAN_FRONTEND=noninteractive \
     apt-get install -y --no-install-recommends \
-        gyp ninja-build binutils-gold gcc-6 g++-6 pkg-config cabextract
+    gyp ninja-build binutils-gold gcc-6 g++-6 pkg-config cabextract
 COPY ./docker/build_dump_syms.sh /tmp
 RUN /tmp/build_dump_syms.sh
 
@@ -50,6 +49,7 @@ RUN apt-get autoremove -y && \
 
 # Install Python dependencies
 COPY requirements.txt /tmp/
+COPY requirements-constraints.txt /tmp/
 # Switch to /tmp to install dependencies outside home dir
 WORKDIR /tmp
 RUN pip install --no-cache-dir -r requirements.txt

Makefile

@@ -17,8 +17,8 @@ help:
 	@echo "  systemtest       Run system tests against a running tecken"
 	@echo "  django-shell     Django integrative shell"
 	@echo "  psql             Open the psql cli"
-	@echo "  lint-frontend    Runs a linting check on the frontend"
-	@echo "  lint-frontend-ci Runs a linting check on the frontend in CI"
+	@echo "  lintcheck        Check that the code is well formatted"
+	@echo "  lintfix          Fix all the possible linting errors"
 	@echo "  build-frontend   Builds the frontend static files\n"
 
 # Dev configuration steps
@@ -81,11 +81,19 @@ systemtest: .env .docker-build
 tag:
 	@bin/make-tag.py
 
-lint-frontend:
-	docker-compose run frontend lint
+#lint-frontend:
+#	docker-compose run frontend lint
 
-lint-frontend-ci:
-	docker-compose run frontend-ci lint
+#lint-frontend-ci:
+#	docker-compose run frontend-ci lint
 
 build-frontend:
 	docker-compose run -u 0 -e CI base ./bin/build_frontend.sh
+
+lintcheck: .env .docker-build
+	docker-compose run web lintcheck
+	docker-compose run frontend lint
+
+lintfix: .env .docker-build
+	docker-compose run web blackfix
+	docker-compose run frontend lintfix

README.md

@@ -1,20 +1,16 @@
-Tecken - All things Mozilla Symbol Server
-=========================================
+# Tecken - All things Mozilla Symbol Server
 
 [![CircleCI](https://circleci.com/gh/mozilla-services/tecken.svg?style=svg)](https://circleci.com/gh/mozilla-services/tecken)
 [![Updates](https://pyup.io/repos/github/mozilla-services/tecken/shield.svg)](https://pyup.io/repos/github/mozilla-services/tecken/)
 [![Renovate enabled](https://img.shields.io/badge/renovate-enabled-brightgreen.svg)](https://renovateapp.com/)
 [![What's Deployed](https://img.shields.io/badge/whatsdeployed-dev,stage,prod-green.svg)](https://whatsdeployed.io/s-5HY)
-
-
+[![Code style](https://img.shields.io/badge/Code%20style-black-000000.svg)](https://github.com/ambv/black)
 
 Please use the documentation on: **https://tecken.readthedocs.io**
 
 The **production server** is: https://symbols.mozilla.org
 
-
-To Get Coding
--------------
+## To Get Coding
 
 You need to be able to run Docker.
 
@@ -29,9 +25,7 @@ To test the symbolication run:
 
     curl -d '{"stacks":[[[0,11723767],[1, 65802]]],"memoryMap":[["xul.pdb","44E4EC8C2F41492B9369D6B9A059577C2"],["wntdll.pdb","D74F79EB1F8D4A45ABCD2F476CCABACC2"]],"version":4}' http://localhost:8000
 
-
-Datadog
--------
+## Datadog
 
 If you have access to a Mozilla Cloud Ops Datadog account, use this to
 consume the metrics Tecken sends via `statsd`. One is for staying health, the other is for
@@ -47,25 +41,20 @@ keeping track how it does things.
 
 [Symbols RDS](https://app.datadoghq.com/screen/280710/symbols-rds)
 
-New Relic
----------
+## New Relic
 
-This requires you have access to the 
+This requires you have access to the
 [Mozilla_25 New Relic account](https://rpm.newrelic.com/accounts/1402187/applications).
 
 [symbols-prod](https://rpm.newrelic.com/accounts/1402187/applications/62681492)
 
 [symbols-stage](https://rpm.newrelic.com/accounts/1402187/applications/52227224)
 
-
-Whatsdeployed
--------------
+## Whatsdeployed
 
 Check out https://whatsdeployed.io/s-5HY
 
-
-The Logo
---------
+## The Logo
 
 ![logo](logo.png "The Logo")
 

bin/run.sh

@@ -9,7 +9,7 @@ set -eo pipefail
 : "${GUNICORN_TIMEOUT:=300}"
 
 usage() {
-  echo "usage: ./bin/run.sh web|web-dev|worker|test|bash|superuser"
+  echo "usage: ./bin/run.sh web|web-dev|worker|test|bash|blackfix|lintcheck|superuser"
   exit 1
 }
 
@@ -55,13 +55,22 @@ case $1 in
     # the docker container.
     exec celery -A tecken.celery:app worker -l info --purge
     ;;
+  blackfix)
+    # This exclude is ugly because it's not additive
+    # See https://github.com/ambv/black/issues/65
+    black tecken tests --exclude '/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist|migrations)/'
+    ;;
+  lintcheck)
+    flake8 tecken tests
+    black --check tecken tests --exclude '/(\.git|\.hg|\.mypy_cache|\.tox|\.venv|_build|buck-out|build|dist|migrations)/'
+    ;;
   superuser)
     exec python manage.py superuser "${@:2}"
     ;;
   test)
     # python manage.py collectstatic --noinput
     coverage erase
-    coverage run -m py.test --flake8 "${@:2}"
+    coverage run -m py.test "${@:2}"
     coverage report -m
     if [[ -z ${CI+check} ]]; then  # when doing local `make test`
       # generate code coverage to disk

docs/dev.rst

@@ -21,15 +21,14 @@ It will exit non-zero if there are files lacking the preamble. It only
 checks git checked in files.
 
 PEP8 is nice. All files are expected to be PEP8 and pyflakes compliant
-and the PEP8 rules (and exceptions) are defined in ``setup.cfg`` under
+and the PEP8 rules (and exceptions) are defined in ``.flake8`` under
 the ``[flake8]`` heading.
 
-The tests use ``py.test`` with a plugin called ``flake8`` which will
-check files according to the flake8 rules as it runs tests.
-
 If you hit issues, instead of re-writing the rules consider
 appending a comment on the end of the line that says ``# noqa``.
 
+All Python code is and should be formatted with `black <https://github.com/ambv/black>`_.
+
 Local development
 =================
 
@@ -652,3 +651,24 @@ if there were multiple files:
     $ zip mysymbols -r *
     $ ls -l mysymbols.zip
     -rw-r--r--  1 peterbe  staff  40945250 Aug 10 14:54 mysymbols.zip
+
+
+``black``
+=========
+
+`black <https://github.com/ambv/black>`_. is the Python code formatting tool we use
+to format all non-generated Python code. In CI, we test that all code passes
+``black --check ...``. When doing local development, consider setting up either
+some sort of "format on save" in your editor or a git pre-commit hook.
+
+To check that all code is formatted correctly, run:
+
+.. code-block:: shell
+
+    $ docker-compose run web lintcheck
+
+If you have a bunch of formatting complaints you can automatically fix them all with:
+
+.. code-block:: shell
+
+    $ docker-compose run web blackfix

requirements-constraints.txt

@@ -0,0 +1,8 @@
+click==6.7 \
+    --hash=sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d \
+    --hash=sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b
+toml==0.9.4 \
+    --hash=sha256:8e86bd6ce8cc11b9620cb637466453d94f5d57ad86f17e98a98d1f73e3baab2d
+appdirs==1.4.3 \
+    --hash=sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92 \
+    --hash=sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e

requirements.txt

@@ -1,3 +1,5 @@
+-c requirements-constraints.txt
+
 Django==2.0.7 \
     --hash=sha256:97886b8a13bbc33bfeba2ff133035d3eca014e2309dff2b6da0bdfc0b8656613 \
     --hash=sha256:e900b73beee8977c7b887d90c6c57d68af10066b9dac898e1eaf0f82313de334
@@ -357,9 +359,6 @@ pytest-mock==1.7.1 \
 requests-mock==1.4.0 \
     --hash=sha256:96a1e45b1c0bd18d14fcb2d55b3b09d6d46237e37bcae3155df4cb75bc42619e \
     --hash=sha256:2931887853c42e1d73879983d5bf03041109472991c5b4b8dba5d11ed23b9d0b
-pytest-flake8==0.9.1 \
-    --hash=sha256:e67686645860009cf1f9a5016e110234b4a2c4584711d62ba753ed651ab9ab28 \
-    --hash=sha256:e716072d07a557defdd5c4141984569731e292961370a5663c1697283aa16200
 mccabe==0.6.1 \
     --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
     --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
@@ -399,3 +398,6 @@ pycryptodome==3.5.1 \
     --hash=sha256:90c1c08431ccf8d6385b5071e5d3f7ecbf85dbd1cbd45439fe136bc514c73136 \
     --hash=sha256:008e0bfc02f8c409fbe31a1dd1ff3aa18b5c109b47a05603ca734506a7b3a8cb \
     --hash=sha256:b7957736f5e868416b06ff033f8525e64630c99a8880b531836605190b0cac96
+black==18.6b4 \
+    --hash=sha256:22158b89c1a6b4eb333a1e65e791a3f8b998cf3b11ae094adb2570f31f769a44 \
+    --hash=sha256:4b475bbd528acce094c503a3d2dbc2d05a4075f6d0ef7d9e7514518e14cc5191

tecken/__init__.py

@@ -3,9 +3,9 @@
 # file, you can obtain one at http://mozilla.org/MPL/2.0/.
 from .celery import app as celery_app
 
-default_app_config = 'tecken.apps.TeckenAppConfig'
+default_app_config = "tecken.apps.TeckenAppConfig"
 
 
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
-__all__ = ['celery_app', 'default_app_config']
+__all__ = ["celery_app", "default_app_config"]