fix: strip chr prefix for grch38 (#10)

- Adapted docker build and github actions
- Performed isort

.github/workflows/docker-build.yml

@@ -0,0 +1,52 @@
+# This workflow is run as part of CI to test that they run through.
+#
+# The images are pushed to `ghcr.io` for each PR and branch.  The ones for
+# the releases are pushed in `release-please.yml`.
+name: Docker Build
+
+on:
+  - push
+  - pull_request
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Write VERSION file for Python package
+        run: |
+          git describe --all | tee VERSION
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: utils/docker/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/docker-cleanup-pr.yml

@@ -0,0 +1,19 @@
+name: Cleanup PR Images
+
+on:
+  pull_request:
+    types:
+        - closed
+
+jobs:
+  purge-image:
+    name: Delete PR images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bots-house/[email protected]
+        with:
+          owner: varfish-org
+          name: cadd-rest-api
+          token: ${{ secrets.GITHUB_TOKEN }}
+          tag: pr-${{github.event.pull_request.number}}
+        continue-on-error: true

.github/workflows/docker-cleanup-untagged.yml

@@ -0,0 +1,18 @@
+name: Cleanup Untagged Images
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * SUN"
+
+jobs:
+  delete-untagged-images:
+    name: Delete untagged images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bots-house/[email protected]
+        with:
+          owner: varfish-org
+          name: cadd-rest-api
+          token: ${{ secrets.GITHUB_TOKEN }}
+          untagged-keep-latest: 3

.gitignore

@@ -1,3 +1,5 @@
+/VERSION
+
 # SQLite database.
 /*.db
 

Makefile

@@ -1,23 +1,35 @@
 SHELL = /bin/bash
 MANAGE = time python manage.py
 
-.PHONY: black serve _migrate migrate celery
-
+.PHONY: black
 black:
 	black -l 100 --exclude '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.?v?env|_build|buck-out|build|dist|src)/' .
 
+.PHONY: isort
+isort:
+	isort --force-sort-within-sections --profile=black .
+
+.PHONY: flake8
+flake8:
+	flake8
+
+.PHONY: serve
 serve:
 	$(MANAGE) runserver
 
+.PHONY: _migrate
 _migrate:
 	$(MANAGE) makemigrations
 	$(MANAGE) migrate
 
+.PHONY: migrate
 migrate: _migrate black
 
+.PHONY: celery
 celery:
 	celery worker -A config.celery_app -l info --concurrency=4
 
+.PHONY: test
 test:
 	$(MANAGE) test --settings=config.settings.test -v2
 

README.rst

@@ -36,4 +36,4 @@ Or:
 
 .. code-block:: bash
 
-    $ docker build . --build-arg app_git_tag=v0.3.2 -t bihealth/cadd-rest-api:0.3.2-0
+    $ docker build . --build-arg app_git_tag=v0.3.2 -t varfish-org/cadd-rest-api:0.3.2-0

config/celery.py

@@ -1,4 +1,5 @@
 import os
+
 from celery import Celery
 
 # set the default Django settings module for the 'celery' program.

config/settings/base.py

@@ -162,7 +162,7 @@
 # Django Admin URL.
 ADMIN_URL = "admin/"
 # https://docs.djangoproject.com/en/dev/ref/settings/#admins
-ADMINS = [("""Oliver Stolpe""", "oliver.stolpe@bihealth.de")]
+ADMINS = [("""Oliver Stolpe""", "oliver.stolpe@bih-charite.de")]
 # https://docs.djangoproject.com/en/dev/ref/settings/#managers
 MANAGERS = ADMINS
 

config/settings/production.py

@@ -73,7 +73,7 @@
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#default-from-email
 DEFAULT_FROM_EMAIL = env(
-    "DJANGO_DEFAULT_FROM_EMAIL", default="CADD REST API <noreply@bihealth.org>"
+    "DJANGO_DEFAULT_FROM_EMAIL", default="CADD REST API <noreply@bih-charite.org>"
 )
 # https://docs.djangoproject.com/en/dev/ref/settings/#server-email
 SERVER_EMAIL = env("DJANGO_SERVER_EMAIL", default=DEFAULT_FROM_EMAIL)

config/urls.py

@@ -1,6 +1,5 @@
 from django.conf import settings
-from django.conf.urls import url
-from django.conf.urls import include
+from django.conf.urls import include, url
 from django.conf.urls.static import static
 
 urlpatterns = [url(r"", include("restapi.urls"))] + static(

requirements/base.txt

@@ -24,3 +24,4 @@ djangorestframework==3.10.3  # https://github.com/encode/django-rest-framework
 coreapi==2.3.3  # https://github.com/core-api/python-client
 
 jsonfield==2.1.1
+setuptools<58.0.0

requirements/test.txt

@@ -10,7 +10,7 @@ pytest-sugar==0.9.2  # https://github.com/Frozenball/pytest-sugar
 # ------------------------------------------------------------------------------
 flake8==3.7.8  # https://github.com/PyCQA/flake8
 coverage==4.5.4  # https://github.com/nedbat/coveragepy
-black==19.3b0  # https://github.com/ambv/black
+black==22.3.0  # https://github.com/ambv/black
 pylint-django==2.0.11  # https://github.com/PyCQA/pylint-django
 pylint-celery==0.3  # https://github.com/PyCQA/pylint-celery
 

restapi/migrations/0001_initial.py

@@ -2,9 +2,10 @@
 # Generated by Django 1.11.24 on 2019-09-19 14:31
 from __future__ import unicode_literals
 
+import uuid
+
 from django.db import migrations, models
 import jsonfield.fields
-import uuid
 
 
 class Migration(migrations.Migration):

restapi/models.py

@@ -1,6 +1,7 @@
+import uuid
+
 from django.db import models
 import jsonfield
-import uuid
 
 STATUS_ACTIVE = "active"
 STATUS_FAILED = "failed"

restapi/tasks.py

@@ -4,17 +4,19 @@
 import subprocess  # nosec
 import tempfile
 
-from restapi.models import AnnotateBackgroundJob
-from config.celery import app
 from celery.exceptions import SoftTimeLimitExceeded
+
+from config.celery import app
 from config.settings.base import CADD_CONDA, CADD_SH, CADD_TIMEOUT
+from restapi.models import AnnotateBackgroundJob
 
 
 @app.task(bind=True)
 def annotate_background_job(_self, bgjob_uuid):
     """Task to execute a CADD scoring background job."""
     bgjob = AnnotateBackgroundJob.objects.get(uuid=bgjob_uuid)
     args = bgjob.args
+    chrom_stripped = False
 
     if not args["variants"]:  # no scores, nothing to do
         bgjob.scores = {}
@@ -28,6 +30,9 @@ def annotate_background_job(_self, bgjob_uuid):
             # Write out the input file for CADD.sh
             with open(os.path.join(tmpdir, "in.vcf"), "wt") as vcff:
                 for variant in args["variants"]:
+                    if variant.startswith("chr"):
+                        chrom_stripped = True
+                        variant = variant[3:]
                     print("%s\t%s\t.\t%s\t%s" % tuple(variant.split("-")), file=vcff)
             # Build command line to CADD.sh and execute.
             cmdline = [
@@ -61,12 +66,32 @@ def annotate_background_job(_self, bgjob_uuid):
                 raise
             # Check bash return code for validity, and raise exception if it is invalid.
             if return_code != 0:
+                print("[processed variants]")
+                for variant in args["variants"]:
+                    if args["genome_build"] == "GRCh38" and variant.startswith("chr"):
+                        variant = variant[3:]
+                    print("%s\t%s\t.\t%s\t%s" % tuple(variant.split("-")))
+                try:
+                    outs, errs = proc.communicate(timeout=15)
+                    print("[stdout]")
+                    print(outs.decode("utf-8"))
+                    print("[stderr]")
+                    print(errs.decode("utf-8"))
+                except subprocess.TimeoutExpired:
+                    proc.kill()
+                    outs, errs = proc.communicate()
+                    print("[stdout]")
+                    print(outs.decode("utf-8"))
+                    print("[stderr]")
+                    print(errs.decode("utf-8"))
                 # Write job status to database before raising.
                 bgjob.status = "failed"
-                bgjob.message = "Command line '{}' exited with error code {} and message: {}".format(
-                    " ".join(cmdline),
-                    return_code,
-                    " ".join(map(lambda x: x.decode(), proc.communicate())),
+                bgjob.message = (
+                    "Command line '{}' exited with error code {} and message: {}".format(
+                        " ".join(cmdline),
+                        return_code,
+                        " ".join(map(lambda x: x.decode(), proc.communicate())),
+                    )
                 )
                 bgjob.save()
                 raise subprocess.CalledProcessError(return_code, " ".join(cmdline))
@@ -83,6 +108,8 @@ def annotate_background_job(_self, bgjob_uuid):
                         header = row
                     elif header:
                         data = dict(zip(header, row))
+                        if chrom_stripped and not data["#Chrom"].startswith("chr"):
+                            data["#Chrom"] = f"chr{data['#Chrom']}"
                         key = "-".join((data[k] for k in ("#Chrom", "Pos", "Ref", "Alt")))
                         val = list(map(float, (data["RawScore"], data["PHRED"])))
                         scores[key] = val

restapi/tests.py

@@ -1,11 +1,11 @@
 import os
 from unittest.mock import patch
+import uuid
 
 from django.test import TestCase
 from django.urls import reverse
 
 from restapi.models import AnnotateBackgroundJob
-import uuid
 
 
 class TestBase(TestCase):
@@ -15,8 +15,7 @@ def setUp(self):
 
 
 class TestAnnotateApiView(TestBase):
-    """Tests for AnnotateApiView.
-    """
+    """Tests for AnnotateApiView."""
 
     # Patching the CADD_SH variable to point to the CADD.sh mocking script.
     # This script will return a valid CADD file with made-up data.
@@ -36,8 +35,7 @@ def test_post_annotate_request(self):
 
 
 class TestResultApiView(TestBase):
-    """Tests for ResultApiView.
-    """
+    """Tests for ResultApiView."""
 
     def test_bgjob_doesnt_exist(self):
         bgjob_uuid = str(uuid.uuid4())

restapi/urls.py

@@ -1,5 +1,6 @@
 from django.conf.urls import url
 from django.views.generic import TemplateView
+
 from . import views
 
 app_name = "restapi"