feat: remove old filter features (#739)

hgmd/admin.py

@@ -1,6 +0,0 @@
-from django.contrib import admin
-
-from .models import HgmdPublicLocus
-
-# Register your models here.
-admin.site.register(HgmdPublicLocus)

hgmd/migrations/0005_delete_hgmdpubliclocus.py

@@ -0,0 +1,16 @@
+# Generated by Django 3.2.25 on 2024-05-15 11:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("hgmd", "0004_auto_20190830_1126"),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name="HgmdPublicLocus",
+        ),
+    ]

hgmd/models.py

@@ -1,39 +0,0 @@
-"""Models for ``HGMD`` annotation in VarFish.
-
-At the moment (and for the forseeable future), only the ``HGMD_PUBLIC`` dump
-from ENSEMBL can be imported.
-"""
-
-from django.db import models
-from postgres_copy import CopyManager
-
-
-class HgmdPublicLocus(models.Model):
-    """Representation of an interval on the genome that has a HGMD_PUBLIC annotation.
-
-    The positions are 0-based as this comes from a BED file.
-    """
-
-    #: The genome release, e.g., ``"GRCh37"``.
-    release = models.CharField(max_length=32)
-    #: The chromosomal, e.g., ``"1"``.
-    chromosome = models.CharField(max_length=32)
-    #: The start position, 0-based.
-    start = models.IntegerField()
-    #: The end position, 0-based.
-    end = models.IntegerField()
-    #: The UCSC bin.
-    bin = models.IntegerField()
-    #: The ``variation_name`` column from ENSEMBL variation table.
-    variation_name = models.CharField(max_length=32)
-
-    #: Enable bulk-import.
-    objects = CopyManager()
-
-    class Meta:
-        indexes = [models.Index(fields=["release", "chromosome", "start", "end"])]
-
-    def __str__(self):
-        """String representation, e.g., used in Django admin."""
-        values = (self.release, self.chromosome, self.start, self.end, self.variation_name)
-        return "HgmdPublicLocus({})".format(", ".join(map(repr, values)))

hgmd/tests/factories.py

@@ -1,26 +0,0 @@
-"""Factory Boy factory classes for ``hgmd``."""
-
-import binning
-import factory
-
-from ..models import HgmdPublicLocus
-
-
-class HgmdPublicLocusFactory(factory.django.DjangoModelFactory):
-    """Factory for ``HgmdPublicLocus`` records."""
-
-    class Meta:
-        model = HgmdPublicLocus
-
-    release = "GRCh37"
-    chromosome = factory.Iterator((list(map(str, range(1, 23))) + ["X", "Y"]))
-    start = factory.Sequence(lambda n: (n + 1) * 100 - 1)
-    end = factory.LazyAttribute(lambda o: o.start + 1)
-    bin = 0
-
-    variation_name = factory.Sequence(lambda n: "CD12345%d" % n)
-
-    @factory.post_generation
-    def fix_bins(obj, *args, **kwargs):
-        obj.bin = binning.assign_bin(obj.start - 1, obj.end)
-        obj.save()

importer/management/commands/import_tables.py

@@ -49,7 +49,6 @@
     TadSet,
     VistaEnhancer,
 )
-from hgmd.models import HgmdPublicLocus
 from pathways.models import EnsemblToKegg, KeggInfo, RefseqToKegg
 from svdbs.models import DbVarSv, DgvGoldStandardSvs, DgvSvs, ExacCnv, GnomAdSv, ThousandGenomesSv
 from variants.helpers import get_engine, get_meta
@@ -73,7 +72,6 @@
     "gnomAD_exomes": (GnomadExomes,),
     "gnomAD_genomes": (GnomadGenomes,),
     "HelixMTdb": (HelixMtDb,),
-    "hgmd_public": (HgmdPublicLocus,),
     "hgnc": (Hgnc, RefseqToHgnc),
     "knowngeneaa": (KnowngeneAA,),
     "MITOMAP": (Mitomap,),

maintenance/management/commands/check_installation.py

@@ -128,8 +128,6 @@ class State(Enum):
     "conservatio_transcr_8aa7d7_idx",
 )
 
-INDEXES_HGMDPUBLICLOCUS = ("hgmd_hgmdpu_release_a5d0c3_idx",)
-
 INDEXES_STRUCTURALVARIANTGENEANNOTATION = (
     "svs_structu_sv_uuid_09c0c4_idx",
     "svs_structu_set_id_42f2ba_idx",
@@ -188,7 +186,6 @@ class State(Enum):
     "genomicfeatures_tadboundaryinterval": INDEXES_TADBOUNDARYINTERVAL,
     "genomicfeatures_geneinterval": INDEXES_GENEINTERVAL,
     "conservation_knowngeneaa": INDEXES_KNOWNGENEAA,
-    "hgmd_hgmdpubliclocus": INDEXES_HGMDPUBLICLOCUS,
     "svs_structuralvariantgeneannotation": INDEXES_STRUCTURALVARIANTGENEANNOTATION,
     "svs_structuralvariantflags": INDEXES_STRUCTURALVARIANTFLAGS,
     "svs_structuralvariantcomment": INDEXES_STRUCTURALVARIANTCOMMENT,

varfish/static/js/filter_form.js

@@ -479,8 +479,6 @@ const presetsFlagsDefault = {
   "flag_summary_negative": true,
   "flag_summary_empty": true,
 
-  "require_in_hgmd_public": false,
-  "remove_if_in_dbsnp": false,
   "require_in_clinvar": false,
   "clinvar_include_benign": false,
   "clinvar_include_likely_benign": false,

varfish/static/js/flags_comments.js

@@ -219,7 +219,7 @@ function clickVariantBookmark() {
 
 // Hide popover when clicking outside of popover.
 $('body').on('click', function (e) {
-    $('.variant-bookmark-comment-group, .hgmd-popover, .variant-acmg').each(function () {
+    $('.variant-bookmark-comment-group, .variant-acmg').each(function () {
         // hide any open popovers when the anywhere else in the body is clicked
         if (!$(this).is(e.target) && $(this).has(e.target).length === 0 && $('.popover').has(e.target).length === 0) {
             $(this).popover('hide');

variants/migrations/0104_remove_hgmd_dbsnp_filter_from_query_settings.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.25 on 2024-05-15 12:50
+
+from django.db import migrations
+
+
+def clear_query_settings(apps, schema_editor):
+    SmallVariantQuery = apps.get_model("variants", "SmallVariantQuery")
+    for s in SmallVariantQuery.objects.all():
+        save = False
+        if "remove_if_in_dbsnp" in s.query_settings:
+            del s.query_settings["remove_if_in_dbsnp"]
+            save = True
+        if "require_in_hgmd_public" in s.query_settings:
+            del s.query_settings["require_in_hgmd_public"]
+            save = True
+        if save:
+            s.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("variants", "0103_presetset_default_presetset"),
+    ]
+
+    operations = [
+        migrations.RunPython(clear_query_settings, lambda _x, _y: None),
+    ]

variants/models/presets.py

@@ -212,11 +212,7 @@ def create_as_copy_of_factory_preset(self, factory_preset_name, preset_label, pr
         return self.create(
             label=preset_label,
             presetset=preset_presetset,
-            **{
-                key: value
-                for key, value in getattr(FLAGSETC_PRESETS, factory_preset_name).items()
-                if key not in ("remove_if_in_dbsnp", "require_in_hgmd_public")
-            },
+            **{key: value for key, value in getattr(FLAGSETC_PRESETS, factory_preset_name).items()},
         )
 
     def create_as_copy_of_other_preset(self, other, presetset, **kwargs):

variants/queries.py

@@ -35,7 +35,6 @@
     RefseqToHgnc,
 )
 from genepanels.models import expand_panels_in_gene_list
-from hgmd.models import HgmdPublicLocus
 from svs.models import StructuralVariant, StructuralVariantGeneAnnotation
 from variants.forms import FILTER_FORM_TRANSLATE_INHERITANCE
 from variants.helpers import get_meta
@@ -199,14 +198,6 @@ def extend_selectable(self, query_parts):
         return query_parts.selectable.outerjoin(self.subquery, true())
 
 
-class ExtendQueryPartsDbsnpJoinAndFilter(ExtendQueryPartsDbsnpJoin):
-    def extend_conditions(self, _query_parts):
-        # Do not enable option if clinvar filter is activated as all clinvar variants have a dbsnp entry.
-        if self.kwargs["remove_if_in_dbsnp"] and not self.kwargs["require_in_clinvar"]:
-            return [column("rsid") == None]  # noqa: E711
-        return []
-
-
 class ExtendQueryPartsHgncJoin(ExtendQueryPartsBase):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -462,51 +453,6 @@ def _build_significance_term(self):
         return or_(*terms)
 
 
-class ExtendQueryPartsHgmdJoin(ExtendQueryPartsBase):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.subquery = (
-            select(
-                [
-                    func.count(HgmdPublicLocus.sa.variation_name).label("hgmd_public_overlap"),
-                    func.max(HgmdPublicLocus.sa.variation_name).label("hgmd_accession"),
-                ]
-            )
-            .select_from(HgmdPublicLocus.sa)
-            # This is a range query that caused an error already. So I'm writing a more elaborate comment.
-            # HGMD is 0-based, and our coordinate system is 1-based, that means the end positions match, but
-            # the start is one less in the HGMD compared to our system, i.e. our.start = hgmd.start + 1.
-            # We need to correct for that.
-            # The default interval query is:   (hgmd.start     <= our.end) and (hgmd.end >= our.start)
-            # The corrected interval query is: (hgmd.start + 1 <= our.end) and (hgmd.end >= our.start)
-            .where(
-                and_(
-                    HgmdPublicLocus.sa.release == SmallVariant.sa.release,
-                    HgmdPublicLocus.sa.chromosome == SmallVariant.sa.chromosome,
-                    (HgmdPublicLocus.sa.start + 1) <= SmallVariant.sa.end,
-                    HgmdPublicLocus.sa.end >= SmallVariant.sa.start,
-                )
-            )
-            .lateral("hgmd_subquery")
-        )
-
-    def extend_fields(self, _query_parts):
-        return [
-            func.coalesce(self.subquery.c.hgmd_public_overlap, 0).label("hgmd_public_overlap"),
-            func.coalesce(self.subquery.c.hgmd_accession, "").label("hgmd_accession"),
-        ]
-
-    def extend_selectable(self, query_parts):
-        return query_parts.selectable.outerjoin(self.subquery, true())
-
-
-class ExtendQueryPartsHgmdJoinAndFilter(ExtendQueryPartsHgmdJoin):
-    def extend_conditions(self, _query_parts):
-        if self.kwargs.get("require_in_hgmd_public"):
-            return [column("hgmd_public_overlap") > 0]
-        return []
-
-
 class ExtendQueryPartsCaseJoinGeneric(ExtendQueryPartsBase):
     """Generic query parts that joins the case"""
 
@@ -1575,7 +1521,7 @@ def extend_selectable(self, query_parts):
 #: QueryPartsBuilderExtender classes list for cases.
 extender_classes_base = [
     ExtendQueryPartsCaseJoinAndFilter,
-    ExtendQueryPartsDbsnpJoinAndFilter,
+    ExtendQueryPartsDbsnpJoin,
     ExtendQueryPartsVarTypeFilter,
     ExtendQueryPartsFrequenciesFilter,
     ExtendQueryPartsInHouseJoinAndFilter,
@@ -1585,7 +1531,6 @@ def extend_selectable(self, query_parts):
     ExtendQueryPartsGeneListsFilter,
     ExtendQueryPartsGenomicRegionFilter,
     ExtendQueryPartsClinvarJoinAndFilter,
-    ExtendQueryPartsHgmdJoinAndFilter,
     ExtendQueryPartsGenotypeGtQualityDefaultFilter,
     ExtendQueryPartsFlagsJoinAndFilter,
     ExtendQueryPartsCommentsJoin,
@@ -1689,7 +1634,6 @@ def get_qp_extender_classes(self):
             ExtendQueryPartsGeneSymbolJoin,
             ExtendQueryPartsGenomicRegionFilter,
             ExtendQueryPartsGnomadConstraintsJoin,
-            ExtendQueryPartsHgmdJoinAndFilter,
             ExtendQueryPartsHgncJoin,
             ExtendQueryPartsInHouseJoin,
             ExtendQueryPartsMgiJoin,
@@ -1716,7 +1660,7 @@ class CaseExportVcfQueryPartsBuilder(QueryPartsBuilder):
 
     # TODO What about DbSNP and HGNC that are used for filtering???
     # TODO Should we just take the stored results and join the required data?
-    # TODO But then, some extensions join AND query ... maybe split them (HGNC?, Clinvar?, dbSNP, HGMD)
+    # TODO But then, some extensions join AND query ... maybe split them (HGNC?, Clinvar?)
     def get_qp_extender_classes(self):
         return extender_classes_base
 
@@ -1760,7 +1704,7 @@ class ProjectExportVcfQueryPartsBuilder(QueryPartsBuilder):
 
     # TODO What about DbSNP and HGNC that are used for filtering???
     # TODO Should we just take the stored results and join the required data?
-    # TODO But then, some extensions join AND query ... maybe split them (HGNC?, Clinvar?, dbSNP, HGMD)
+    # TODO But then, some extensions join AND query ... maybe split them (HGNC?, Clinvar?)
     def get_qp_extender_classes(self):
         return extender_classes_base
 

variants/query_presets.py

@@ -942,10 +942,8 @@ class _FlagsEtcPresets:
         "flag_visual_negative": True,
         "flag_visual_positive": True,
         "flag_visual_uncertain": True,
-        "remove_if_in_dbsnp": False,
         "require_in_clinvar": False,
         "clinvar_paranoid_mode": False,
-        "require_in_hgmd_public": False,
     }
     #: Presets for the "Clinvar only" flags etc. settings
     clinvar_only: typing.Dict[str, typing.Any] = {
@@ -978,10 +976,8 @@ class _FlagsEtcPresets:
         "flag_visual_negative": True,
         "flag_visual_positive": True,
         "flag_visual_uncertain": True,
-        "remove_if_in_dbsnp": False,
         "require_in_clinvar": True,
         "clinvar_paranoid_mode": False,
-        "require_in_hgmd_public": False,
     }
     #: Presets for the "user flagged" flags etc. settings
     user_flagged: typing.Dict[str, typing.Any] = {
@@ -1019,10 +1015,8 @@ class _FlagsEtcPresets:
         "flag_visual_negative": True,
         "flag_visual_positive": True,
         "flag_visual_uncertain": True,
-        "remove_if_in_dbsnp": False,
         "require_in_clinvar": False,
         "clinvar_paranoid_mode": False,
-        "require_in_hgmd_public": False,
     }
 
 

variants/query_schemas.py

@@ -237,9 +237,6 @@ class CaseQueryV1:
     gene_blocklist: typing.Optional[typing.List[str]] = None
     genomic_region: typing.Optional[typing.List[GenomicRegionV1]] = None
 
-    remove_if_in_dbsnp: bool = False
-
-    require_in_hgmd_public: bool = False
     require_in_clinvar: bool = False
     clinvar_include_benign: bool = True
     clinvar_include_pathogenic: bool = True
@@ -334,8 +331,6 @@ def convert(self, case: Case, query: CaseQueryV1) -> typing.Dict[str, typing.Any
             "mitomap_count": query.mitomap_count,
             "transcripts_coding": query.transcripts_coding,
             "transcripts_noncoding": query.transcripts_noncoding,
-            "remove_if_in_dbsnp": query.remove_if_in_dbsnp,
-            "require_in_hgmd_public": query.require_in_hgmd_public,
             "require_in_clinvar": query.require_in_clinvar,
             "clinvar_include_benign": query.clinvar_include_benign,
             "clinvar_include_likely_benign": query.clinvar_include_likely_benign,

variants/schemas/case-query-v1.json

@@ -1078,17 +1078,6 @@
         "pattern": "^([a-zA-Z0-9_-]+)$"
       }
     },
-    "remove_if_in_dbsnp": {
-      "$id": "#/properties/remove_if_in_dbsnp",
-      "type": "boolean",
-      "title": "Remove variant if it exists in local copy dbSNP",
-      "description": "Set to true to exclude variants that are present in dbSNP from the result set",
-      "default": false,
-      "examples": [
-        true,
-        false
-      ]
-    },
     "require_in_clinvar": {
       "$id": "#/properties/require_in_clinvar",
       "type": "boolean",
@@ -1272,16 +1261,6 @@
         }
       ]
     },
-    "require_in_hgmd_public": {
-      "$id": "#/properties/require_in_hgmd_public",
-      "type": "boolean",
-      "title": "The require_in_hgmd_public schema",
-      "description": "An explanation about the purpose of this instance.",
-      "default": false,
-      "examples": [
-        false
-      ]
-    },
     "recessive_mode": {
       "anyOf": [
         {