Merge pull request #179 from CogStack/remove-icd-opcs-refs

8693j9e13: remove all ICD / OPCS refs that aren't used.

webapp/api/api/admin.py

@@ -94,10 +94,6 @@ def download_projects_without_text(projects, with_doc_name):
                 out_ann['last_modified'] = str(ann.last_modified)
                 out_ann['manually_created'] = ann.manually_created
                 out_ann['acc'] = ann.acc
-                if ann.icd_code:
-                    out_ann['icd_code'] = ann.icd_code.code
-                if ann.opcs_code:
-                    out_ann['opcs_code'] = ann.opcs_code.code
                 out_ann['meta_anns'] = {}
 
                 # Get MetaAnnotations
@@ -267,18 +263,6 @@ def retrieve_project_data(projects: QuerySet) -> Dict[str, List]:
                 out_ann['comment'] = ann.comment
                 out_ann['manually_created'] = ann.manually_created
                 out_ann['acc'] = ann.acc
-                # if ann.icd_code:
-                #     out_ann['icd_code'] = {'code': ann.icd_code.code, 'desc': ann.icd_code.desc}
-                # if ann.opcs_code:
-                #     out_ann['opcs_codes'] = {'code': ann.opcs_code, 'desc': ann.opcs_code.desc}
-                #
-                # out_ann['acc'] = ann.acc
-                # if ann.comment:
-                #     out_ann['comment'] = ann.comment
-                # if ann.icd_code:
-                #     out_ann['icd_code'] = ann.icd_code.code
-                # if ann.opcs_code:
-                #     out_ann['opcs_code'] = ann.opcs_code.code
                 out_ann['meta_anns'] = {}
 
                 # Get MetaAnnotations
@@ -438,12 +422,6 @@ def import_concepts(modeladmin, request, queryset):
 def delete_indexed_concepts(modeladmin, request, queryset):
     for concept_db in queryset:
         drop_collection(concept_db)
-        ICDCode.objects.filter(cdb=concept_db).delete()
-        OPCSCode.objects.filter(cdb=concept_db).delete()
-
-
-admin.site.register(ICDCode)
-admin.site.register(OPCSCode)
 
 
 class ConceptDBAdmin(admin.ModelAdmin):

webapp/api/api/migrations/0074_auto_20240215_1024.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.2.28 on 2024-02-15 10:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0073_auto_20231022_0028'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='opcscode',
+            name='cdb',
+        ),
+        migrations.RemoveField(
+            model_name='annotatedentity',
+            name='icd_code',
+        ),
+        migrations.RemoveField(
+            model_name='annotatedentity',
+            name='opcs_code',
+        ),
+        migrations.AlterField(
+            model_name='projectmetrics',
+            name='projects',
+            field=models.ManyToManyField(blank=True, to='api.ProjectAnnotateEntities'),
+        ),
+        migrations.DeleteModel(
+            name='ICDCode',
+        ),
+        migrations.DeleteModel(
+            name='OPCSCode',
+        ),
+    ]

webapp/api/api/models.py

@@ -20,24 +20,6 @@
         ]
 
 
-class ICDCode(models.Model):
-    code = models.CharField(max_length=10, unique=True)
-    desc = models.CharField(max_length=300)
-    cdb = models.ForeignKey('ConceptDB', on_delete=models.SET_NULL, blank=True, null=True)
-
-    def __str__(self):
-        return f'{self.code} | {self.desc}'
-
-
-class OPCSCode(models.Model):
-    code = models.CharField(max_length=10, unique=True)
-    desc = models.CharField(max_length=300)
-    cdb = models.ForeignKey('ConceptDB', on_delete=models.SET_NULL, blank=True, null=True)
-
-    def __str__(self):
-        return f'{self.code} | {self.desc}'
-
-
 cdb_name_validator = RegexValidator(r'^[0-9a-zA-Z_-]*$', 'Only alpahanumeric characters, -, _ are allowed for CDB names')
 
 
@@ -197,11 +179,6 @@ class AnnotatedEntity(models.Model):
     create_time = models.DateTimeField(auto_now_add=True)
     last_modified = models.DateTimeField(auto_now=True)
 
-    # Specific to the Clinical Coding use case - feels hacky being directly on this model.
-    # Should AnnotatedEntity be a polymorphic model?? and there be a specific ClinicalCodingAnnotatedEntity??
-    icd_code = models.ForeignKey('ICDCode', on_delete=models.SET_NULL, blank=True, null=True)
-    opcs_code = models.ForeignKey('OPCSCode', on_delete=models.SET_NULL, blank=True, null=True)
-
     class Meta:
         ordering = ['id']
 

webapp/api/api/serializers.py

@@ -53,18 +53,6 @@ class Meta:
         fields = '__all__'
 
 
-class ICDCodeSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = ICDCode
-        fields = ['code', 'desc', 'id', 'concept']
-
-
-class OPCSCodeSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = OPCSCode
-        fields = ['code', 'desc', 'id', 'concept']
-
-
 class ProjectAnnotateEntitiesSerializer(serializers.ModelSerializer):
     class Meta:
         model = ProjectAnnotateEntities

webapp/api/api/solr_utils.py

@@ -54,10 +54,6 @@ def _process_result_repsonse(resp: Dict):
                 'type_ids': d.get('type_ids', []),
                 'synonyms': d['synonyms']
             }
-            if d.get('icd10'):
-                parsed_doc['icd10'] = d['icd10'][0]
-            if d.get('opcs4'):
-                parsed_doc['opcs4'] = d['opcs4'][0]
             uniq_results_map[d['cui'][0]] = parsed_doc
     return uniq_results_map
 
@@ -199,25 +195,6 @@ def _concept_dct(cui: str, cdb: CDB):
         'desc': cdb.addl_info.get('cui2description', {}).get(cui, ''),
         'synonyms': list(cdb.addl_info.get('cui2original_names', {}).get(cui, set())),
     }
-    icd_codes = cdb.addl_info.get('cui2icd10', {}).get(cui, None)
-    if icd_codes is not None:
-        try:
-            concept_dct['icd10'] = ', '.join([f'{code["code"]} : {code["name"]}'
-                                              for code in icd_codes])
-        except Exception:
-            logger.warning(f'Tried to extract ICD codes for cui:{cui} for concept (solr) search - '
-                           f'but encountered icd_codes of the form:{icd_codes}, expected a list of '
-                           '{code: <the code>, name: <human readable desc>, ...}')
-    opcs_codes = cdb.addl_info.get('cui2opcs4', {}).get(cui, None)
-    if opcs_codes is not None:
-        try:
-            concept_dct['opcs4'] = ', '.join([f'{code["code"]} : {code["name"]}'
-                                              for code in opcs_codes])
-        except Exception:
-            logger.warning(f'Tried to upload OPCS codes for cui:{cui} for concept (solr) search - '
-                           f'but encountered OPCS codes of the form:{opcs_codes}, expected a list of '
-                           '{code: <the code>, name: <human readable desc> ...}')
-
     return concept_dct
 
 

webapp/api/api/utils.py

@@ -13,7 +13,7 @@
 from medcat.utils.helpers import tkns_from_doc
 from medcat.vocab import Vocab
 
-from .models import Entity, AnnotatedEntity, ICDCode, OPCSCode, ProjectAnnotateEntities, \
+from .models import Entity, AnnotatedEntity, ProjectAnnotateEntities, \
     ConceptDB
 
 log = logging.getLogger('trainer')
@@ -91,27 +91,6 @@ def check_ents(ent):
             ann_ent.save()
 
 
-def _create_linked_codes(mod: Union[Type[ICDCode], Type[OPCSCode]], linked_codes: List[Dict], cdb_model: ConceptDB):
-    """
-    Expects the cui2icd10, cui2opcs4 dicts to include code and name.
-    """
-    for code in linked_codes:
-        if len(mod.objects.filter(code=code['code'])) == 0:
-            code_mod = mod()
-            code_mod.code = code['code']
-            code_mod.desc = code['name']
-            code_mod.cdb = cdb_model
-            code_mod.save()
-
-
-def create_linked_icd_codes(codes: List, cdb_model):
-    _create_linked_codes(ICDCode, codes, cdb_model)
-
-
-def create_linked_opcs_codes(codes: List, cdb_model):
-    _create_linked_codes(OPCSCode, codes, cdb_model)
-
-
 def get_create_cdb_infos(cdb, concept, cui, cui_info_prop, code_prop, desc_prop, model_clazz):
     codes = [c[code_prop] for c in cdb.cui2info.get(cui, {}).get(cui_info_prop, []) if code_prop in c]
     existing_codes = model_clazz.objects.filter(code__in=codes)
@@ -139,8 +118,7 @@ def _remove_overlap(project, document, start, end):
 
 
 def create_annotation(source_val: str, selection_occurrence_index: int, cui: str, user: User,
-                      project: ProjectAnnotateEntities, document, cat: CAT, icd_code=None,
-                      opcs_code=None):
+                      project: ProjectAnnotateEntities, document, cat: CAT):
     text = document.text
     id = None
 
@@ -180,25 +158,9 @@ def create_annotation(source_val: str, selection_occurrence_index: int, cui: str
         ann_ent.validated = True
         ann_ent.manually_created = True
         ann_ent.correct = True
-
-        if icd_code:
-            ann_ent.icd_code = icd_code
-        if opcs_code:
-            ann_ent.opcs_code = opcs_code
-
         ann_ent.save()
         id = ann_ent.id
 
-    # upload icd / opcs codes if available
-    # also expects icd / opcs addl info dicts to include:
-    # {code: <the code>: name: <human readable desc>}
-    icd_codes = cat.cdb.addl_info.get('cui2icd10', {}).get(cui, None)
-    if icd_codes is not None:
-        create_linked_icd_codes(icd_codes, project.concept_db)
-    opcs_codes = cat.cdb.addl_info.get('cui2opcs4', {}).get(cui, None)
-    if opcs_codes is not None:
-        create_linked_opcs_codes(opcs_codes, project.concept_db)
-
     return id
 
 

webapp/api/api/views.py

@@ -180,40 +180,16 @@ class DatasetViewSet(viewsets.ModelViewSet):
     serializer_class = DatasetSerializer
 
 
-class ICDCodeFilter(drf.FilterSet):
-    code__in = TextInFilter(field_name='code', lookup_expr='in')
-    id__in = NumInFilter(field_name='id', lookup_expr='in')
-
-    class Meta:
-        model = ICDCode
-        fields = ['code', 'id']
-
-
-class OPCSCodeFilter(drf.FilterSet):
-    code__in = TextInFilter(field_name='code', lookup_expr='in')
-    id__in = NumInFilter(field_name='id', lookup_expr='in')
-
-    class Meta:
-        model = OPCSCode
-        fields = ['code', 'id']
-
-
-class ICDCodeViewSet(viewsets.ModelViewSet):
-    permission_classes = [permissions.IsAuthenticated]
-    http_method_names = ['get']
-    queryset = ICDCode.objects.all()
-    serializer_class = ICDCodeSerializer
-    filterset_class = ICDCodeFilter
-    filterset_fields = ['code', 'id']
-
-
-class OPCSCodeViewSet(viewsets.ModelViewSet):
-    permission_classes = [permissions.IsAuthenticated]
-    http_method_names = ['get']
-    queryset = OPCSCode.objects.all()
-    serializer_class = OPCSCodeSerializer
-    filterset_class = OPCSCodeFilter
-    filterset_fields = ['code', 'id']
+class ResetPasswordView(PasswordResetView):
+    email_template_name = 'password_reset_email.html'
+    subject_template_name = 'password_reset_subject.txt'
+    def post(self, request, *args, **kwargs):
+        try:
+            return super().post(request, *args, **kwargs)
+        except SMTPException:
+            return HttpResponseServerError('''SMTP settings are not configured correctly. <br>
+                                           Please visit https://medcattrainer.readthedocs.io for more information to resolve this. <br>
+                                           You can also ask a question at: https://discourse.cogstack.org/c/medcat/5''')
 
 class ResetPasswordView(PasswordResetView):
     email_template_name = 'password_reset_email.html'
@@ -310,9 +286,6 @@ def add_annotation(request):
     sel_occur_idx = int(request.data['selection_occur_idx'])
     cui = str(request.data['cui'])
 
-    icd_code = request.data.get('icd_code')
-    opcs_code = request.data.get('opcs_code')
-
     logger.debug("Annotation being added")
     logger.debug(str(request.data))
 
@@ -321,11 +294,6 @@ def add_annotation(request):
     project = ProjectAnnotateEntities.objects.get(id=p_id)
     document = Document.objects.get(id=d_id)
 
-    if icd_code:
-        icd_code = ICDCode.objects.filter(id=icd_code).first()
-    if opcs_code:
-        opcs_code = OPCSCode.objects.filter(id=opcs_code).first()
-
     cat = get_medcat(CDB_MAP=CDB_MAP, VOCAB_MAP=VOCAB_MAP,
                      CAT_MAP=CAT_MAP, project=project)
     id = create_annotation(source_val=source_val,
@@ -334,9 +302,7 @@ def add_annotation(request):
                            user=user,
                            project=project,
                            document=document,
-                           cat=cat,
-                           icd_code=icd_code,
-                           opcs_code=opcs_code)
+                           cat=cat)
     logger.debug('Annotation added.')
     return Response({'message': 'Annotation added successfully', 'id': id})
 
@@ -768,8 +734,6 @@ def cdb_cui_children(request, cdb_id):
 
     # root SNOMED CT code: 138875005
     # root UMLS code: CUI:
-    # root level ICD term:
-    # root level OPCS term:
 
     if cdb.addl_info.get('pt2ch') is None:
         return HttpResponseBadRequest('Requested MedCAT CDB model does not include parent2child metadata to'

webapp/api/core/urls.py

@@ -34,8 +34,6 @@
 router.register(r'concept-dbs', api.views.ConceptDBViewSet)
 router.register(r'vocabs', api.views.VocabularyViewSet)
 router.register(r'datasets', api.views.DatasetViewSet)
-router.register(r'icd-codes', api.views.ICDCodeViewSet)
-router.register(r'opcs-codes', api.views.OPCSCodeViewSet)
 
 
 urlpatterns = [

webapp/frontend/src/components/anns/AddAnnotation.vue

@@ -53,18 +53,6 @@
           <td>Concept ID</td>
           <td >{{selectedCUI.cui || 'n/a'}}</td>
         </tr>
-        <tr v-if="(selectedCUI.icd10 || []).length > 0">
-          <td>ICD-10</td>
-          <td class="fit-content">
-            <div v-for="code of selectedCUI.icd10" :key="code.code">{{`${code.code} | ${code.desc}`}}</div>
-          </td>
-        </tr>
-        <tr v-if="(selectedCUI.opcs4 || []).length > 0">
-          <td>OPCS-4</td>
-          <td class="fit-content">
-            <div v-for="code of selectedCUI.opcs4" :key="code.code">{{`${code.code} | ${code.desc}`}}</div>
-          </td>
-        </tr>
         <tr>
           <td>Description</td>
           <td class="fit-content" v-html="selectedCUI.desc === 'nan' ? 'n/a' : selectedCUI.desc || 'n/a'"></td>
@@ -122,15 +110,6 @@ export default {
   methods: {
     enrichCUI (concept) {
       this.selectedCUI = concept
-      if (this.selectedCUI.icd10 || this.selectedCUI.opcs4) {
-        this.fetchConcept(this.selectedCUI, this.searchFilterDBIndex, () => {
-          if (concept.icdCode) {
-            this.selectedCUI.icd10 = this.selectedCUI.icd10.filter(i => i.id === concept.icdCode)
-          } else if (concept.opcsCode) {
-            this.selectedCUI.opcs4 = this.selectedCUI.opcs4.filter(i => i.id === concept.opcsCode)
-          }
-        })
-      }
     },
     submit () {
       const payload = {
@@ -140,11 +119,6 @@ export default {
         selection_occur_idx: this.selection.selectionOccurrenceIdx,
         cui: this.selectedCUI.cui
       }
-      if (this.selectedCUI.icd10 && this.selectedCUI.icd10.length === 1) {
-        payload.icd_code = this.selectedCUI.icd10[0].id
-      } else if (this.selectedCUI.opcs4 && this.selectedCUI.opcs4.length === 1) {
-        payload.opcs_code = this.selectedCUI.opcs4[0].id
-      }
 
       this.$http.post('/api/add-annotation/', payload).then(resp => {
         this.$emit('request:addAnnotationComplete', resp.data.id)