8693j9e13: remove all ICD / OPCS refs that aren't used.

docker-compose-prod.yml

@@ -7,7 +7,7 @@ services:
     container_name: medcattrainer
     hostname: medcat
     restart: always
-    image: cogstacksystems/medcat-trainer:v2.13.3
+    image: cogstacksystems/medcat-trainer:v2.14.1
     volumes:
       - ./configs:/home/configs
       - api-media:/home/api/media
@@ -19,12 +19,12 @@ services:
     env_file:
       - ./envs/env-prod
     environment:
-      - MCT_VERSION=v2.13.3
+      - MCT_VERSION=v2.14.1
     command: /home/scripts/run.sh
 
   # crontab - for db backup
   medcattrainer-db-backup:
-    image: cogstacksystems/medcat-trainer:v2.13.3
+    image: cogstacksystems/medcat-trainer:v2.14.1
     restart: always
     volumes:
       - ./configs:/home/configs
@@ -39,7 +39,7 @@ services:
 
   nginx:
     container_name: medcattrainer_nginx
-    image: cogstacksystems/medcat-trainer-nginx:v2.13.3
+    image: cogstacksystems/medcat-trainer-nginx:v2.14.1
     restart: always
     volumes:
       - api-media:/home/api/media

docker-compose.yml

@@ -5,7 +5,7 @@ version: '3.4'
 
 services:
   medcattrainer:
-    image: cogstacksystems/medcat-trainer:v2.13.3
+    image: cogstacksystems/medcat-trainer:v2.14.1
     restart: always
     volumes:
       - ./configs:/home/configs
@@ -16,12 +16,12 @@ services:
     env_file:
       - ./envs/env
     environment:
-      - MCT_VERSION=v2.13.3
+      - MCT_VERSION=v2.14.1
     command: /home/scripts/run.sh
 
   # crontab - for db backup
   medcattrainer-db-backup:
-    image: cogstacksystems/medcat-trainer:v2.13.3
+    image: cogstacksystems/medcat-trainer:v2.14.1
     restart: always
     volumes:
       - ./configs:/home/configs
@@ -35,7 +35,7 @@ services:
     command: cron -f -l 2
 
   nginx:
-    image: cogstacksystems/medcat-trainer-nginx:v2.13.3
+    image: cogstacksystems/medcat-trainer-nginx:v2.14.1
     restart: always
     volumes:
       - api-media:/home/api/media

docs/installation.md

@@ -40,6 +40,16 @@ On MAC: https://docs.docker.com/docker-for-mac/#memory
 
 On Windows: https://docs.docker.com/docker-for-windows/#resources
 
+### (Optional) SMTP Setup
+
+For password resets and other emailing services email environment variables are required to be set up.
+
+Personal email accounts can be set up by users to do this, or you can contact someone in CogStack for a cogstack no email credentials. 
+
+The environment variables required are listed in [Environment Variables.](#(optional)-environment-variables) 
+
+Environment Variables are located in envs/env or envs/env-prod, when those are set webapp/frontend/.env must change "VITE_APP_EMAIL" to 1.
+
 ### (Optional) Environment Variables
 Environment variables are used to configure the app:
 
@@ -48,6 +58,10 @@ Environment variables are used to configure the app:
 |MEDCAT_CONFIG_FILE|MedCAT config file as described [here](https://github.com/CogStack/MedCAT/blob/master/medcat/config.py)|
 |BEHIND_RP| If you're running MedCATtrainer, use 1, otherwise this defaults to 0 i.e. False|
 |MCTRAINER_PORT|The port to run the trainer app on|
+|EMAIL_USER|Email address which will be used to send users emails regarding password resets|
+|EMAIL_PASS|The password or authentication key which will be used with the email address|
+|EMAIL_HOST|The hostname of the SMTP server which will be used to send email (default: mail.cogstack.org)|
+|EMAIL_PORT|The port that the SMTP server is listening to, common numbers are 25, 465, 587 (default: 465)|
 
 Set these and re-run the docker-compose file.
 

docs/requirements.txt

@@ -1,3 +1,3 @@
-Sphinx~=4.0
+Sphinx~=5.0
 sphinx-rtd-theme~=1.0
 myst-parser~=0.17

envs/env

@@ -33,3 +33,9 @@ RESUBMIT_ALL_ON_STARTUP=1
 
 # Front end env vars
 LOAD_NUM_DOC_PAGES=10
+
+# SMTP email settings - when settings are configured go to webapp/frontend/.env and set VITE_APP_EMAIL to 1
+[email protected]
+EMAIL_PASS="to be changed"
+EMAIL_HOST=mail.cogstack.org
+EMAIL_PORT=465

envs/env-prod

@@ -35,3 +35,8 @@ RESUBMIT_ALL_ON_STARTUP=1
 # Front end env vars
 LOAD_NUM_DOC_PAGES=10
 
+# SMTP EMAIL SETTINGS
+[email protected]
+EMAIL_PASS="to be changed"
+EMAIL_HOST=mail.cogstack.org
+EMAIL_PORT=465

webapp/api/api/admin.py

@@ -94,10 +94,6 @@ def download_projects_without_text(projects, with_doc_name):
                 out_ann['last_modified'] = str(ann.last_modified)
                 out_ann['manually_created'] = ann.manually_created
                 out_ann['acc'] = ann.acc
-                if ann.icd_code:
-                    out_ann['icd_code'] = ann.icd_code.code
-                if ann.opcs_code:
-                    out_ann['opcs_code'] = ann.opcs_code.code
                 out_ann['meta_anns'] = {}
 
                 # Get MetaAnnotations
@@ -267,18 +263,6 @@ def retrieve_project_data(projects: QuerySet) -> Dict[str, List]:
                 out_ann['comment'] = ann.comment
                 out_ann['manually_created'] = ann.manually_created
                 out_ann['acc'] = ann.acc
-                # if ann.icd_code:
-                #     out_ann['icd_code'] = {'code': ann.icd_code.code, 'desc': ann.icd_code.desc}
-                # if ann.opcs_code:
-                #     out_ann['opcs_codes'] = {'code': ann.opcs_code, 'desc': ann.opcs_code.desc}
-                #
-                # out_ann['acc'] = ann.acc
-                # if ann.comment:
-                #     out_ann['comment'] = ann.comment
-                # if ann.icd_code:
-                #     out_ann['icd_code'] = ann.icd_code.code
-                # if ann.opcs_code:
-                #     out_ann['opcs_code'] = ann.opcs_code.code
                 out_ann['meta_anns'] = {}
 
                 # Get MetaAnnotations
@@ -438,12 +422,6 @@ def import_concepts(modeladmin, request, queryset):
 def delete_indexed_concepts(modeladmin, request, queryset):
     for concept_db in queryset:
         drop_collection(concept_db)
-        ICDCode.objects.filter(cdb=concept_db).delete()
-        OPCSCode.objects.filter(cdb=concept_db).delete()
-
-
-admin.site.register(ICDCode)
-admin.site.register(OPCSCode)
 
 
 class ConceptDBAdmin(admin.ModelAdmin):

webapp/api/api/migrations/0074_auto_20240215_1024.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.2.28 on 2024-02-15 10:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0073_auto_20231022_0028'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='opcscode',
+            name='cdb',
+        ),
+        migrations.RemoveField(
+            model_name='annotatedentity',
+            name='icd_code',
+        ),
+        migrations.RemoveField(
+            model_name='annotatedentity',
+            name='opcs_code',
+        ),
+        migrations.AlterField(
+            model_name='projectmetrics',
+            name='projects',
+            field=models.ManyToManyField(blank=True, to='api.ProjectAnnotateEntities'),
+        ),
+        migrations.DeleteModel(
+            name='ICDCode',
+        ),
+        migrations.DeleteModel(
+            name='OPCSCode',
+        ),
+    ]

webapp/api/api/models.py

@@ -20,24 +20,6 @@
         ]
 
 
-class ICDCode(models.Model):
-    code = models.CharField(max_length=10, unique=True)
-    desc = models.CharField(max_length=300)
-    cdb = models.ForeignKey('ConceptDB', on_delete=models.SET_NULL, blank=True, null=True)
-
-    def __str__(self):
-        return f'{self.code} | {self.desc}'
-
-
-class OPCSCode(models.Model):
-    code = models.CharField(max_length=10, unique=True)
-    desc = models.CharField(max_length=300)
-    cdb = models.ForeignKey('ConceptDB', on_delete=models.SET_NULL, blank=True, null=True)
-
-    def __str__(self):
-        return f'{self.code} | {self.desc}'
-
-
 cdb_name_validator = RegexValidator(r'^[0-9a-zA-Z_-]*$', 'Only alpahanumeric characters, -, _ are allowed for CDB names')
 
 
@@ -197,11 +179,6 @@ class AnnotatedEntity(models.Model):
     create_time = models.DateTimeField(auto_now_add=True)
     last_modified = models.DateTimeField(auto_now=True)
 
-    # Specific to the Clinical Coding use case - feels hacky being directly on this model.
-    # Should AnnotatedEntity be a polymorphic model?? and there be a specific ClinicalCodingAnnotatedEntity??
-    icd_code = models.ForeignKey('ICDCode', on_delete=models.SET_NULL, blank=True, null=True)
-    opcs_code = models.ForeignKey('OPCSCode', on_delete=models.SET_NULL, blank=True, null=True)
-
     class Meta:
         ordering = ['id']
 

webapp/api/api/serializers.py

@@ -53,18 +53,6 @@ class Meta:
         fields = '__all__'
 
 
-class ICDCodeSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = ICDCode
-        fields = ['code', 'desc', 'id', 'concept']
-
-
-class OPCSCodeSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = OPCSCode
-        fields = ['code', 'desc', 'id', 'concept']
-
-
 class ProjectAnnotateEntitiesSerializer(serializers.ModelSerializer):
     class Meta:
         model = ProjectAnnotateEntities

webapp/api/api/solr_utils.py

@@ -54,10 +54,6 @@ def _process_result_repsonse(resp: Dict):
                 'type_ids': d.get('type_ids', []),
                 'synonyms': d['synonyms']
             }
-            if d.get('icd10'):
-                parsed_doc['icd10'] = d['icd10'][0]
-            if d.get('opcs4'):
-                parsed_doc['opcs4'] = d['opcs4'][0]
             uniq_results_map[d['cui'][0]] = parsed_doc
     return uniq_results_map
 
@@ -199,25 +195,6 @@ def _concept_dct(cui: str, cdb: CDB):
         'desc': cdb.addl_info.get('cui2description', {}).get(cui, ''),
         'synonyms': list(cdb.addl_info.get('cui2original_names', {}).get(cui, set())),
     }
-    icd_codes = cdb.addl_info.get('cui2icd10', {}).get(cui, None)
-    if icd_codes is not None:
-        try:
-            concept_dct['icd10'] = ', '.join([f'{code["code"]} : {code["name"]}'
-                                              for code in icd_codes])
-        except Exception:
-            logger.warning(f'Tried to extract ICD codes for cui:{cui} for concept (solr) search - '
-                           f'but encountered icd_codes of the form:{icd_codes}, expected a list of '
-                           '{code: <the code>, name: <human readable desc>, ...}')
-    opcs_codes = cdb.addl_info.get('cui2opcs4', {}).get(cui, None)
-    if opcs_codes is not None:
-        try:
-            concept_dct['opcs4'] = ', '.join([f'{code["code"]} : {code["name"]}'
-                                              for code in opcs_codes])
-        except Exception:
-            logger.warning(f'Tried to upload OPCS codes for cui:{cui} for concept (solr) search - '
-                           f'but encountered OPCS codes of the form:{opcs_codes}, expected a list of '
-                           '{code: <the code>, name: <human readable desc> ...}')
-
     return concept_dct
 
 

webapp/api/api/utils.py

@@ -13,7 +13,7 @@
 from medcat.utils.helpers import tkns_from_doc
 from medcat.vocab import Vocab
 
-from .models import Entity, AnnotatedEntity, ICDCode, OPCSCode, ProjectAnnotateEntities, \
+from .models import Entity, AnnotatedEntity, ProjectAnnotateEntities, \
     ConceptDB
 
 log = logging.getLogger('trainer')
@@ -91,27 +91,6 @@ def check_ents(ent):
             ann_ent.save()
 
 
-def _create_linked_codes(mod: Union[Type[ICDCode], Type[OPCSCode]], linked_codes: List[Dict], cdb_model: ConceptDB):
-    """
-    Expects the cui2icd10, cui2opcs4 dicts to include code and name.
-    """
-    for code in linked_codes:
-        if len(mod.objects.filter(code=code['code'])) == 0:
-            code_mod = mod()
-            code_mod.code = code['code']
-            code_mod.desc = code['name']
-            code_mod.cdb = cdb_model
-            code_mod.save()
-
-
-def create_linked_icd_codes(codes: List, cdb_model):
-    _create_linked_codes(ICDCode, codes, cdb_model)
-
-
-def create_linked_opcs_codes(codes: List, cdb_model):
-    _create_linked_codes(OPCSCode, codes, cdb_model)
-
-
 def get_create_cdb_infos(cdb, concept, cui, cui_info_prop, code_prop, desc_prop, model_clazz):
     codes = [c[code_prop] for c in cdb.cui2info.get(cui, {}).get(cui_info_prop, []) if code_prop in c]
     existing_codes = model_clazz.objects.filter(code__in=codes)
@@ -139,8 +118,7 @@ def _remove_overlap(project, document, start, end):
 
 
 def create_annotation(source_val: str, selection_occurrence_index: int, cui: str, user: User,
-                      project: ProjectAnnotateEntities, document, cat: CAT, icd_code=None,
-                      opcs_code=None):
+                      project: ProjectAnnotateEntities, document, cat: CAT):
     text = document.text
     id = None
 
@@ -180,25 +158,9 @@ def create_annotation(source_val: str, selection_occurrence_index: int, cui: str
         ann_ent.validated = True
         ann_ent.manually_created = True
         ann_ent.correct = True
-
-        if icd_code:
-            ann_ent.icd_code = icd_code
-        if opcs_code:
-            ann_ent.opcs_code = opcs_code
-
         ann_ent.save()
         id = ann_ent.id
 
-    # upload icd / opcs codes if available
-    # also expects icd / opcs addl info dicts to include:
-    # {code: <the code>: name: <human readable desc>}
-    icd_codes = cat.cdb.addl_info.get('cui2icd10', {}).get(cui, None)
-    if icd_codes is not None:
-        create_linked_icd_codes(icd_codes, project.concept_db)
-    opcs_codes = cat.cdb.addl_info.get('cui2opcs4', {}).get(cui, None)
-    if opcs_codes is not None:
-        create_linked_opcs_codes(opcs_codes, project.concept_db)
-
     return id