diff --git a/webapp/api/api/admin/__init__.py b/webapp/api/api/admin/__init__.py new file mode 100644 index 0000000..60727e4 --- /dev/null +++ b/webapp/api/api/admin/__init__.py @@ -0,0 +1,19 @@ +from .models import * +from .actions import * +from ..models import * + +admin.site.register(Entity) +admin.site.register(MetaTaskValue) +admin.site.register(MetaTask) +admin.site.register(MetaAnnotation) +admin.site.register(Vocabulary) +admin.site.register(Relation) +admin.site.register(EntityRelation) +admin.site.register(ProjectGroup, ProjectGroupAdmin) +admin.site.register(ProjectAnnotateEntities, ProjectAnnotateEntitiesAdmin) +admin.site.register(AnnotatedEntity, AnnotatedEntityAdmin) +admin.site.register(ConceptDB, ConceptDBAdmin) +admin.site.register(Document, DocumentAdmin) +admin.site.register(ExportedProject, ExportedProjectAdmin) +admin.site.register(ProjectMetrics, ProjectMetricsAdmin) +admin.site.register(Dataset, DatasetAdmin) diff --git a/webapp/api/api/admin.py b/webapp/api/api/admin/actions.py similarity index 81% rename from webapp/api/api/admin.py rename to webapp/api/api/admin/actions.py index 1cff5fa..feb754c 100644 --- a/webapp/api/api/admin.py +++ b/webapp/api/api/admin/actions.py @@ -1,26 +1,17 @@ import copy +import json import logging from datetime import datetime from io import StringIO from typing import Dict, List from background_task import background -from django.contrib import admin from django.db.models import QuerySet -from django.http import HttpResponse, HttpResponseRedirect +from django.http import HttpResponse from rest_framework.exceptions import PermissionDenied -from .forms import * -from .models import * -from .solr_utils import import_all_concepts, drop_collection - -admin.site.register(Entity) -admin.site.register(MetaTaskValue) -admin.site.register(MetaTask) -admin.site.register(MetaAnnotation) -admin.site.register(Vocabulary) -admin.site.register(Relation) -admin.site.register(EntityRelation) +from api.models import AnnotatedEntity, MetaAnnotation, EntityRelation, Document, ConceptDB +from api.solr_utils import drop_collection, import_all_concepts logger = logging.getLogger(__name__) @@ -157,6 +148,10 @@ def retrieve_project_data(projects: QuerySet) -> Dict[str, List]: { "name": "" # the auto-generated id of the project (optional) + "project_group_id": "" # the auto-generated id of the project - nullable + "project_group_name": "" # the name of the project group if set. + "project_status": "" # status - either annotating, discontinued, complete + "project_locked": "" # locked - for no further annotations "cuis": ["cui_1", "cui_2" ... ] # the CUI filter for the project, includes those from file / and text-box "meta_anno_defs": [ # list of meta annotation tasks configured for this project. @@ -241,6 +236,8 @@ def retrieve_project_data(projects: QuerySet) -> Dict[str, List]: out['name'] = project.name out['id'] = project.id out['cuis'] = project.cuis + out['project_group_id'] = project.group.id if project.group else None + out['project_group_name'] = project.group.name if project.group else None out['project_status'] = project.project_status out['project_locked'] = project.project_locked out['meta_anno_defs'] = [{'name': t.name, 'values': [v.name for v in t.values.all()]} @@ -351,64 +348,10 @@ def clone_projects(modeladmin, request, queryset): project_copy.save() -class ReportErrorModelAdminMixin: - """Mixin to catch all errors in the Django Admin and map them to user-visible errors.""" - def changeform_view(self, request, object_id=None, form_url='', extra_context=None): - try: - return super().changeform_view(request, object_id, form_url, extra_context) - except Exception as e: - self.message_user(request, f'Error with previous action: {e}', level=logging.ERROR) - return HttpResponseRedirect(request.path) - - def dataset_document_counts(dataset): return f'{Document.objects.filter(dataset=dataset).count()}' -dataset_document_counts.short_description = 'Document Count' - - -class DatasetAdmin(ReportErrorModelAdminMixin, admin.ModelAdmin): - model = Dataset - form = DatasetForm - list_display = ['name', 'create_time', 'description', dataset_document_counts] - - -admin.site.register(Dataset, DatasetAdmin) -class ProjectAnnotateEntitiesAdmin(admin.ModelAdmin): - model = ProjectAnnotateEntities - actions = [download, download_without_text, download_without_text_with_doc_names, reset_project, clone_projects] - list_filter = ('members', 'project_status', 'project_locked', 'annotation_classification') - list_display = ['name'] - - def formfield_for_foreignkey(self, db_field, request, **kwargs): - if db_field.name == 'concept_db': - kwargs['queryset'] = ConceptDB.objects.filter(use_for_training=True) - return super(ProjectAnnotateEntitiesAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs) - - def formfield_for_manytomany(self, db_field, request, **kwargs): - if db_field.name == 'cdb_search_filter': - kwargs['queryset'] = ConceptDB.objects.all() - if db_field.name == 'validated_documents': - project_id = request.path.replace('/admin/api/projectannotateentities/', '').split('/')[0] - try: - proj = ProjectAnnotateEntities.objects.get(id=int(project_id)) - kwargs['queryset'] = Document.objects.filter(dataset=proj.dataset.id) - except ValueError: # a blank project has no validated_documents - kwargs['queryset'] = Document.objects.none() - return super(ProjectAnnotateEntitiesAdmin, self).formfield_for_manytomany(db_field, request, **kwargs) - - -admin.site.register(ProjectAnnotateEntities, ProjectAnnotateEntitiesAdmin) - - -class AnnotatedEntityAdmin(admin.ModelAdmin): - list_display = ('user', 'project', 'entity', 'value', 'deleted', 'validated') - list_filter = ('user', 'project', 'deleted', 'validated') - model = AnnotatedEntity -admin.site.register(AnnotatedEntity, AnnotatedEntityAdmin) - - @background(schedule=5) def _reset_cdb_filters(id): from medcat.cdb import CDB @@ -444,38 +387,5 @@ def delete_indexed_concepts(modeladmin, request, queryset): drop_collection(concept_db) -class ConceptDBAdmin(admin.ModelAdmin): - model = ConceptDB - actions = [import_concepts, delete_indexed_concepts, reset_cdb_filters] - -admin.site.register(ConceptDB, ConceptDBAdmin) - - def remove_all_documents(modeladmin, request, queryset): Document.objects.all().delete() - - -class DocumentAdmin(admin.ModelAdmin): - model = Document - actions = [remove_all_documents] - list_filter = ('dataset',) - list_display = ['name', 'create_time', 'dataset', 'last_modified'] - - -admin.site.register(Document, DocumentAdmin) - - -class ExportedProjectAdmin(admin.ModelAdmin): - model = ExportedProject - - -admin.site.register(ExportedProject, ExportedProjectAdmin) - - -class ProjectMetricsAdmin(admin.ModelAdmin): - model = ProjectMetrics - list_display = ('report_name', 'report_name_generated') - list_filter = ['projects'] - - -admin.site.register(ProjectMetrics, ProjectMetricsAdmin) diff --git a/webapp/api/api/admin/models.py b/webapp/api/api/admin/models.py new file mode 100644 index 0000000..76f7c8a --- /dev/null +++ b/webapp/api/api/admin/models.py @@ -0,0 +1,172 @@ +from django.contrib import admin +from django.contrib.auth.models import User +from django.forms import fields +from django.http import HttpResponseRedirect +from django.shortcuts import get_object_or_404 + +from .actions import * +from ..models import * + +_PROJECT_ANNO_ENTS_SETTINGS_FIELD_ORDER = ( + 'concept_db', 'vocab', 'cdb_search_filter', 'require_entity_validation', 'train_model_on_submit', + 'add_new_entities', 'restrict_concept_lookup', 'terminate_available', 'irrelevant_available', + 'enable_entity_annotation_comments', 'tasks', 'relations' +) + +_PROJECT_FIELDS_ORDER = ( + 'cuis', 'cuis_file', 'annotation_classification', 'project_locked', 'project_status' +) + + +class ReportErrorModelAdminMixin: + """Mixin to catch all errors in the Django Admin and map them to user-visible errors.""" + def changeform_view(self, request, object_id=None, form_url='', extra_context=None): + try: + return super().changeform_view(request, object_id, form_url, extra_context) + except Exception as e: + self.message_user(request, f'Error with previous action: {e}', level=logging.ERROR) + return HttpResponseRedirect(request.path) + + + +dataset_document_counts.short_description = 'Document Count' + + +class DatasetAdmin(ReportErrorModelAdminMixin, admin.ModelAdmin): + model = Dataset + form = DatasetForm + list_display = ['name', 'create_time', 'description', dataset_document_counts] + + +class ProjectAnnotateEntitiesAdmin(admin.ModelAdmin): + model = ProjectAnnotateEntities + actions = [download, download_without_text, download_without_text_with_doc_names, reset_project, clone_projects] + list_filter = ('members', 'project_status', 'project_locked', 'annotation_classification') + list_display = ['name'] + fields = (('group', 'name', 'description', 'annotation_guideline_link', 'members', 'dataset', 'validated_documents') + + _PROJECT_FIELDS_ORDER + + _PROJECT_ANNO_ENTS_SETTINGS_FIELD_ORDER) + + def formfield_for_foreignkey(self, db_field, request, **kwargs): + if db_field.name == 'concept_db': + kwargs['queryset'] = ConceptDB.objects.filter(use_for_training=True) + return super(ProjectAnnotateEntitiesAdmin, self).formfield_for_foreignkey(db_field, request, **kwargs) + + def formfield_for_manytomany(self, db_field, request, **kwargs): + if db_field.name == 'cdb_search_filter': + kwargs['queryset'] = ConceptDB.objects.all() + if db_field.name == 'validated_documents': + project_id = request.path.replace('/admin/api/projectannotateentities/', '').split('/')[0] + try: + proj = ProjectAnnotateEntities.objects.get(id=int(project_id)) + kwargs['queryset'] = Document.objects.filter(dataset=proj.dataset.id) + except ValueError: # a blank project has no validated_documents + kwargs['queryset'] = Document.objects.none() + return super(ProjectAnnotateEntitiesAdmin, self).formfield_for_manytomany(db_field, request, **kwargs) + + +class ProjectGroupAdmin(admin.ModelAdmin): + model = ProjectGroup + list_display = ('name', 'description') + fields = (('name', 'description', 'create_associated_projects', 'annotation_guideline_link', 'administrators', + 'annotators', 'dataset') + + _PROJECT_FIELDS_ORDER + _PROJECT_ANNO_ENTS_SETTINGS_FIELD_ORDER) + + class Meta: + model = ProjectGroup + + def get_form(self, request, obj=None, change=False, **kwargs): + form = super().get_form(request, obj, **kwargs) + return form + + def _set_proj_from_group(self, proj: ProjectAnnotateEntities, group: ProjectGroup, + annotator: settings.AUTH_USER_MODEL, admins: List[User], + cdb_search_filters: List[ConceptDB], tasks: List[MetaTask], relations: List[Relation]): + proj.group = group + proj.name = f'{group.name} - {str(annotator)}' + proj.description = group.description + proj.dataset = group.dataset + proj.annotation_guideline_link = group.annotation_guideline_link + proj.create_time = group.create_time + proj.cuis = group.cuis + proj.cuis_file = group.cuis_file + proj.annotation_classification = group.annotation_classification + proj.project_locked = group.project_locked + proj.project_status = group.project_status + proj.concept_db = group.concept_db + proj.vocab = group.vocab + proj.require_entity_validation = group.require_entity_validation + proj.train_model_on_submit = group.train_model_on_submit + proj.add_new_entities = group.add_new_entities + proj.restrict_concept_lookup = group.restrict_concept_lookup + proj.terminate_available = group.terminate_available + proj.irrelevant_available = group.irrelevant_available + proj.enable_entity_annotation_comments = group.enable_entity_annotation_comments + + # project specific attrs / m2m fields + proj.save() + proj.cdb_search_filter.set(cdb_search_filters) + proj.members.set(admins) + proj.members.add(annotator) + proj.tasks.set(tasks) + proj.relations.set(relations) + proj.save() + return proj + + def save_model(self, request, obj, form, change): + super().save_model(request, obj, form, change) + annotators = [get_object_or_404(User, pk=id) for id in request.POST.getlist('annotators')] + admins = [get_object_or_404(User, pk=id) for id in request.POST.getlist('administrators')] + cdb_search_filters = [get_object_or_404(ConceptDB, pk=id) for id in request.POST.getlist('cdb_search_filter')] + tasks = [get_object_or_404(MetaTask, pk=id) for id in request.POST.getlist('tasks')] + relations = [get_object_or_404(Relation, pk=id) for id in request.POST.getlist('relations')] + + # create the underlying ProjectAnnotateEntities models or edit them + if obj.create_associated_projects: + if not change: + # new ProjectGroup being created + for annotator in annotators: + self._set_proj_from_group(ProjectAnnotateEntities(), obj, annotator, + admins, cdb_search_filters, tasks, relations) + else: + # applying these settings to all previously created projects within this group + projs = ProjectAnnotateEntities.objects.filter(group=obj) + if len(projs) == len(obj.annotators.all()): + for proj, annotator in zip(projs, obj.annotators.all()): + self._set_proj_from_group(proj, obj, annotator, admins, cdb_search_filters, + tasks, relations) + else: + raise ValueError("Attempting to update a ProjectGroup but one or more " + "of underlying ProjectAnnotateEntities have been removed / or added " + "manually. To fix, go into each project separately, or create new projects " + "and link to the ProjectGroup within ProjectAnnotateEntities page.") + + +class AnnotatedEntityAdmin(admin.ModelAdmin): + list_display = ('user', 'project', 'entity', 'value', 'deleted', 'validated') + list_filter = ('user', 'project', 'deleted', 'validated') + model = AnnotatedEntity + + +class ConceptDBAdmin(admin.ModelAdmin): + model = ConceptDB + actions = [import_concepts, delete_indexed_concepts, reset_cdb_filters] + + +class DocumentAdmin(admin.ModelAdmin): + model = Document + actions = [remove_all_documents] + list_filter = ('dataset',) + list_display = ['name', 'create_time', 'dataset', 'last_modified'] + + +class ExportedProjectAdmin(admin.ModelAdmin): + model = ExportedProject + + +class ProjectMetricsAdmin(admin.ModelAdmin): + model = ProjectMetrics + list_display = ('report_name', 'report_name_generated') + list_filter = ['projects'] + + diff --git a/webapp/api/api/migrations/0076_auto_20240510_1451.py b/webapp/api/api/migrations/0076_auto_20240510_1451.py new file mode 100644 index 0000000..571d974 --- /dev/null +++ b/webapp/api/api/migrations/0076_auto_20240510_1451.py @@ -0,0 +1,115 @@ +# Generated by Django 2.2.28 on 2024-05-10 14:51 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('api', '0075_auto_20240429_1350'), + ] + + operations = [ + migrations.AlterField( + model_name='project', + name='annotation_classification', + field=models.BooleanField(default=False, help_text='If these annotations are suitable for training a general purpose model. If in doubt uncheck this.'), + ), + migrations.AlterField( + model_name='project', + name='annotation_guideline_link', + field=models.TextField(blank=True, default='', help_text='link to an external document (i.e. GoogleDoc, MS Sharepoint)outlining a guide for annotators to follow for this project,an example is available here: https://docs.google.com/document/d/1xxelBOYbyVzJ7vLlztP2q1Kw9F5Vr1pRwblgrXPS7QM/edit?usp=sharing'), + ), + migrations.AlterField( + model_name='project', + name='cuis', + field=models.TextField(blank=True, default=None, help_text='A list of comma seperated concept unique identifiers (CUIs) to be annotated'), + ), + migrations.AlterField( + model_name='project', + name='dataset', + field=models.ForeignKey(help_text='The dataset to be annotated.', on_delete=django.db.models.deletion.CASCADE, to='api.Dataset'), + ), + migrations.AlterField( + model_name='project', + name='description', + field=models.TextField(blank=True, default='', help_text='A short description of the annotations to be collected and why'), + ), + migrations.AlterField( + model_name='project', + name='members', + field=models.ManyToManyField(help_text='The list users that have access to this annotation project', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='project', + name='name', + field=models.CharField(help_text='A name of the annotation project', max_length=150), + ), + migrations.AlterField( + model_name='project', + name='project_locked', + field=models.BooleanField(default=False, help_text='Locked indicates annotation collection is complete and this dataset should not be touched any further.'), + ), + migrations.AlterField( + model_name='project', + name='project_status', + field=models.CharField(choices=[('A', 'Annotating'), ('D', 'Discontinued (Fail)'), ('C', 'Complete')], default='A', help_text='The status of the annotation collection exercise', max_length=1), + ), + migrations.AlterField( + model_name='projectannotateentities', + name='concept_db', + field=models.ForeignKey(help_text='The MedCAT CDB used to annotate / validate', null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.ConceptDB'), + ), + migrations.AlterField( + model_name='projectannotateentities', + name='tasks', + field=models.ManyToManyField(blank=True, default=None, help_text='The set of MetaAnnotation tasks configured for this project', to='api.MetaTask'), + ), + migrations.AlterField( + model_name='projectannotateentities', + name='vocab', + field=models.ForeignKey(help_text='The MedCAT Vocab used to annotate / validate', null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.Vocabulary'), + ), + migrations.CreateModel( + name='ProjectGroup', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='A name of the annotation project', max_length=150)), + ('description', models.TextField(blank=True, default='', help_text='A short description of the annotations to be collected and why')), + ('annotation_guideline_link', models.TextField(blank=True, default='', help_text='link to an external document (i.e. GoogleDoc, MS Sharepoint)outlining a guide for annotators to follow for this project,an example is available here: https://docs.google.com/document/d/1xxelBOYbyVzJ7vLlztP2q1Kw9F5Vr1pRwblgrXPS7QM/edit?usp=sharing')), + ('create_time', models.DateTimeField(auto_now_add=True)), + ('last_modified', models.DateTimeField(auto_now=True)), + ('cuis', models.TextField(blank=True, default=None, help_text='A list of comma seperated concept unique identifiers (CUIs) to be annotated')), + ('cuis_file', models.FileField(blank=True, help_text='A file containing a JSON formatted list of CUI code strings, i.e. ["1234567","7654321"]', null=True, upload_to='')), + ('annotation_classification', models.BooleanField(default=False, help_text='If these annotations are suitable for training a general purpose model. If in doubt uncheck this.')), + ('project_locked', models.BooleanField(default=False, help_text='Locked indicates annotation collection is complete and this dataset should not be touched any further.')), + ('project_status', models.CharField(choices=[('A', 'Annotating'), ('D', 'Discontinued (Fail)'), ('C', 'Complete')], default='A', help_text='The status of the annotation collection exercise', max_length=1)), + ('require_entity_validation', models.BooleanField(default=True, help_text='Entities appear grey and are required to be validated before submission')), + ('train_model_on_submit', models.BooleanField(default=True, help_text='Active learning - configured CDB is trained on each submit')), + ('add_new_entities', models.BooleanField(default=False, help_text='Allow the creation of new terms to be added to the CDB')), + ('restrict_concept_lookup', models.BooleanField(default=False, help_text='Users can only search for concept terms from the list configured for the project, i.e. either from the cuis or cuis_file lists. Checking this when bothcuis and cuis_file are empty does nothing. If "add new entities" is available & added, and cuis or cuis_fileis non-empty the new CUI will be added.')), + ('terminate_available', models.BooleanField(default=True, help_text='Enable the option to terminate concepts.')), + ('irrelevant_available', models.BooleanField(default=False, help_text='Enable the option to add the irrelevant button.')), + ('enable_entity_annotation_comments', models.BooleanField(default=False, help_text='Enable to allow annotators to leave comments for each annotation')), + ('administrators', models.ManyToManyField(help_text='The set of users that will have visibility of all projects in this project group', related_name='administrators', to=settings.AUTH_USER_MODEL)), + ('annotators', models.ManyToManyField(help_text='The set of users that will each be provided an annotation project', related_name='annotators', to=settings.AUTH_USER_MODEL)), + ('cdb_search_filter', models.ManyToManyField(blank=True, default=None, help_text='The CDB that will be used for concept lookup. This specific CDB should have been "imported" via the CDB admin screen', related_name='project_group_concept_source', to='api.ConceptDB')), + ('concept_db', models.ForeignKey(help_text='The MedCAT CDB used to annotate / validate', null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.ConceptDB')), + ('dataset', models.ForeignKey(help_text='The dataset to be annotated.', on_delete=django.db.models.deletion.CASCADE, to='api.Dataset')), + ('relations', models.ManyToManyField(blank=True, default=None, help_text='Relations that will be available for this project', to='api.Relation')), + ('tasks', models.ManyToManyField(blank=True, default=None, help_text='The set of MetaAnnotation tasks configured for this project', to='api.MetaTask')), + ('vocab', models.ForeignKey(help_text='The MedCAT Vocab used to annotate / validate', null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.Vocabulary')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='project', + name='group', + field=models.ForeignKey(blank=True, help_text='The annotation project group that this project is part of', null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.ProjectGroup'), + ), + ] diff --git a/webapp/api/api/migrations/0077_projectgroup_create_associated_projects.py b/webapp/api/api/migrations/0077_projectgroup_create_associated_projects.py new file mode 100644 index 0000000..0905d55 --- /dev/null +++ b/webapp/api/api/migrations/0077_projectgroup_create_associated_projects.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2024-05-14 12:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0076_auto_20240510_1451'), + ] + + operations = [ + migrations.AddField( + model_name='projectgroup', + name='create_associated_projects', + field=models.BooleanField(default=True, help_text='This only functions on new Project Group entries. If creating a new Project Group and this is checked, it will create a ProjectAnnotateEntities for each annotator. If unchecked it will not create associated ProjectAnnotateEntities instead, leaving the admin to manually configure groups of projects.'), + ), + ] diff --git a/webapp/api/api/models.py b/webapp/api/api/models.py index 85d6fe1..96f385a 100644 --- a/webapp/api/api/models.py +++ b/webapp/api/api/models.py @@ -92,34 +92,45 @@ def __str__(self): return f'{self.name} | {self.dataset.name} | {self.dataset.id}' -class Project(PolymorphicModel): +class ProjectFields(models.Model): + class Meta: + abstract = True + PROJECT_STATUSES = [ ("A", "Annotating"), ("D", "Discontinued (Fail)"), ("C", "Complete"), ] - name = models.CharField(max_length=150) - description = models.TextField(default="", blank=True) + + name = models.CharField(max_length=150, help_text='A name of the annotation project') + description = models.TextField(default="", blank=True, help_text='A short description of the annotations to be ' + 'collected and why') + dataset = models.ForeignKey('Dataset', on_delete=models.CASCADE, help_text='The dataset to be annotated.') annotation_guideline_link = models.TextField(default="", blank=True, help_text="link to an external document (i.e. GoogleDoc, MS Sharepoint)" - "outlininng a guide for annotators to follow for this project," + "outlining a guide for annotators to follow for this project," "an example is available here: https://docs.google.com/document/d/1xxelBOYbyVzJ7vLlztP2q1Kw9F5Vr1pRwblgrXPS7QM/edit?usp=sharing") create_time = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) - members = models.ManyToManyField(settings.AUTH_USER_MODEL) - dataset = models.ForeignKey('Dataset', on_delete=models.CASCADE) - validated_documents = models.ManyToManyField(Document, default=None, blank=True) - cuis = models.TextField(default=None, blank=True) + cuis = models.TextField(default=None, blank=True, help_text='A list of comma seperated concept unique identifiers (CUIs) to be annotated') cuis_file = models.FileField(null=True, blank=True, help_text='A file containing a JSON formatted list of CUI code strings, ' 'i.e. ["1234567","7654321"]') - annotation_classification = models.BooleanField(default=False, help_text="If these project annotations are suitable" - " for training a general purpose model. If" + annotation_classification = models.BooleanField(default=False, help_text="If these annotations are suitable " + "for training a general purpose model. If" " in doubt uncheck this.") - project_locked = models.BooleanField(default=False, help_text="If this project is locked and cannot or should " + project_locked = models.BooleanField(default=False, help_text="Locked indicates annotation collection is complete and this dataset should " "not be touched any further.") project_status = models.CharField(max_length=1, choices=PROJECT_STATUSES, default="A", - help_text="The status of the project to indicate the readiness") + help_text="The status of the annotation collection exercise") + + +class Project(PolymorphicModel, ProjectFields): + members = models.ManyToManyField(settings.AUTH_USER_MODEL, + help_text='The list users that have access to this annotation project') + group = models.ForeignKey('ProjectGroup', on_delete=models.SET_NULL, blank=True, null=True, + help_text='The annotation project group that this project is part of') + validated_documents = models.ManyToManyField(Document, default=None, blank=True) def __str__(self): return str(self.name) @@ -214,9 +225,17 @@ def __str__(self): return str(self.name) -class ProjectAnnotateEntities(Project): - concept_db = models.ForeignKey('ConceptDB', on_delete=models.SET_NULL, blank=False, null=True) - vocab = models.ForeignKey('Vocabulary', on_delete=models.SET_NULL, null=True) +class ProjectAnnotateEntitiesFields(models.Model): + """ + Abstract class for all model fields for ProjectAnnotateEntities models. + """ + class Meta: + abstract = True + + concept_db = models.ForeignKey('ConceptDB', on_delete=models.SET_NULL, blank=False, null=True, + help_text='The MedCAT CDB used to annotate / validate') + vocab = models.ForeignKey('Vocabulary', on_delete=models.SET_NULL, null=True, + help_text='The MedCAT Vocab used to annotate / validate') cdb_search_filter = models.ManyToManyField('ConceptDB', blank=True, default=None, help_text='The CDB that will be used for concept lookup. ' 'This specific CDB should have been "imported" ' @@ -243,11 +262,44 @@ class ProjectAnnotateEntities(Project): enable_entity_annotation_comments = models.BooleanField(default=False, help_text="Enable to allow annotators to leave comments" " for each annotation") - tasks = models.ManyToManyField(MetaTask, blank=True, default=None) - relations = models.ManyToManyField(Relation, blank=True, default=None, + tasks = models.ManyToManyField('MetaTask', blank=True, default=None, + help_text='The set of MetaAnnotation tasks configured for this project') + relations = models.ManyToManyField('Relation', blank=True, default=None, help_text='Relations that will be available for this project') +class ProjectAnnotateEntities(Project, ProjectAnnotateEntitiesFields): + """ + Class for any single ProjectAnnotateEntities model fields that should not be inherited by ProjectGroup + In practise its unlikely further fields are needed. + """ + pass + + +class ProjectGroup(ProjectFields, ProjectAnnotateEntitiesFields): + administrators = models.ManyToManyField(settings.AUTH_USER_MODEL, + help_text="The set of users that will have visibility of all " + "projects in this project group", related_name='administrators') + annotators = models.ManyToManyField(settings.AUTH_USER_MODEL, + help_text="The set of users that will each be provided an annotation project", + related_name='annotators') + cdb_search_filter = models.ManyToManyField('ConceptDB', blank=True, default=None, + help_text='The CDB that will be used for concept lookup. ' + 'This specific CDB should have been "imported" ' + 'via the CDB admin screen', + related_name='project_group_concept_source') + create_associated_projects = models.BooleanField(default=True, + help_text='This only functions on new Project Group entries. ' + ' If creating a new Project Group and this is checked, ' + 'it will create a ProjectAnnotateEntities for each' + ' annotator. If unchecked it will not create associated' + ' ProjectAnnotateEntities instead, leaving the admin to ' + ' manually configure groups of projects.') + + def __str__(self): + return self.name + + class MetaAnnotation(models.Model): annotated_entity = models.ForeignKey('AnnotatedEntity', on_delete=models.CASCADE) meta_task = models.ForeignKey('MetaTask', on_delete=models.CASCADE) diff --git a/webapp/api/api/serializers.py b/webapp/api/api/serializers.py index b9d15e2..9be1430 100644 --- a/webapp/api/api/serializers.py +++ b/webapp/api/api/serializers.py @@ -72,6 +72,23 @@ def to_representation(self, instance): return data +class ProjectGroupSerializer(serializers.ModelSerializer): + class Meta: + model = ProjectGroup + fields = '__all__' + + def to_representation(self, instance): + # enrich with other fields? last_modified etc.? + data = super(ProjectGroupSerializer, self).to_representation(instance) + projects = ProjectAnnotateEntities.objects.filter(group=instance) + if len(projects) > 0: + projects = sorted(projects, key=lambda p: p.last_modified, reverse=True) + data['last_modified'] = projects[0].last_modified + else: + data['last_modified'] = None + return data + + class DocumentSerializer(serializers.ModelSerializer): class Meta: model = Document diff --git a/webapp/api/api/views.py b/webapp/api/api/views.py index b94f0de..a98ffed 100644 --- a/webapp/api/api/views.py +++ b/webapp/api/api/views.py @@ -19,7 +19,8 @@ from core.settings import MEDIA_ROOT from .admin import download_projects_with_text, download_projects_without_text, \ - import_concepts_from_cdb, upload_projects_export, retrieve_project_data + import_concepts_from_cdb +from .data_utils import upload_projects_export from .medcat_utils import ch2pt_from_pt2ch, get_all_ch, dedupe_preserve_order, snomed_ct_concept_path from .metrics import calculate_metrics from .permissions import * @@ -83,6 +84,21 @@ def get_queryset(self): return projects +class ProjectGroupFilter(drf.FilterSet): + id__in = NumInFilter(field_name='id', lookup_expr='in') + + class Meta: + model = ProjectGroup + fields = ['id', 'name', 'description'] + +class ProjectGroupViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + queryset = ProjectGroup.objects.all() + serializer_class = ProjectGroupSerializer + filterset_fields = ['id'] + filterset_class = ProjectGroupFilter + + class AnnotatedEntityFilter(drf.FilterSet): id__in = NumInFilter(field_name='id', lookup_expr='in') diff --git a/webapp/api/core/urls.py b/webapp/api/core/urls.py index 5855999..732dff7 100644 --- a/webapp/api/core/urls.py +++ b/webapp/api/core/urls.py @@ -24,6 +24,7 @@ router.register(r'users', api.views.UserViewSet) router.register(r'entities', api.views.EntityViewSet) router.register(r'project-annotate-entities', api.views.ProjectAnnotateEntitiesViewSet) +router.register(r'project-groups', api.views.ProjectGroupViewSet) router.register(r'documents', api.views.DocumentViewSet) router.register(r'annotated-entities', api.views.AnnotatedEntityViewSet) router.register(r'meta-annotations', api.views.MetaAnnotationViewSet) diff --git a/webapp/frontend/src/components/common/ProjectList.vue b/webapp/frontend/src/components/common/ProjectList.vue new file mode 100644 index 0000000..4af2b56 --- /dev/null +++ b/webapp/frontend/src/components/common/ProjectList.vue @@ -0,0 +1,428 @@ + + + + + diff --git a/webapp/frontend/src/views/Home.vue b/webapp/frontend/src/views/Home.vue index e39bffa..2c1cdca 100644 --- a/webapp/frontend/src/views/Home.vue +++ b/webapp/frontend/src/views/Home.vue @@ -1,172 +1,36 @@ @@ -422,111 +183,26 @@ h3 { margin: 10% } -.table-container { - height: calc(100% - 125px); - overflow-y: auto; +.view-bar { + height: 30px; + padding: 5px 0; + width: 95%; + margin: auto; } -.project-table { +.project-group-table { + height: calc(100% - 30px); padding: 10px 0; width: 95%; margin: auto; } -.status-cell { - text-align: center; -} - -.status-unlocked { - text-align: center; - color: $color-1; - padding: 0 5px; - opacity: .5; -} - -.status-locked { - @extend .status-unlocked; - opacity: 1; - color: $danger; -} - -.term-list { - display: block; - max-width: 20vw; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -} .home-title { font-size: 23px; padding: 30px 0; } -.complete-project, .success { - color: $success; - font-size: 20px; -} - -.danger { - color: $danger; - font-size: 20px; -} - -.model-up { - position: relative; - &:hover { - cursor: initial !important; - } -} - -.clear-model-cache, { - font-size: 15px; - color: $task-color-2; - cursor: pointer; - position: absolute; - right: -5px; - top: -5px; -} - -.selected-project { - font-size: 15px; - color: green; - position: relative; - right: -40px; - top: -10px; -} - -.load-metrics { - padding: 0 5px; - -} - -.progress-container { - position: relative; - padding-left: 2px; -} - -.progress-gradient-fill { - position: absolute; - z-index: -1; - top: 0; - height: 25px; - padding: 0 1px; - background-image: linear-gradient(to right, #32ab60, #E8EDEE); - box-shadow: 0 5px 5px -5px #32ab60; -} -.good-perf { - color: #E5EBEA; -} - -.bad-perf { - color: #45503B; -} - -.submit-report-job-alert { - text-align: right; -}