Merge branch 'develop' into release-2.2

aether-client-library/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim-buster
+FROM python:3.8-slim
 
 ARG VERSION=0.0.0
 

aether-entity-extraction-module/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim-buster
+FROM python:3.8-slim
 
 LABEL description="Aether Entity Extraction Module" \
       name="aether-extractor" \

aether-entity-extraction-module/conf/pip/requirements.txt

@@ -13,43 +13,44 @@
 ################################################################################
 
 aether.python==1.3.0
+async-timeout==4.0.2
 attrs==21.4.0
 avro-python3==1.10.2
-certifi==2021.10.8
-charset-normalizer==2.0.10
-coverage==6.2
+certifi==2022.6.15
+charset-normalizer==2.0.12
+coverage==6.4.1
 decorator==5.1.1
 Deprecated==1.2.13
 eha-jsonpath==0.6.0
-fakeredis==1.7.0
+fakeredis==1.8.1
 flake8==4.0.1
 flake8-quotes==3.3.1
 gevent==21.12.0
 greenlet==1.1.2
 idna==3.3
-importlib-resources==5.4.0
+importlib-resources==5.8.0
 iniconfig==1.1.1
 jsonpath-ng==1.5.3
-jsonschema==4.4.0
+jsonschema==4.6.0
 mccabe==0.6.1
 packaging==21.3
 pluggy==1.0.0
 ply==3.11
 py==1.11.0
 pycodestyle==2.8.0
 pyflakes==2.4.0
-pyparsing==3.0.7
+pyparsing==3.0.9
 pyrsistent==0.18.1
-pytest==6.2.5
-redis==4.0.2
-requests==2.27.1
+pytest==7.1.2
+redis==4.3.3
+requests==2.28.0
 six==1.16.0
 sortedcontainers==2.4.0
 spavro==1.1.24
 tblib==1.7.0
-toml==0.10.2
-urllib3==1.26.8
-wrapt==1.13.3
-zipp==3.7.0
+tomli==2.0.1
+urllib3==1.26.9
+wrapt==1.14.1
+zipp==3.8.0
 zope.event==4.5.0
 zope.interface==5.4.0

aether-kernel/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim-buster
+FROM python:3.8-slim
 
 LABEL description="Aether Kernel" \
       name="aether-kernel" \

aether-kernel/aether/kernel/api/tests/test_views.py

@@ -310,7 +310,7 @@ def test_validate_mappings__success(self):
             'schemas': {'Person': EXAMPLE_SCHEMA},
         })
         response = self.client.post(url, data=data, content_type='application/json')
-        response_data = json.loads(response.content)
+        response_data = response.json()
         self.assertEqual(
             len(response_data['entities']),
             len(EXAMPLE_SOURCE_DATA['data']['people']),
@@ -341,7 +341,7 @@ def test_validate_mappings__failure(self):
             },
         })
         response = self.client.post(url, data=data, content_type='application/json')
-        response_data = json.loads(response.content)
+        response_data = response.json()
         self.assertEqual(len(response_data['entities']), 0, response_data)
         expected = [
             'Could not find schema "person"',
@@ -372,7 +372,7 @@ def test_validate_mappings__400_BAD_REQUEST(self):
             },
         })
         response = self.client.post(url, data=data, content_type='application/json')
-        response_data = json.loads(response.content)
+        response_data = response.json()
         self.assertEqual(response.status_code, 400)
         self.assertIn('This field is required', response_data['schemas'][0])
         self.assertIn('This field is required', response_data['submission_payload'][0])
@@ -385,7 +385,7 @@ def test_validate_mappings__400_BAD_REQUEST(self):
             'schemas': [],
         })
         response = self.client.post(url, data=data, content_type='application/json')
-        response_data = json.loads(response.content)
+        response_data = response.json()
         self.assertEqual(response.status_code, 400)
         self.assertEqual('Value [] is not an Object', response_data['schemas'][0])
 
@@ -409,7 +409,7 @@ def helper_read_linked_data_entities(self, obj, depth):
         url = reverse('entity-detail', kwargs={'pk': obj.pk}) + '?depth=' + str(depth)
         response = self.client.get(url)
         self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)
-        return json.loads(response.content)
+        return response.json()
 
     def test_read_linked_data(self):
         location_schema = models.Schema.objects.create(
@@ -612,7 +612,7 @@ def test_schema_validate_definition__errors(self):
 
         for schema in bad_schemas:
             response = self.client.post(url, json.dumps(schema), content_type='application/json')
-            response_content = json.loads(response.content)
+            response_content = response.json()
             self.assertIn(
                 'A schema is required to have a field "id" of type "string"',
                 response_content['definition'][0],
@@ -1193,7 +1193,7 @@ def test__generate_avro_input(self):
         # no data
         response = self.client.post(url)
         self.assertEqual(response.status_code, 400)
-        data = json.loads(response.content)
+        data = response.json()
         self.assertEqual(data['message'], 'Missing "schema" data')
 
         # from schema to input
@@ -1217,7 +1217,7 @@ def test__generate_avro_input(self):
             content_type='application/json',
         )
         self.assertEqual(response.status_code, 200)
-        data = json.loads(response.content)
+        data = response.json()
 
         self.assertEqual(data['schema'], schema)
         # input conforms the schema

aether-kernel/aether/kernel/api/views.py

@@ -52,7 +52,7 @@
 class ProjectViewSet(MtViewSetMixin, FilteredMixin, ExtractMixin, viewsets.ModelViewSet):
     queryset = models.Project.objects.all()
     serializer_class = serializers.ProjectSerializer
-    filter_class = filters.ProjectFilter
+    filterset_class = filters.ProjectFilter
     search_fields = ('name',)
 
     @action(detail=True, methods=['patch'], url_name='erase_data', url_path='delete-data')
@@ -320,7 +320,7 @@ def __upsert_schemas(self, request, pk=None):
 class MappingSetViewSet(MtViewSetMixin, FilteredMixin, ExtractMixin, viewsets.ModelViewSet):
     queryset = models.MappingSet.objects.all()
     serializer_class = serializers.MappingSetSerializer
-    filter_class = filters.MappingSetFilter
+    filterset_class = filters.MappingSetFilter
     search_fields = ('name',)
     mt_field = 'project'
 
@@ -342,7 +342,7 @@ def delete_artefacts(self, request, pk=None, *args, **kwargs):
 class MappingViewSet(MtViewSetMixin, FilteredMixin, viewsets.ModelViewSet):
     queryset = models.Mapping.objects.all()
     serializer_class = serializers.MappingSerializer
-    filter_class = filters.MappingFilter
+    filterset_class = filters.MappingFilter
     search_fields = ('name',)
     mt_field = 'mappingset__project'
 
@@ -370,7 +370,7 @@ def topics(self, request, pk=None, *args, **kwargs):
 class SubmissionViewSet(MtViewSetMixin, FilteredMixin, ExtractMixin, ExporterMixin, viewsets.ModelViewSet):
     queryset = models.Submission.objects.all().prefetch_related('attachments')
     serializer_class = serializers.SubmissionSerializer
-    filter_class = filters.SubmissionFilter
+    filterset_class = filters.SubmissionFilter
     search_fields = ('project__name', 'mappingset__name',)
     mt_field = 'project'
 
@@ -525,7 +525,7 @@ def destroy(self, request, pk=None, *args, **kwargs):
 class AttachmentViewSet(MtViewSetMixin, FilteredMixin, viewsets.ModelViewSet):
     queryset = models.Attachment.objects.all().prefetch_related('submission')
     serializer_class = serializers.AttachmentSerializer
-    filter_class = filters.AttachmentFilter
+    filterset_class = filters.AttachmentFilter
     search_fields = ('name',)
     mt_field = 'submission__project'
 
@@ -538,7 +538,7 @@ def content(self, request, pk=None, *args, **kwargs):
 class SchemaViewSet(FilteredMixin, viewsets.ModelViewSet):
     queryset = models.Schema.objects.all()
     serializer_class = serializers.SchemaSerializer
-    filter_class = filters.SchemaFilter
+    filterset_class = filters.SchemaFilter
     search_fields = ('name',)
 
     @action(detail=True, methods=['get'])
@@ -608,7 +608,7 @@ def unique_usage(self, request, *args, **kwargs):
 class SchemaDecoratorViewSet(MtViewSetMixin, FilteredMixin, viewsets.ModelViewSet):
     queryset = models.SchemaDecorator.objects.all()
     serializer_class = serializers.SchemaDecoratorSerializer
-    filter_class = filters.SchemaDecoratorFilter
+    filterset_class = filters.SchemaDecoratorFilter
     search_fields = ('name',)
     mt_field = 'project'
 
@@ -642,7 +642,7 @@ def skeleton(self, request, pk=None, *args, **kwargs):
 class EntityViewSet(MtViewSetMixin, FilteredMixin, ExporterMixin, viewsets.ModelViewSet):
     queryset = models.Entity.objects.all().prefetch_related('submission__attachments')
     serializer_class = serializers.EntitySerializer
-    filter_class = filters.EntityFilter
+    filterset_class = filters.EntityFilter
     search_fields = ('project__name', 'schema__name',)
     mt_field = 'project'
 
@@ -814,7 +814,7 @@ def _get_group_by_cols(*args):  # pragma: no cover
 class ProjectStatsViewSet(SubmissionStatsMixin, viewsets.ReadOnlyModelViewSet):
     queryset = models.Project.objects.all()
     serializer_class = serializers.ProjectStatsSerializer
-    filter_class = filters.ProjectFilter
+    filterset_class = filters.ProjectFilter
     search_fields = ('name',)
     entities_fk = 'project'
     fields_list = ('id', 'name', 'created', 'active',)
@@ -823,15 +823,15 @@ class ProjectStatsViewSet(SubmissionStatsMixin, viewsets.ReadOnlyModelViewSet):
 class MappingSetStatsViewSet(SubmissionStatsMixin, viewsets.ReadOnlyModelViewSet):
     queryset = models.MappingSet.objects.all()
     serializer_class = serializers.MappingSetStatsSerializer
-    filter_class = filters.MappingSetFilter
+    filterset_class = filters.MappingSetFilter
     search_fields = ('name',)
     mt_field = 'project'
 
 
 class ExportTaskViewSet(MtViewSetMixin, viewsets.ReadOnlyModelViewSet, mixins.DestroyModelMixin):
     queryset = models.ExportTask.objects.all()
     serializer_class = serializers.ExportTaskSerializer
-    filter_class = filters.ExportTaskFilter
+    filterset_class = filters.ExportTaskFilter
     mt_field = 'project'
 
     @action(

aether-kernel/conf/docker/apt-packages.txt

@@ -1,4 +1,5 @@
 gcc
 gettext-base
 gnupg
+lsb-release
 wget

aether-kernel/conf/docker/setup.sh

@@ -45,7 +45,7 @@ if [ -f "$PACKAGE_LIST" ]; then
 fi
 
 # add postgres apt repo to get more recent postgres versions
-echo 'deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main' > /etc/apt/sources.list.d/pgdg.list
+echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" > /etc/apt/sources.list.d/pgdg.list
 wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 apt-get update -qq > /dev/null
 apt-get -qq \