global: add support for custom fields and feature branches tests

Co-authored-by: Zacharias Zacharodimos <[email protected]>

.github/workflows/tests-feature.yaml

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Invenio.
+# Copyright (C) 2022 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+name: Feature development CI
+
+on:
+  pull_request:
+    branches:
+      - "feature/**"
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron: "0 3 * * 6"
+  workflow_dispatch:
+    inputs:
+      reason:
+        description: "Reason"
+        required: false
+        default: "Manual trigger"
+
+jobs:
+  Tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.8] # for efficiency test only one specific python version
+        requirements-level: [pypi]
+        cache-service: [redis]
+        db-service: [postgresql13]
+        search-service: [elasticsearch7]
+        include:
+          - db-service: postgresql13
+            DB_EXTRAS: "postgresql"
+
+          - search-service: elasticsearch7
+            SEARCH_EXTRAS: "elasticsearch7"
+
+    env:
+      CACHE: ${{ matrix.cache-service }}
+      DB: ${{ matrix.db-service }}
+      SEARCH: ${{ matrix.search-service }}
+      EXTRAS: tests,${{ matrix.SEARCH_EXTRAS }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Generate dependencies
+        run: |
+          pip install wheel requirements-builder
+          requirements-builder -e "$EXTRAS" --level=${{ matrix.requirements-level }} setup.py > .${{ matrix.requirements-level }}-${{ matrix.python-version }}-requirements.txt
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('.${{ matrix.requirements-level }}-${{ matrix.python-version }}-requirements.txt') }}
+
+      - name: Install dependencies
+        run: |
+
+          pip install -r .${{ matrix.requirements-level }}-${{ matrix.python-version }}-requirements.txt
+          pip install -r requirements-feature.txt # this file is used only when targeting a feature/* branch
+          pip install ".[$EXTRAS]"
+          pip freeze
+          docker --version
+          docker-compose --version
+
+      - name: Run tests
+        run: |
+          ./run-tests.sh

.github/workflows/tests.yml

@@ -13,7 +13,9 @@ on:
   push:
     branches: master
   pull_request:
-    branches: master
+    branches:
+      - master
+      - "maint-**"
   schedule:
     # * is a special character in YAML so you have to quote this string
     - cron: "0 3 * * 6"

.gitignore

@@ -9,7 +9,7 @@ __pycache__/
 .vscode/
 
 # DS_Store
-.DS_Store/
+*.DS_Store
 
 # C extensions
 *.so

invenio_vocabularies/contrib/affiliations/schema.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2021 CERN.
+# Copyright (C) 2020-2022 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -11,11 +11,11 @@
 from functools import partial
 
 from flask_babelex import lazy_gettext as _
-from marshmallow import Schema, ValidationError, fields, validates_schema
+from marshmallow import fields
 from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
 from marshmallow_utils.schemas import IdentifierSchema
 
-from ...services.schema import BaseVocabularySchema
+from ...services.schema import BaseVocabularySchema, ContribVocabularyRelationSchema
 from .config import affiliation_schemes
 
 
@@ -35,23 +35,9 @@ class AffiliationSchema(BaseVocabularySchema):
     name = SanitizedUnicode(required=True)
 
 
-class AffiliationRelationSchema(Schema):
+class AffiliationRelationSchema(ContribVocabularyRelationSchema):
     """Schema to define an optional affialiation relation in another schema."""
 
-    id = SanitizedUnicode()
+    ftf_name = "name"
+    parent_field_name = "affiliations"
     name = SanitizedUnicode()
-
-    @validates_schema
-    def validate_affiliation(self, data, **kwargs):
-        """Validates that either id either name are present."""
-        id_ = data.get("id")
-        name = data.get("name")
-        if id_:
-            data = {"id": id_}
-        elif name:
-            data = {"name": name}
-
-        if not id_ and not name:
-            raise ValidationError(
-                _("An existing id or a free text name must be present."), "affiliations"
-            )

invenio_vocabularies/contrib/funders/schema.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2022 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -12,7 +12,6 @@
 
 from flask_babelex import lazy_gettext as _
 from marshmallow import (
-    Schema,
     ValidationError,
     fields,
     post_load,
@@ -23,32 +22,18 @@
 from marshmallow_utils.fields import IdentifierSet, SanitizedUnicode
 from marshmallow_utils.schemas import IdentifierSchema
 
-from ...services.schema import BaseVocabularySchema
+from ...services.schema import BaseVocabularySchema, ContribVocabularyRelationSchema
 from .config import funder_schemes
 
 
-class FunderRelationSchema(Schema):
+class FunderRelationSchema(ContribVocabularyRelationSchema):
     """Funder schema."""
 
+    ftf_name = "name"
+    parent_field_name = "funder"
     name = SanitizedUnicode(
         validate=validate.Length(min=1, error=_("Name cannot be blank."))
     )
-    id = SanitizedUnicode()
-
-    @validates_schema
-    def validate_funder(self, data, **kwargs):
-        """Validates that either id either name are present."""
-        id_ = data.get("id")
-        name = data.get("name")
-        if id_:
-            data = {"id": id_}
-        elif name:
-            data = {"name": name}
-
-        if not id_ and not name:
-            raise ValidationError(
-                _("An existing id or a free text name must be present."), "funder"
-            )
 
 
 class FunderSchema(BaseVocabularySchema):

invenio_vocabularies/contrib/subjects/schema.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2021 Northwestern University.
-# Copyright (C) 2021 CERN.
+# Copyright (C) 2021-2022 CERN.
 #
 # Invenio-Vocabularies is free software; you can redistribute it and/or
 # modify it under the terms of the MIT License; see LICENSE file for more
@@ -10,10 +10,9 @@
 """Subjects schema."""
 
 from flask_babelex import lazy_gettext as _
-from marshmallow import Schema, ValidationError, validates_schema
 from marshmallow_utils.fields import SanitizedUnicode
 
-from ...services.schema import BaseVocabularySchema
+from ...services.schema import BaseVocabularySchema, ContribVocabularyRelationSchema
 
 
 class SubjectSchema(BaseVocabularySchema):
@@ -27,23 +26,9 @@ class SubjectSchema(BaseVocabularySchema):
     subject = SanitizedUnicode(required=True)
 
 
-class SubjectRelationSchema(Schema):
+class SubjectRelationSchema(ContribVocabularyRelationSchema):
     """Schema to define an optional subject relation in another schema."""
 
-    id = SanitizedUnicode()
+    ftf_name = "subject"
+    parent_field_name = "subjects"
     subject = SanitizedUnicode()
-
-    @validates_schema
-    def validate_subject(self, data, **kwargs):
-        """Validates that either id either name are present."""
-        id_ = data.get("id")
-        subject = data.get("subject")
-        if id_:
-            data = {"id": id_}
-        elif subject:
-            data = {"subject": subject}
-
-        if not id_ and not subject:
-            raise ValidationError(
-                _("An existing id or a free text subject must be present."), "subjects"
-            )

invenio_vocabularies/records/systemfields/relations.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-Records-Resources is free software; you can redistribute it and/or
+# modify it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Relations system fields."""
+
+from flask import current_app
+from invenio_records.systemfields import RelationsField
+from werkzeug.local import LocalProxy
+
+from invenio_vocabularies.services.custom_fields import VocabularyCF
+
+from ..api import Vocabulary
+
+
+class CustomFieldsRelation(RelationsField):
+    """Relation field to manage custom fields.
+
+    Iterates through all configured custom fields and collects the ones
+    defining a relation dependency e.g vocabularies.
+    """
+
+    def __init__(self, fields_var):
+        """Initialize the field."""
+        super().__init__()
+        self._fields_var = fields_var
+        self._fields = LocalProxy(lambda: self._load_custom_fields_relations())
+
+    def _load_custom_fields_relations(self):
+        """Loads custom fields relations from config."""
+        custom_fields = current_app.config.get(self._fields_var, {})
+
+        relations = {}
+        for cf in custom_fields:
+            if getattr(cf, "relation_cls", None):
+                relations[cf.name] = cf.relation_cls(
+                    f"custom_fields.{cf.name}",
+                    keys=cf.field_keys,
+                    pid_field=Vocabulary.pid.with_type_ctx(cf.vocabulary_id),
+                    cache_key=cf.vocabulary_id,
+                )
+
+        return relations
+
+    def __set__(self, instance, values):
+        """Setting the attribute."""
+        raise ValueError(
+            f"This field can only be set through config ({self._fields_var})"
+        )

invenio_vocabularies/services/custom_fields/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Custom Fields for InvenioRDM."""
+
+from .vocabulary import VocabularyCF
+
+__all__ = "VocabularyCF"

invenio_vocabularies/services/custom_fields/vocabulary.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2022 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Custom Fields for InvenioRDM."""
+
+from invenio_records_resources.records.systemfields import PIDListRelation, PIDRelation
+from invenio_records_resources.services.custom_fields.base import BaseCF
+from marshmallow import fields
+
+from ...proxies import current_service
+from ...resources.serializer import VocabularyL10NItemSchema
+from ...services.schema import VocabularyRelationSchema
+
+
+class VocabularyCF(BaseCF):
+    """Vocabulary custom field.
+
+    Supporting common vocabulary structure.
+    """
+
+    field_keys = ["id", "props", "title", "icon"]
+    """Return field's keys for querying.
+
+    These keys are used to select which information to return from the
+    vocabulary that is queried.
+    """
+
+    def __init__(self, name, vocabulary_id, multiple=False, dump_options=True):
+        """Constructor."""
+        super().__init__(name)
+        self.relation_cls = PIDRelation if not multiple else PIDListRelation
+        self.vocabulary_id = vocabulary_id
+        self.dump_options = dump_options
+        self.multiple = multiple
+
+    @property
+    def mapping(self):
+        """Return the mapping."""
+        _mapping = {
+            "type": "object",
+            "properties": {
+                "@v": {"type": "keyword"},
+                "id": {"type": "keyword"},
+                "title": {"type": "object", "dynamic": True},
+            },
+        }
+
+        return _mapping
+
+    @property
+    def field(self):
+        """Marshmallow schema for vocabulary custom fields."""
+        return fields.Nested(VocabularyRelationSchema, many=self.multiple)
+
+    @property
+    def ui_field(self):
+        """Marshmallow UI schema for vocabulary custom fields.
+
+        This schema is used in the UIJSONSerializer and controls how the field will be
+        dumped in the UI. It takes responsibility of the localization of strings.
+        """
+        return fields.Nested(VocabularyL10NItemSchema, many=self.multiple)
+
+    def options(self, identity):
+        """Return UI serialized vocabulary items."""
+        if self.dump_options:
+            vocabs = current_service.read_all(
+                identity,
+                fields=self.field_keys,
+                type=self.vocabulary_id,
+            )
+            options = []
+            for vocab in vocabs:
+                options.append(VocabularyL10NItemSchema().dump(vocab))
+
+            return options