diff --git a/data/contributions_big.json b/data/entities_big.json similarity index 100% rename from data/contributions_big.json rename to data/entities_big.json diff --git a/data/contributions_small.json b/data/entities_small.json similarity index 100% rename from data/contributions_small.json rename to data/entities_small.json diff --git a/pyproject.toml b/pyproject.toml index 64df20077d..f16187c8bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,7 +161,7 @@ acq_orders = "rero_ils.modules.acquisition.acq_orders.views:api_blueprint" acq_receipts = "rero_ils.modules.acquisition.acq_receipts.views:api_blueprint" api_documents = "rero_ils.modules.documents.views:api_blueprint" circ_policies = "rero_ils.modules.circ_policies.views:blueprint" -contributions = "rero_ils.modules.contributions.views:api_blueprint" +entities = "rero_ils.modules.entities.views:api_blueprint" documents = "rero_ils.modules.documents.views:api_blueprint" holdings = "rero_ils.modules.holdings.api_views:api_blueprint" imports = "rero_ils.modules.imports.views:api_blueprint" @@ -183,7 +183,7 @@ rero-ils = "rero_ils.modules.ext:REROILSAPP" [tool.poetry.plugins."invenio_base.blueprints"] circ_policies = "rero_ils.modules.circ_policies.views:blueprint" collections = "rero_ils.modules.collections.views:blueprint" -contributions = "rero_ils.modules.contributions.views:blueprint" +entities = "rero_ils.modules.entities.views:blueprint" documents = "rero_ils.modules.documents.views:blueprint" holdings = "rero_ils.modules.holdings.views:blueprint" ill_requests = "rero_ils.modules.ill_requests.views:blueprint" @@ -201,7 +201,7 @@ users = "rero_ils.modules.users.views:blueprint" apiharvester = "rero_ils.modules.apiharvester.tasks" collections = "rero_ils.modules.collections.tasks" documents = "rero_ils.modules.documents.tasks" -contributions = "rero_ils.modules.contributions.tasks" +entities = "rero_ils.modules.entities.tasks" ebooks = "rero_ils.modules.ebooks.tasks" holdings = "rero_ils.modules.holdings.tasks" items = "rero_ils.modules.items.tasks" @@ -236,7 +236,7 @@ items = "rero_ils.modules.items.models" libraries = "rero_ils.modules.libraries.models" local_fields = "rero_ils.modules.local_fields.models" locations = "rero_ils.modules.locations.models" -mef = "rero_ils.modules.contributions.models" +entities = "rero_ils.modules.entities.models" notifications = "rero_ils.modules.notifications.models" organisations = "rero_ils.modules.organisations.models" patron_transaction_events = "rero_ils.modules.patron_transaction_events.models" @@ -263,7 +263,7 @@ budgets = "rero_ils.modules.acquisition.budgets.jsonschemas" circ_policies = "rero_ils.modules.circ_policies.jsonschemas" collections = "rero_ils.modules.collections.jsonschemas" common = "rero_ils.jsonschemas" -contributions = "rero_ils.modules.contributions.jsonschemas" +entities = "rero_ils.modules.entities.jsonschemas" documents = "rero_ils.modules.documents.jsonschemas" holdings = "rero_ils.modules.holdings.jsonschemas" ill_requests = "rero_ils.modules.ill_requests.jsonschemas" @@ -303,7 +303,7 @@ acq_receipt_line_id = "rero_ils.modules.acquisition.acq_receipt_lines.api:acq_re budget_id = "rero_ils.modules.acquisition.budgets.api:budget_id_fetcher" circ_policy_id = "rero_ils.modules.circ_policies.api:circ_policy_id_fetcher" collection_id = "rero_ils.modules.collections.api:collection_id_fetcher" -contribution_id = "rero_ils.modules.contributions.api:contribution_id_fetcher" +entity_id = "rero_ils.modules.entities.api:entity_id_fetcher" document_id = "rero_ils.modules.documents.api:document_id_fetcher" holding_id = "rero_ils.modules.holdings.api:holding_id_fetcher" ill_request_id = "rero_ils.modules.ill_requests.api:ill_request_id_fetcher" @@ -333,7 +333,7 @@ acq_receipt_line_id = "rero_ils.modules.acquisition.acq_receipt_lines.api:acq_re budget_id = "rero_ils.modules.acquisition.budgets.api:budget_id_minter" circ_policy_id = "rero_ils.modules.circ_policies.api:circ_policy_id_minter" collection_id = "rero_ils.modules.collections.api:collection_id_minter" -contribution_id = "rero_ils.modules.contributions.api:contribution_id_minter" +entity_id = "rero_ils.modules.entities.api:entity_id_minter" document_id = "rero_ils.modules.documents.api:document_id_minter" holding_id = "rero_ils.modules.holdings.api:holding_id_minter" ill_request_id = "rero_ils.modules.ill_requests.api:ill_request_id_minter" @@ -361,7 +361,6 @@ acq_receipt_lines = "rero_ils.modules.acquisition.acq_receipt_lines.jsonresolver acq_receipts = "rero_ils.modules.acquisition.acq_receipts.jsonresolver" budgets = "rero_ils.modules.acquisition.budgets.jsonresolver" collections = "rero_ils.modules.collections.jsonresolver" -contributions = "rero_ils.modules.contributions.jsonresolver" documents = "rero_ils.modules.documents.jsonresolver" holdings = "rero_ils.modules.holdings.jsonresolver" ill_requests = "rero_ils.modules.ill_requests.jsonresolver" @@ -390,7 +389,7 @@ acq_receipts = "rero_ils.modules.acquisition.acq_receipts.mappings" budgets = "rero_ils.modules.acquisition.budgets.mappings" circ_policies = "rero_ils.modules.circ_policies.mappings" collections = "rero_ils.modules.collections.mappings" -contributions = "rero_ils.modules.contributions.mappings" +entities = "rero_ils.modules.entities.mappings" documents = "rero_ils.modules.documents.mappings" holdings = "rero_ils.modules.holdings.mappings" ill_requests = "rero_ils.modules.ill_requests.mappings" diff --git a/rero_ils/modules/contributions/jsonresolver.py b/rero_ils/alembic/a710021979fe_migrate_contribution_to_entity.py similarity index 52% rename from rero_ils/modules/contributions/jsonresolver.py rename to rero_ils/alembic/a710021979fe_migrate_contribution_to_entity.py index 27d0b32750..7cb75ffed1 100644 --- a/rero_ils/modules/contributions/jsonresolver.py +++ b/rero_ils/alembic/a710021979fe_migrate_contribution_to_entity.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,17 +16,24 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Contributions resolver.""" +"""Migrate Contribution to Entity.""" +from alembic import op -import jsonresolver -from requests import get as requests_get # noqa +# revision identifiers, used by Alembic. +revision = 'a710021979fe' +down_revision = '5f0b086e4b82' +branch_labels = () +depends_on = None -from .api import Contribution +def upgrade(): + """Upgrade database.""" + op.rename_table('contribution_id', 'entity_id') + op.rename_table('contribution_metadata', 'entity_metadata') -@jsonresolver.route('/api//', host='mef.rero.ch') -def contribution_resolver(agency, pid): - """MEF contribution resolver.""" - contribution = Contribution.get_contribution(agency, pid) - return contribution + +def downgrade(): + """Downgrade database.""" + op.rename_table('entity_id', 'contribution_id') + op.rename_table('entity_metadata', 'contribution_metadata') diff --git a/rero_ils/celery.py b/rero_ils/celery.py index 708da0b7d6..84e59171e0 100644 --- a/rero_ils/celery.py +++ b/rero_ils/celery.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by diff --git a/rero_ils/config.py b/rero_ils/config.py index 27613be77c..3fed807be7 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -68,12 +68,12 @@ CirculationPolicyPermissionPolicy from .modules.collections.api import Collection from .modules.collections.permissions import CollectionPermissionPolicy -from .modules.contributions.api import Contribution -from .modules.contributions.permissions import ContributionPermissionPolicy from .modules.documents.api import Document from .modules.documents.permissions import DocumentPermissionPolicy from .modules.documents.query import acquisition_filter, \ nested_identified_filter +from .modules.entities.api import Entity +from .modules.entities.permissions import EntityPermissionPolicy from .modules.holdings.api import Holding from .modules.holdings.models import HoldingCirculationAction from .modules.holdings.permissions import HoldingsPermissionPolicy @@ -443,7 +443,7 @@ def _(x): 'enabled': False, }, 'sync-agents': { - 'task': 'rero_ils.modules.contributions.tasks.sync_agents', + 'task': 'rero_ils.modules.entities.tasks.sync_agents', 'schedule': crontab(minute=0, hour=1), # Every day at 01:00 UTC, 'enabled': False, }, @@ -1121,13 +1121,13 @@ def _(x): update_permission_factory_imp=lambda record: LocationPermissionPolicy('update', record=record), delete_permission_factory_imp=lambda record: LocationPermissionPolicy('delete', record=record) ), - cont=dict( - pid_type='cont', - pid_minter='contribution_id', - pid_fetcher='contribution_id', - search_class='rero_ils.modules.contributions.api:ContributionsSearch', - search_index='contributions', - indexer_class='rero_ils.modules.contributions.api:ContributionsIndexer', + ent=dict( + pid_type='ent', + pid_minter='entity_id', + pid_fetcher='entity_id', + search_class='rero_ils.modules.entities.api:EntitiesSearch', + search_index='entities', + indexer_class='rero_ils.modules.entities.api:EntitiesIndexer', search_type=None, record_serializers={ 'application/json': 'rero_ils.modules.serializers:json_v1_response' @@ -1138,22 +1138,23 @@ def _(x): search_serializers={ 'application/json': 'rero_ils.modules.serializers:json_v1_search' }, - list_route='/contributions/', + search_serializers_aliases={ + 'json': 'application/json' + }, + list_route='/entities/', record_loaders={ - 'application/json': lambda: Contribution(request.get_json()), + 'application/json': lambda: Entity(request.get_json()), }, - record_class='rero_ils.modules.contributions.api:Contribution', - item_route=('/contributions/'), + record_class='rero_ils.modules.entities.api:Entity', + item_route='/entities/', default_media_type='application/json', max_result_window=MAX_RESULT_WINDOW, - search_factory_imp='rero_ils.query:contribution_view_search_factory', - list_permission_factory_imp=lambda record: ContributionPermissionPolicy('search', record=record), - read_permission_factory_imp=lambda record: ContributionPermissionPolicy('read', record=record), - create_permission_factory_imp=lambda record: ContributionPermissionPolicy('create', record=record), - update_permission_factory_imp=lambda record: ContributionPermissionPolicy('update', record=record), - delete_permission_factory_imp=lambda record: ContributionPermissionPolicy('delete', record=record) + search_factory_imp='rero_ils.query:entity_view_search_factory', + list_permission_factory_imp=lambda record: EntityPermissionPolicy('search', record=record), + read_permission_factory_imp=lambda record: EntityPermissionPolicy('read', record=record), + create_permission_factory_imp=lambda record: EntityPermissionPolicy('create', record=record), + update_permission_factory_imp=lambda record: EntityPermissionPolicy('update', record=record), + delete_permission_factory_imp=lambda record: EntityPermissionPolicy('delete', record=record) ), cipo=dict( pid_type='cipo', @@ -2044,7 +2045,7 @@ def _(x): ) }, ), - contributions=dict( + entities=dict( aggs=dict( sources=dict( terms=dict( @@ -2272,7 +2273,7 @@ def _(x): 'budgets', 'circ_policies', 'collections', - 'contributions', + 'entities', 'documents', 'holdings', 'items', @@ -2395,7 +2396,7 @@ def _(x): query='bestmatch', noquery='start_date') # ------ CONTRIBUTIONS SORT -RECORDS_REST_SORT_OPTIONS['contributions']['fr_name'] = dict( +RECORDS_REST_SORT_OPTIONS['entities']['fr_name'] = dict( fields=[ 'idref_authorized_access_point_sort', 'rero_authorized_access_point_sort', @@ -2404,7 +2405,7 @@ def _(x): title='Collection french name', default_order='asc' ) -RECORDS_REST_SORT_OPTIONS['contributions']['de_name'] = dict( +RECORDS_REST_SORT_OPTIONS['entities']['de_name'] = dict( fields=[ 'gnd_authorized_access_point_sort', 'idref_authorized_access_point_sort', @@ -2825,7 +2826,7 @@ def _(x): 'budg': '/budgets/budget-v0.0.1.json', 'cipo': '/circ_policies/circ_policy-v0.0.1.json', 'coll': '/collections/collection-v0.0.1.json', - 'cont': '/contributions/contribution-v0.0.1.json', + 'ent': '/entities/entity-v0.0.1.json', 'doc': '/documents/document-v0.0.1.json', 'hold': '/holdings/holding-v0.0.1.json', 'illr': '/ill_requests/ill_request-v0.0.1.json', @@ -3000,14 +3001,13 @@ def _(x): #: Cover service RERO_ILS_THUMBNAIL_SERVICE_URL = 'https://services.test.rero.ch/cover' -#: Contributions -RERO_ILS_CONTRIBUTIONS_MEF_SCHEMA = 'contributions/contribution-v0.0.1.json' -RERO_ILS_CONTRIBUTIONS_SOURCES = ['idref', 'gnd', 'rero'] -RERO_ILS_CONTRIBUTIONS_AGENT_TYPES = { +#: Entities +RERO_ILS_AGENTS_SOURCES = ['idref', 'gnd', 'rero'] +RERO_ILS_AGENTS_AGENT_TYPES = { 'bf:Person': 'persons', 'bf:Organisation': 'corporate-bodies' } -RERO_ILS_CONTRIBUTIONS_LABEL_ORDER = { +RERO_ILS_AGENTS_LABEL_ORDER = { 'fallback': 'fr', 'fr': ['idref', 'rero', 'gnd'], 'de': ['gnd', 'idref', 'rero'], @@ -3056,15 +3056,11 @@ def _(x): RERO_ILS_DEFAULT_PICKUP_HOLD_DURATION = 10 # ============================================================================= -# ANONYMISATION PROCESS CONFIGURATION +# ANONYMIZATION PROCESS CONFIGURATION # ============================================================================= - -# Specify the delay (in days) under which no loan can't be anonymized anyway ( -# for circulation management process). +# Specify the delay (in days) under which no loan can't be anonymized anyway (for circulation management process). RERO_ILS_ANONYMISATION_MIN_TIME_LIMIT = 3 * 365 / 12 - -# Specify the delay (in days) when a loan should be anonymized anyway after it -# concluded. +# Specify the delay (in days) when a loan should be anonymized anyway after it concluded. RERO_ILS_ANONYMISATION_MAX_TIME_LIMIT = 6 * 365 / 12 #: Invenio circulation configuration. diff --git a/rero_ils/es_templates/v7/record.json b/rero_ils/es_templates/v7/record.json index 8ddc91d338..614fd3f03c 100644 --- a/rero_ils/es_templates/v7/record.json +++ b/rero_ils/es_templates/v7/record.json @@ -9,8 +9,8 @@ "budgets-*", "circ_policies-*", "collections-*", - "contributions-*", "documents-*", + "entities-*", "holdings-*", "ill_requests-*", "item_types-*", diff --git a/rero_ils/modules/api.py b/rero_ils/modules/api.py index e6d48fca08..27f9fd889c 100644 --- a/rero_ils/modules/api.py +++ b/rero_ils/modules/api.py @@ -666,7 +666,7 @@ def _index_action(self, payload): arguments = {} index = payload.get('index') or index body = self._prepare_record(record, index, doc_type, arguments) - action = { + return { '_op_type': 'index', '_index': index, '_type': doc_type, @@ -674,10 +674,7 @@ def _index_action(self, payload): '_version': record.revision_id, '_version_type': self._version_type, '_source': body - } - action.update(arguments) - - return action + } | arguments def _prepare_record( self, record, index, doc_type, arguments=None, **kwargs): diff --git a/rero_ils/modules/cli/reroils.py b/rero_ils/modules/cli/reroils.py index 36418a1089..ccf03bb583 100644 --- a/rero_ils/modules/cli/reroils.py +++ b/rero_ils/modules/cli/reroils.py @@ -24,8 +24,8 @@ from rero_ils.modules.acquisition.cli import acquisition from rero_ils.modules.apiharvester.cli import apiharvester -from rero_ils.modules.contributions.cli import contribution from rero_ils.modules.ebooks.cli import oaiharvester +from rero_ils.modules.entities.cli import entity from rero_ils.modules.monitoring.cli import monitoring from rero_ils.modules.notifications.cli import notifications from rero_ils.modules.stats.cli import stats @@ -44,7 +44,7 @@ def reroils(): reroils.add_command(acquisition) reroils.add_command(apiharvester) -reroils.add_command(contribution) +reroils.add_command(entity) reroils.add_command(fixtures) reroils.add_command(index) reroils.add_command(monitoring) diff --git a/rero_ils/modules/cli/utils.py b/rero_ils/modules/cli/utils.py index 0c8460d399..481c763951 100644 --- a/rero_ils/modules/cli/utils.py +++ b/rero_ils/modules/cli/utils.py @@ -56,19 +56,20 @@ from werkzeug.local import LocalProxy from werkzeug.security import gen_salt +from rero_ils.modules.documents.api import Document, DocumentsSearch +from rero_ils.modules.documents.dojson.contrib.marc21tojson.rero import marc21 +from rero_ils.modules.documents.views import get_cover_art +from rero_ils.modules.entities.api import Entity +from rero_ils.modules.items.api import Item +from rero_ils.modules.libraries.api import Library +from rero_ils.modules.loans.tasks import \ + delete_loans_created as task_delete_loans_created +from rero_ils.modules.local_fields.api import LocalField from rero_ils.modules.locations.api import Location - -from ..contributions.api import Contribution -from ..documents.api import Document, DocumentsSearch -from ..documents.dojson.contrib.marc21tojson.rero import marc21 -from ..documents.views import get_cover_art -from ..items.api import Item -from ..libraries.api import Library -from ..loans.tasks import delete_loans_created as task_delete_loans_created -from ..local_fields.api import LocalField -from ..patrons.cli import users_validate -from ..selfcheck.cli import create_terminal, list_terminal, update_terminal -from ..utils import JsonWriter, extracted_data_from_ref, \ +from rero_ils.modules.patrons.cli import users_validate +from rero_ils.modules.selfcheck.cli import create_terminal, list_terminal, \ + update_terminal +from rero_ils.modules.utils import JsonWriter, extracted_data_from_ref, \ get_record_class_from_schema_or_pid_type, get_schema_for_resource, \ read_json_record, read_xml_record @@ -79,12 +80,10 @@ def queue_count(): """Count tasks in celery.""" inspector = current_celery.control.inspect() task_count = 0 - reserved = inspector.reserved() - if reserved: + if reserved := inspector.reserved(): for _, values in reserved.items(): task_count += len(values) - active = inspector.active() - if active: + if active := inspector.active(): for _, values in active.items(): task_count += len(values) return task_count @@ -1572,9 +1571,7 @@ def export(verbose, pid_type, outfile_name, pidfile, indent, schema): else: pids = record_class.get_all_pids() - contributions_sources = current_app.config.get( - 'RERO_ILS_CONTRIBUTIONS_SOURCES', []) - + agents_sources = current_app.config.get('RERO_ILS_AGENTS_SOURCES', []) for count, pid in enumerate(pids, 1): try: rec = record_class.get_record_by_pid(pid) @@ -1583,9 +1580,9 @@ def export(verbose, pid_type, outfile_name, pidfile, indent, schema): f'{count: <8} {pid_type} export {rec.pid}:{rec.id}') if not schema: rec.pop('$schema', None) - if isinstance(rec, Contribution): - for contribution_source in contributions_sources: - rec.get(contribution_source, {}).pop('$schema', None) + if isinstance(rec, Entity): + for agent_source in agents_sources: + rec.get(agent_source, {}).pop('$schema', None) outfile.write(rec) except Exception as err: click.echo(err) diff --git a/rero_ils/modules/commons/dumpers.py b/rero_ils/modules/commons/dumpers.py index e4c63c3e8b..b0441c87a5 100644 --- a/rero_ils/modules/commons/dumpers.py +++ b/rero_ils/modules/commons/dumpers.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO -# Copyright (C) 2019-2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -25,13 +25,13 @@ class MultiDumper(InvenioRecordsDumper): """Aggregate several dumpers.""" - def __init__(self, dumpers=[]): + def __init__(self, dumpers=None): """Constructor. :param dumpers: list - list of dumpers to aggregate. """ super().__init__() - self._dumpers = dumpers + self._dumpers = dumpers or [] def dump(self, record, data): """Dump a record that can be used a source document. diff --git a/rero_ils/modules/commons/exceptions.py b/rero_ils/modules/commons/exceptions.py new file mode 100644 index 0000000000..2c4346ee8a --- /dev/null +++ b/rero_ils/modules/commons/exceptions.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Common exceptions for RERO-ILS resources.""" + + +class RecordNotFound(Exception): + """Record con't be found into Invenio.""" + + def __init__(self, record_cls, record_pid): + """Initialization method. + + :param record_cls: (IlsRecord) the resource class. + :param record_pid: (string) the resource pid. + """ + self.record_cls = record_cls + self.record_pid = record_pid + + def __str__(self): + """String representation of the exception.""" + return f'{self.record_cls.__name__}#{self.record_pid} not found' diff --git a/rero_ils/modules/documents/api.py b/rero_ils/modules/documents/api.py index c9a5d181d3..d2032cd118 100644 --- a/rero_ils/modules/documents/api.py +++ b/rero_ils/modules/documents/api.py @@ -41,7 +41,7 @@ from rero_ils.modules.providers import Provider from rero_ils.modules.utils import sorted_pids -from .dumpers import document_indexer, document_replace_refs +from .dumpers import document_indexer_dumper, document_replace_refs_dumper from .extensions import AddMEFPidExtension, EditionStatementExtension, \ ProvisionActivitiesExtension, SeriesStatementExtension, TitleExtension from .models import DocumentIdentifier, DocumentMetadata @@ -128,7 +128,6 @@ def _validate(self, **kwargs): def is_available(cls, pid, view_code, raise_exception=False): """Get availability for document.""" from ..holdings.api import Holding - holding_pids = [] if view_code != current_app.config.get( 'RERO_ILS_SEARCH_GLOBAL_VIEW_CODE'): view_id = Organisation.get_record_by_viewcode(view_code)['pid'] @@ -235,20 +234,22 @@ def reasons_not_to_delete(self): def index_contributions(self, bulk=False): """Index all attached contributions.""" - from ..contributions.api import Contribution, ContributionsIndexer + from ..entities.api import EntitiesIndexer, Entity from ..tasks import process_bulk_queue contributions_ids = [] for contribution in self.get('contribution', []): ref = contribution['entity'].get('$ref') if not ref and (cont_pid := contribution['entity'].get('pid')): + print("OK...") if bulk: - uid = Contribution.get_id_by_pid(cont_pid) + uid = Entity.get_id_by_pid(cont_pid) contributions_ids.append(uid) else: - contrib = Contribution.get_record_by_pid(cont_pid) + contrib = Entity.get_record_by_pid(cont_pid) contrib.reindex() if contributions_ids: - ContributionsIndexer().bulk_index(contributions_ids) + print("ids are", contributions_ids) + EntitiesIndexer().bulk_index(contributions_ids) process_bulk_queue.apply_async() @classmethod @@ -351,7 +352,7 @@ def resolve(self): :returns: a fresh copy of the resolved data. """ - return self.dumps(document_replace_refs) + return self.dumps(document_replace_refs_dumper) class DocumentsIndexer(IlsRecordsIndexer): @@ -359,7 +360,7 @@ class DocumentsIndexer(IlsRecordsIndexer): record_cls = Document # data dumper for indexing - record_dumper = document_indexer + record_dumper = document_indexer_dumper @classmethod def _es_document(cls, record): diff --git a/rero_ils/modules/documents/commons/subjects.py b/rero_ils/modules/documents/commons/subjects.py index d47b5c7d8b..d60f0851d8 100644 --- a/rero_ils/modules/documents/commons/subjects.py +++ b/rero_ils/modules/documents/commons/subjects.py @@ -40,10 +40,9 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from rero_ils.modules.contributions.api import Contribution -from rero_ils.modules.contributions.utils import \ - get_contribution_localized_value from rero_ils.modules.documents.models import DocumentSubjectType +from rero_ils.modules.entities.api import Entity +from rero_ils.modules.entities.utils import get_entity_localized_value # ============================================================================= @@ -87,7 +86,7 @@ def render(self, language=None, **kwargs) -> str: :param language: preferred language for the subject. :return the string representation of this subject. """ - sub, _ = Contribution.get_record_by_ref(self.reference) + sub, _ = Entity.get_record_by_ref(self.reference) return sub.get_authorized_access_point(language=language) @@ -107,8 +106,8 @@ def render(self, language=None, **kwargs) -> str: :param language: preferred language for the subject. :return the string representation of this subject. """ - return get_contribution_localized_value( - contribution=self.data, + return get_entity_localized_value( + entity=self.data, key='authorized_access_point', language=language) diff --git a/rero_ils/modules/documents/dojson/contrib/jsontodc/model.py b/rero_ils/modules/documents/dojson/contrib/jsontodc/model.py index 655bd99297..485a4e7e71 100644 --- a/rero_ils/modules/documents/dojson/contrib/jsontodc/model.py +++ b/rero_ils/modules/documents/dojson/contrib/jsontodc/model.py @@ -20,9 +20,8 @@ from dojson import Overdo, utils from flask_babelex import gettext as _ -from rero_ils.modules.contributions.utils import \ - get_contribution_localized_value from rero_ils.modules.documents.extensions import TitleExtension +from rero_ils.modules.entities.utils import get_entity_localized_value class DublinCoreOverdo(Overdo): @@ -84,8 +83,8 @@ def do(self, blob, ignore_missing=True, exception_handlers=None, @utils.ignore_value def json_to_contributors(self, key, value): """Get creators and contributors data.""" - authorized_access_point = get_contribution_localized_value( - contribution=value.get('entity', {}), + authorized_access_point = get_entity_localized_value( + entity=value.get('entity', {}), key='authorized_access_point', language=dublincore.language ) @@ -214,8 +213,8 @@ def json_to_subject(self, key, value): subject_type = value.get('type') if subject_type in ['bf:Person', 'bf:Organisation', 'bf:Place']: # TODO: set the language - authorized_access_point = get_contribution_localized_value( - contribution=value, + authorized_access_point = get_entity_localized_value( + entity=value, key='authorized_access_point', language=dublincore.language ) diff --git a/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py b/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py index 0e7545fe5a..d3c4bfb3ac 100644 --- a/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py +++ b/rero_ils/modules/documents/dojson/contrib/jsontomarc21/model.py @@ -24,9 +24,9 @@ from flask_babelex import gettext as translate from invenio_db import db -from rero_ils.modules.contributions.api import Contribution from rero_ils.modules.documents.utils import display_alternate_graphic_first from rero_ils.modules.documents.views import create_title_responsibilites +from rero_ils.modules.entities.api import Entity from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch from rero_ils.modules.libraries.api import Library @@ -248,8 +248,7 @@ def do(self, blob, language='en', ignore_missing=True, )) } # Fix ContributionsSearch - order = current_app.config.get( - 'RERO_ILS_CONTRIBUTIONS_LABEL_ORDER', []) + order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', []) source_order = order.get( self.language, order.get(order['fallback'], []) @@ -257,7 +256,7 @@ def do(self, blob, language='en', ignore_missing=True, contributions = blob.get('contribution', []) for contribution in contributions: if ref := contribution['entity'].get('$ref'): - agent, _ = Contribution.get_record_by_ref(ref) + agent, _ = Entity.get_record_by_ref(ref) if agent: db.session.commit() contribution['entity'] = agent diff --git a/rero_ils/modules/documents/dumpers/__init__.py b/rero_ils/modules/documents/dumpers/__init__.py index eb11184f33..545cacfcbf 100644 --- a/rero_ils/modules/documents/dumpers/__init__.py +++ b/rero_ils/modules/documents/dumpers/__init__.py @@ -23,38 +23,38 @@ from rero_ils.modules.commons.dumpers import MultiDumper from .indexer import IndexerDumper -from .replace_refs import ReplaceRefsContributionsDumper, ReplaceRefsDumper, \ +from .replace_refs import ReplaceRefsDumper, ReplaceRefsEntitiesDumper, \ ReplaceRefsSubjectsDumper from .title import TitleDumper __all__ = ( 'TitleDumper', - 'ReplaceRefsContributionsDumper', + 'ReplaceRefsEntitiesDumper', 'ReplaceRefsSubjectsDumper', 'ReplaceRefsDumper' ) # replace linked data -document_replace_refs = MultiDumper(dumpers=[ +document_replace_refs_dumper = MultiDumper(dumpers=[ # make a fresh copy Dumper(), - ReplaceRefsContributionsDumper(), + ReplaceRefsEntitiesDumper(), ReplaceRefsSubjectsDumper(), ReplaceRefsDumper() ]) # create a string version of the complex title field -document_title = MultiDumper(dumpers=[ +document_title_dumper = MultiDumper(dumpers=[ # make a fresh copy Dumper(), TitleDumper() ]) # dumper used for indexing -document_indexer = MultiDumper(dumpers=[ +document_indexer_dumper = MultiDumper(dumpers=[ # make a fresh copy Dumper(), - ReplaceRefsContributionsDumper(), + ReplaceRefsEntitiesDumper(), ReplaceRefsSubjectsDumper(), ReplaceRefsDumper(), IndexerDumper() diff --git a/rero_ils/modules/documents/dumpers/replace_refs.py b/rero_ils/modules/documents/dumpers/replace_refs.py index 8185a5feab..0d805375f8 100644 --- a/rero_ils/modules/documents/dumpers/replace_refs.py +++ b/rero_ils/modules/documents/dumpers/replace_refs.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO -# Copyright (C) 2019-2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -23,27 +23,27 @@ from invenio_records.api import _records_state from invenio_records.dumpers import Dumper +from rero_ils.modules.commons.exceptions import RecordNotFound + from ..models import DocumentSubjectType -class ReplaceRefsContributionsDumper(Dumper): +class ReplaceRefsEntitiesDumper(Dumper): """Replace linked contributions in document.""" @staticmethod - def _replace_contribution(data): + def _replace_entity(data): """Replace the `$ref` linked contributions.""" - from rero_ils.modules.contributions.api import Contribution - - if entity := Contribution.get_record_by_pid(data['pid']): - _type, _ = Contribution.get_type_and_pid_from_ref(data['$ref']) - contribution = entity.dumps_for_document() - contribution.update({ - 'primary_source': _type, - 'pid': data['pid'] - }) - return contribution - else: - raise Exception(f'Contribution does not exists for {self.pid}') + from rero_ils.modules.entities.api import Entity + if not (entity := Entity.get_record_by_pid(data['pid'])): + raise RecordNotFound(Entity, data['pid']) + _type, _ = Entity.get_type_and_pid_from_ref(data['$ref']) + contribution = entity.dumps_for_document() + contribution.update({ + 'primary_source': _type, + 'pid': data['pid'] + }) + return contribution def dump(self, record, data): """Dump an item instance for notification. @@ -54,14 +54,13 @@ def dump(self, record, data): """ new_contributions = [] for contribution in data.get('contribution', []): - if not contribution['entity'].get('$ref'): - new_contributions.append(contribution) - else: + if contribution['entity'].get('$ref'): new_contributions.append({ - 'entity': self._replace_contribution( - contribution['entity']), + 'entity': self._replace_entity(contribution['entity']), 'role': contribution['role'] - }) + }) + else: + new_contributions.append(contribution) if new_contributions: data['contribution'] = new_contributions return data @@ -76,13 +75,11 @@ def _replace_subjects(data): :param data: dict - subjects data. """ - from rero_ils.modules.contributions.api import Contribution - - if not (entity := Contribution.get_record_by_pid(data['pid'])): - raise Exception(f'Contribution does not exists for {data["pid"]}') + from rero_ils.modules.entities.api import Entity - _type, _ = Contribution.get_type_and_pid_from_ref( - data['$ref']) + if not (entity := Entity.get_record_by_pid(data['pid'])): + raise RecordNotFound(Entity, data['pid']) + _type, _ = Entity.get_type_and_pid_from_ref(data['$ref']) contribution = deepcopy(data) contribution.update(dict(entity)) contribution.update({ @@ -99,21 +96,20 @@ def dump(self, record, data): :param data: The initial dump data passed in by ``record.dumps()``. :return a dict with dumped data. """ - for subjects in ['subjects', 'subjects_imported']: - new_contributions = [] - for subject in data.get(subjects, []): + for field in ['subjects', 'subjects_imported']: + entities = [] + for subject in data.get(field, []): subject_type = subject.get('type') subject_ref = subject.get('$ref') if subject_ref and subject_type in [ DocumentSubjectType.PERSON, DocumentSubjectType.ORGANISATION ]: - new_contributions.append( - self._replace_subjects(subject)) + entities.append(self._replace_subjects(subject)) else: - new_contributions.append(subject) - if new_contributions: - data[subjects] = new_contributions + entities.append(subject) + if entities: + data[field] = entities return data diff --git a/rero_ils/modules/documents/extensions/add_mef_pid.py b/rero_ils/modules/documents/extensions/add_mef_pid.py index 472f097b65..b60cb59e6c 100644 --- a/rero_ils/modules/documents/extensions/add_mef_pid.py +++ b/rero_ils/modules/documents/extensions/add_mef_pid.py @@ -31,14 +31,14 @@ def add_mef_pid(self, record): :params record: dict - a document record. """ - from rero_ils.modules.contributions.api import Contribution + from rero_ils.modules.entities.api import Entity agents = record.get('subjects', []) +\ record.get('subjects_imported', []) + \ [contrib['entity'] for contrib in record.get('contribution', []) if 'entity' in contrib] for agent in agents: if contrib_ref := agent.get('$ref'): - cont, _ = Contribution.get_record_by_ref( + cont, _ = Entity.get_record_by_ref( contrib_ref) if cont: # inject mef pid diff --git a/rero_ils/modules/documents/extensions/contributions.py b/rero_ils/modules/documents/extensions/contributions.py deleted file mode 100644 index 4e3a10bb87..0000000000 --- a/rero_ils/modules/documents/extensions/contributions.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# Copyright (C) 2021 UCLouvain -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Document record extension to enrich the edition statement.""" - - -from invenio_records.extensions import RecordExtension - -from rero_ils.dojson.utils import remove_trailing_punctuation - -from ..utils import display_alternate_graphic_first - - -class EditionStatementExtension(RecordExtension): - """Adds textual information for editionStatement.""" - - @staticmethod - def format_text(edition): - """Format edition for _text. - - :param edition: dict - edition data - :returns: a string version of the edition document field - :rtype: string - """ - designations = edition.get('editionDesignation', []) - responsibilities = edition.get('responsibility', []) - designation_output = {} - for designation in designations: - language = designation.get('language', 'default') - value = designation.get('value', '') - designation_output[language] = value - responsibility_output = {} - for responsibility in responsibilities: - language = responsibility.get('language', 'default') - value = responsibility.get('value', '') - responsibility_output[language] = value - - edition_text = [] - for key, value in designation_output.items(): - value = remove_trailing_punctuation( - '{designation} / {responsibility}'.format( - designation=designation_output.get(key), - responsibility=responsibility_output.get(key, ''), - ) - ) - if display_alternate_graphic_first(key): - edition_text.insert(0, {'value': value, 'language': key}) - else: - edition_text.append({'value': value, 'language': key}) - return edition_text - - def post_dump(self, record, data, dumper=None): - """Called before a record is dumped. - - Inject `editionStatement` string version. - - :param record: invenio record - the original record. - :param data: dict - the data. - :param dumper: record dumper - dumper helper. - """ - editions = data.get('editionStatement', []) - for edition in editions: - edition['_text'] = self.format_text(edition) diff --git a/rero_ils/modules/documents/serializers/dc.py b/rero_ils/modules/documents/serializers/dc.py index 22baa83450..9401fe27d3 100644 --- a/rero_ils/modules/documents/serializers/dc.py +++ b/rero_ils/modules/documents/serializers/dc.py @@ -29,7 +29,7 @@ from rero_ils.modules.documents.api import Document from rero_ils.modules.documents.dojson.contrib.jsontodc import dublincore -from ..dumpers import document_replace_refs +from ..dumpers import document_replace_refs_dumper from ..utils import process_literal_contributions DEFAULT_LANGUAGE = LocalProxy( @@ -61,7 +61,7 @@ class DublinCoreSerializer(_DublinCoreSerializer): def transform_record(self, pid, record, links_factory=None, language=DEFAULT_LANGUAGE, **kwargs): """Transform record into an intermediate representation.""" - record = record.dumps(document_replace_refs) + record = record.dumps(document_replace_refs_dumper) contributions = process_literal_contributions( record.get('contribution', []) ) diff --git a/rero_ils/modules/documents/serializers/json.py b/rero_ils/modules/documents/serializers/json.py index b00850a49d..85e4d2b6fa 100644 --- a/rero_ils/modules/documents/serializers/json.py +++ b/rero_ils/modules/documents/serializers/json.py @@ -30,7 +30,7 @@ from rero_ils.modules.organisations.api import OrganisationsSearch from rero_ils.modules.serializers import JSONSerializer -from ..dumpers import document_replace_refs +from ..dumpers import document_replace_refs_dumper from ..extensions import TitleExtension GLOBAL_VIEW_CODE = LocalProxy(lambda: current_app.config.get( @@ -207,7 +207,7 @@ def serialize(self, pid, record, links_factory=None, **kwargs): :param record: Record instance. :param links_factory: Factory function for record links. """ - record = record.dumps(document_replace_refs) + record = record.dumps(document_replace_refs_dumper) if contributions := process_literal_contributions( record.get('contribution', [])): record['contribution'] = contributions diff --git a/rero_ils/modules/documents/serializers/marc.py b/rero_ils/modules/documents/serializers/marc.py index c40fd14311..405e5275ba 100644 --- a/rero_ils/modules/documents/serializers/marc.py +++ b/rero_ils/modules/documents/serializers/marc.py @@ -28,10 +28,10 @@ from lxml.builder import ElementMaker from werkzeug.local import LocalProxy -from rero_ils.modules.contributions.api import ContributionsSearch from rero_ils.modules.documents.dojson.contrib.jsontomarc21 import to_marc21 from rero_ils.modules.documents.dojson.contrib.jsontomarc21.model import \ replace_contribution_sources +from rero_ils.modules.entities.api import EntitiesSearch from rero_ils.modules.serializers import JSONSerializer from rero_ils.modules.utils import strip_chars @@ -114,15 +114,14 @@ def transform_records(self, hits, pid_fetcher, language, contribution_pid = contribution.get('entity', {}).get('pid') if contribution_pid: contribution_pids.append(contribution_pid) - search = ContributionsSearch() \ + search = EntitiesSearch() \ .filter('terms', pid=list(set(contribution_pids))) es_contributions = {} for hit in search.scan(): contribution = hit.to_dict() es_contributions[contribution['pid']] = contribution - order = current_app.config.get( - 'RERO_ILS_CONTRIBUTIONS_LABEL_ORDER', []) + order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', {}) source_order = order.get(language, order.get(order['fallback'], [])) records = [] for hit in hits: diff --git a/rero_ils/modules/documents/serializers/ris.py b/rero_ils/modules/documents/serializers/ris.py index bf0a845e47..a699346af7 100644 --- a/rero_ils/modules/documents/serializers/ris.py +++ b/rero_ils/modules/documents/serializers/ris.py @@ -27,7 +27,7 @@ from rero_ils.utils import get_i18n_supported_languages from .base import BaseDocumentFormatterMixin -from ..dumpers import document_replace_refs +from ..dumpers import document_replace_refs_dumper from ..utils import process_literal_contributions @@ -41,7 +41,7 @@ def serialize(self, pid, record, links_factory=None, **kwargs): :param record: Record instance. :param links_factory: Factory function for record links. """ - record = record.dumps(document_replace_refs) + record = record.dumps(document_replace_refs_dumper) if contributions := process_literal_contributions( record.get('contribution', [])): record['contribution'] = contributions diff --git a/rero_ils/modules/documents/tasks.py b/rero_ils/modules/documents/tasks.py index ece3fadea5..d047369859 100644 --- a/rero_ils/modules/documents/tasks.py +++ b/rero_ils/modules/documents/tasks.py @@ -23,7 +23,7 @@ from .utils_mef import ReplaceMefIdentifiedByContribution, \ ReplaceMefIdentifiedBySubjects -from ..contributions.api import Contribution +from ..entities.api import Entity def get_contribution_or_create(ref_pid, ref_type, count_found, count_exists, @@ -32,7 +32,7 @@ def get_contribution_or_create(ref_pid, ref_type, count_found, count_exists, ref = f'{ref_type}/{ref_pid}' if ref_type and ref_pid: # Try to get existing contribution - cont = Contribution.get_contribution(ref_type, ref_pid) + cont = Entity.get_entity(ref_type, ref_pid) if cont: # contribution exist allready count_exists.setdefault(ref, 0) @@ -41,7 +41,7 @@ def get_contribution_or_create(ref_pid, ref_type, count_found, count_exists, # contribution does not exist try: # try to get the contribution online - data = Contribution._get_mef_data_by_type(ref_pid, ref_type) + data = Entity._get_mef_data_by_type(ref_pid, ref_type) if ( data.get('idref') or data.get('gnd') or @@ -55,7 +55,7 @@ def get_contribution_or_create(ref_pid, ref_type, count_found, count_exists, # delete mef $schema data.pop('$schema', None) # create local contribution - cont = Contribution.create( + cont = Entity.create( data=data, dbcommit=True, reindex=True) else: # online contribution has no IdREf, GND or RERO diff --git a/rero_ils/modules/documents/utils.py b/rero_ils/modules/documents/utils.py index 3bff2cef24..45dae0ae63 100644 --- a/rero_ils/modules/documents/utils.py +++ b/rero_ils/modules/documents/utils.py @@ -245,8 +245,8 @@ def create_authorized_access_point(agent): if not agent: return None authorized_access_point = agent.get('preferred_name') - from ..contributions.models import ContributionType - if agent.get('type') == ContributionType.PERSON: + from ..entities.models import EntityType + if agent.get('type') == EntityType.PERSON: date_of_birth = agent.get('date_of_birth') date_of_death = agent.get('date_of_death') date = date_of_birth or '' @@ -269,7 +269,7 @@ def create_authorized_access_point(agent): authorized_access_point += f', {date}' if qualifier: authorized_access_point += f', {qualifier}' - elif agent.get('type') == ContributionType.ORGANISATION: + elif agent.get('type') == EntityType.ORGANISATION: subordinate_unit = agent.get('subordinate_unit') if subordinate_unit: authorized_access_point += f'''. {'. '.join(subordinate_unit)}''' diff --git a/rero_ils/modules/documents/utils_mef.py b/rero_ils/modules/documents/utils_mef.py index 74db24eccc..b0aff662a2 100644 --- a/rero_ils/modules/documents/utils_mef.py +++ b/rero_ils/modules/documents/utils_mef.py @@ -30,7 +30,7 @@ from webargs import ValidationError from .api import Document, DocumentsSearch -from ..contributions.api import Contribution +from ..entities.api import Entity from ..utils import set_timestamp @@ -190,14 +190,14 @@ def _query_filter(self): def get_local(self, ref_type, ref_pid): """Get local MEF record.""" - return Contribution.get_contribution(ref_type, ref_pid) + return Entity.get_entity(ref_type, ref_pid) def get_online(self, doc_pid, ref_type, ref_pid): """Get online MEF record.""" ref = f'{ref_type}/{ref_pid}' try: # try to get the contribution online - data = Contribution._get_mef_data_by_type(ref_pid, ref_type) + data = Entity._get_mef_data_by_type(ref_pid, ref_type) if data.get('idref') or data.get('gnd'): if data.get('deleted'): self.increment_count(self.count_deleted, ref, @@ -206,8 +206,8 @@ def get_online(self, doc_pid, ref_type, ref_pid): self.increment_count(self.count_found, ref, f'{doc_pid} Online found') # create and return local contribution - return Contribution.create(data=data, dbcommit=True, - reindex=True) + return Entity.create(data=data, dbcommit=True, + reindex=True) else: # online contribution has no IdREf, GND or RERO self.increment_count(self.count_no_data, ref, diff --git a/rero_ils/modules/documents/views.py b/rero_ils/modules/documents/views.py index d0788a5b6c..8efb2468b3 100644 --- a/rero_ils/modules/documents/views.py +++ b/rero_ils/modules/documents/views.py @@ -31,14 +31,14 @@ from .api import Document, DocumentsSearch from .commons import SubjectFactory -from .dumpers import document_replace_refs +from .dumpers import document_replace_refs_dumper from .extensions import EditionStatementExtension, \ ProvisionActivitiesExtension, SeriesStatementExtension, TitleExtension from .utils import display_alternate_graphic_first, get_remote_cover, \ title_format_text, title_format_text_alternate_graphic, \ title_variant_format_text from ..collections.api import CollectionsSearch -from ..contributions.api import Contribution +from ..entities.api import Entity from ..holdings.models import HoldingNoteTypes from ..items.models import ItemCirculationAction from ..libraries.api import Library @@ -262,26 +262,18 @@ def contribution_format(pid, language, viewcode, role=False): :return the contribution in formatted form. """ doc = Document.get_record_by_pid(pid) - doc = doc.dumps(document_replace_refs) + doc = doc.dumps(document_replace_refs_dumper) output = [] for contribution in doc.get('contribution', []): - cont_pid = contribution['entity'].get('pid') - if cont_pid: - contrib = Contribution.get_record_by_pid(cont_pid) + if entity_pid := contribution['entity'].get('pid'): + entity = Entity.get_record_by_pid(entity_pid) # add link link text - authorized_access_point = contrib.get_authorized_access_point( - language=language - ) - contribution_type = 'persons' - if contrib.get('type') == 'bf:Organisation': - contribution_type = 'corporate-bodies' + text = entity.get_authorized_access_point(language=language) + entity_type = 'persons' + if entity.get('type') == 'bf:Organisation': + entity_type = 'corporate-bodies' line = \ - '{text}'.format( - viewcode=viewcode, - c_type=contribution_type, - pid=cont_pid, - text=authorized_access_point - ) + f'{text}' else: line = contribution['entity']['authorized_access_point'] @@ -301,10 +293,8 @@ def edition_format(editions): """Format edition for template.""" output = [] for edition in editions: - languages = EditionStatementExtension.format_text(edition) - if languages: - for edition_text in languages: - output.append(edition_text.get('value')) + if languages := EditionStatementExtension.format_text(edition): + output.extend(edition.get('value') for edition in languages) return output @@ -472,8 +462,7 @@ def create_publication_statement(provision_activity): """Create publication statement from place, agent and date values.""" output = [] publication_texts = \ - ProvisionActivitiesExtension.format_text( - provision_activity) + ProvisionActivitiesExtension.format_text(provision_activity) for publication_text in publication_texts: language = publication_text.get('language', 'default') if display_alternate_graphic_first(language): diff --git a/rero_ils/modules/contributions/__init__.py b/rero_ils/modules/entities/__init__.py similarity index 100% rename from rero_ils/modules/contributions/__init__.py rename to rero_ils/modules/entities/__init__.py diff --git a/rero_ils/modules/contributions/api.py b/rero_ils/modules/entities/api.py similarity index 70% rename from rero_ils/modules/contributions/api.py rename to rero_ils/modules/entities/api.py index 9a0f50f221..a359184dad 100644 --- a/rero_ils/modules/contributions/api.py +++ b/rero_ils/modules/entities/api.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO -# Copyright (C) 2019-2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,7 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""API for manipulating documents.""" +"""API for manipulating entities.""" import contextlib from functools import partial @@ -29,34 +29,34 @@ from requests import codes as requests_codes from requests.exceptions import RequestException -from .models import ContributionIdentifier, ContributionMetadata, \ - ContributionUpdateAction -from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch -from ..documents.api import DocumentsIndexer, DocumentsSearch -from ..fetchers import id_fetcher -from ..minters import id_minter -from ..providers import Provider -from ...utils import get_i18n_supported_languages +from rero_ils.modules.api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch +from rero_ils.modules.documents.api import DocumentsIndexer, DocumentsSearch +from rero_ils.modules.fetchers import id_fetcher +from rero_ils.modules.minters import id_minter +from rero_ils.modules.providers import Provider +from rero_ils.utils import get_i18n_supported_languages + +from .models import EntityIdentifier, EntityMetadata, EntityUpdateAction # provider -ContributionProvider = type( - 'ContributionProvider', +EntityProvider = type( + 'EntityProvider', (Provider,), - dict(identifier=ContributionIdentifier, pid_type='cont') + dict(identifier=EntityIdentifier, pid_type='ent') ) # minter -contribution_id_minter = partial(id_minter, provider=ContributionProvider) +entity_id_minter = partial(id_minter, provider=EntityProvider) # fetcher -contribution_id_fetcher = partial(id_fetcher, provider=ContributionProvider) +entity_id_fetcher = partial(id_fetcher, provider=EntityProvider) -class ContributionsSearch(IlsRecordsSearch): +class EntitiesSearch(IlsRecordsSearch): """Mef contribution search.""" - class Meta(): + class Meta: """Meta class.""" - index = 'contributions' + index = 'entities' doc_types = None fields = ('*', ) facets = {} @@ -64,26 +64,26 @@ class Meta(): default_filter = None -class Contribution(IlsRecord): +class Entity(IlsRecord): """Mef contribution class.""" - minter = contribution_id_minter - fetcher = contribution_id_fetcher - provider = ContributionProvider - model_cls = ContributionMetadata + minter = entity_id_minter + fetcher = entity_id_fetcher + provider = EntityProvider + model_cls = EntityMetadata @classmethod - def get_contribution(cls, ref_type, ref_pid): + def get_entity(cls, ref_type, ref_pid): """Get contribution.""" if ref_type == 'mef': return cls.get_record_by_pid(ref_pid) - es_filter = Q({'term': {f'{ref_type}.pid': ref_pid}}) + es_filter = Q('term', **{f'{ref_type}.pid': ref_pid}) if ref_type == 'viaf': es_filter = Q('term', viaf_pid=ref_pid) # in case of multiple results get the more recent - query = ContributionsSearch() \ + query = EntitiesSearch() \ .params(preserve_order=True) \ .sort({'_created': {'order': 'desc'}})\ .filter(es_filter) @@ -96,23 +96,24 @@ def get_contribution(cls, ref_type, ref_pid): def get_type_and_pid_from_ref(cls, ref): """Extract agent type and pid form the MEF URL. - :params ref: MEF URL. + :params ref: MEF URI. :returns: the ref type such as idref, and the pid value. """ ref_split = ref.split('/') - ref_type = ref_split[-2] - ref_pid = ref_split[-1] - return ref_type, ref_pid + return ref_split[-2], ref_split[-1] @classmethod def get_record_by_ref(cls, ref): """Get a record from DB. If the record dos not exist get it from MEF and create it. + + :param ref: MEF URI + :returns: the corresponding `Entity` class instance """ online = False ref_type, ref_pid = cls.get_type_and_pid_from_ref(ref) - contribution = cls.get_contribution(ref_type, ref_pid) + contribution = cls.get_entity(ref_type, ref_pid) if not contribution: # We dit not find the record in DB get it from MEF and create it. nested = db.session.begin_nested() @@ -151,9 +152,7 @@ def remove_schema(cls, data): :rtype: dict. """ data.pop('$schema', None) - sources = current_app.config.get( - 'RERO_ILS_CONTRIBUTIONS_SOURCES', []) - for source in sources: + for source in current_app.config.get('RERO_ILS_AGENTS_SOURCES', []): if source in data: data[source].pop('$schema', None) return data @@ -169,11 +168,10 @@ def _get_mef_data_by_type(cls, pid, pid_type, verbose=False, url = current_app.config.get('RERO_ILS_MEF_AGENTS_URL') if pid_type == 'mef': mef_url = f'{url}/mef/?q=pid:"{pid}"' + elif pid_type == 'viaf': + mef_url = f'{url}/mef/?q=viaf_pid:"{pid}"' else: - if pid_type == 'viaf': - mef_url = f'{url}/mef/?q=viaf_pid:"{pid}"' - else: - mef_url = f'{url}/mef/?q={pid_type}.pid:"{pid}"' + mef_url = f'{url}/mef/?q={pid_type}.pid:"{pid}"' request = requests.get( url=mef_url, params=dict( @@ -186,8 +184,7 @@ def _get_mef_data_by_type(cls, pid, pid_type, verbose=False, if status == requests_codes.ok: try: data = request.json().get('hits', {}).get('hits', [None])[0] - metadata = cls.remove_schema(data['metadata']) - return metadata + return cls.remove_schema(data['metadata']) except Exception: msg = f'MEF resolver no metadata: {mef_url}' if verbose: @@ -199,19 +196,9 @@ def _get_mef_data_by_type(cls, pid, pid_type, verbose=False, current_app.logger.error(msg) raise RequestException(msg) - def get_first(self, key, default=None): - """Get the first value for given key among MEF source list.""" - value = None - sources = current_app.config.get('RERO_ILS_CONTRIBUTIONS_SOURCES', []) - for source in sources: - if source in self: - if value := self[source].get(key, default): - return value - def _get_mef_localized_value(self, key, language): """Get the 1st localized value for given key among MEF source list.""" - order = current_app.config.get( - 'RERO_ILS_CONTRIBUTIONS_LABEL_ORDER', []) + order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', []) source_order = order.get(language, order.get(order['fallback'], [])) for source in source_order: if value := self.get(source, {}).get(key, None): @@ -221,7 +208,7 @@ def _get_mef_localized_value(self, key, language): def dumps_for_document(self): """Transform the record into document contribution format.""" agent = {'pid': self.pid} - for agency in current_app.config['RERO_ILS_CONTRIBUTIONS_SOURCES']: + for agency in current_app.config['RERO_ILS_AGENTS_SOURCES']: if self.get(agency): agent['type'] = self[agency]['bf:Agent'] agent[f'id_{agency}'] = self[agency]['pid'] @@ -245,20 +232,22 @@ def dumps_for_document(self): @property def organisation_pids(self): """Get organisations pids.""" - search = DocumentsSearch().filter( - 'term', contribution__agent__pid=self.pid) - size = current_app.config.get( - 'RERO_ILS_AGGREGATION_SIZE' - ).get('organisations') - agg = A('terms', - field='holdings.organisation.organisation_pid', size=size) + search = DocumentsSearch()\ + .filter('term', contribution__entity__pid=self.pid) + agg = A( + 'terms', + field='holdings.organisation.organisation_pid', + min_doc_count=1, + size=current_app.config + .get('RERO_ILS_AGGREGATION_SIZE') + .get('organisations') + ) search.aggs.bucket('organisation', agg) results = search.execute() - organisations = { - result.key for result in results.aggregations.organisation.buckets - if result.doc_count - } - return list(organisations) + return list({ + result.key + for result in results.aggregations.organisation.buckets + }) def get_authorized_access_point(self, language): """Get localized authorized_access_point. @@ -274,18 +263,16 @@ def get_authorized_access_point(self, language): def _search_documents(self, with_subjects=True, with_subjects_imported=True): """Get documents pids.""" - search_filters = Q("term", contribution__agent__pid=self.pid) + filters = Q('term', contribution__entity__pid=self.pid) if with_subjects: - subject_filters = Q("term", subjects__pid=self.pid) & \ - Q("terms", subjects__type=['bf:Person', 'bf:Organisation']) - search_filters = search_filters | subject_filters + filters |= \ + Q('term', subjects__pid=self.pid) & \ + Q('terms', subjects__type=['bf:Person', 'bf:Organisation']) if with_subjects_imported: - subject_filters = Q("term", subjects_imported__pid=self.pid) & \ - Q("terms", subjects__type=['bf:Person', 'bf:Organisation']) - search_filters = search_filters | subject_filters - - return DocumentsSearch() \ - .query('bool', filter=[search_filters]) + filters |= \ + Q('term', subjects_imported__pid=self.pid) & \ + Q('terms', subjects__type=['bf:Person', 'bf:Organisation']) + return DocumentsSearch().filter(filters) def documents_pids(self, with_subjects=True, with_subjects_imported=True): """Get documents pids.""" @@ -312,9 +299,10 @@ def update_online( :param reindex: reindex record by record :param dbcommit: commit record to database :param verbose: verbose print + :param reindex_doc: is the related document should be reindex ? :return: updated record status and updated record """ - action = ContributionUpdateAction.UPTODATE + action = EntityUpdateAction.UPTODATE pid = self.get('pid') try: if data := self._get_mef_data_by_type( @@ -323,43 +311,45 @@ def update_online( if data.get('deleted'): current_app.logger.warning( f'UPDATE ONLINE {pid}: was deleted') - action = ContributionUpdateAction.ERROR + action = EntityUpdateAction.ERROR elif not data.get('sources'): current_app.logger.warning( f'UPDATE ONLINE {pid}: has no sources') - action = ContributionUpdateAction.ERROR + action = EntityUpdateAction.ERROR elif not data.get('type'): current_app.logger.warning( f'UPDATE ONLINE {pid}: has no type') - action = ContributionUpdateAction.ERROR + action = EntityUpdateAction.ERROR elif dict(self) != data: - action = ContributionUpdateAction.REPLACE + action = EntityUpdateAction.REPLACE self.replace(data=data, dbcommit=dbcommit, reindex=reindex) if reindex and reindex_doc: indexer = DocumentsIndexer() indexer.bulk_index(self.documents_ids()) indexer.process_bulk_queue() except Exception as err: - action = ContributionUpdateAction.ERROR + action = EntityUpdateAction.ERROR current_app.logger.warning(f'UPDATE ONLINE {pid}: {err}') return action, self def source_pids(self): """Get agents pids.""" - sources = current_app.config.get('RERO_ILS_CONTRIBUTIONS_SOURCES', []) + sources = current_app.config.get('RERO_ILS_AGENTS_SOURCES', []) return { - source: self[source]['pid'] for source in sources - if source in self} + source: self[source]['pid'] + for source in sources + if source in self + } -class ContributionsIndexer(IlsRecordsIndexer): - """Contribution indexing class.""" +class EntitiesIndexer(IlsRecordsIndexer): + """Entity indexing class.""" - record_cls = Contribution + record_cls = Entity def bulk_index(self, record_id_iterator): """Bulk index records. :param record_id_iterator: Iterator yielding record UUIDs. """ - super().bulk_index(record_id_iterator, doc_type='cont') + super().bulk_index(record_id_iterator, doc_type='ent') diff --git a/rero_ils/modules/contributions/cli.py b/rero_ils/modules/entities/cli.py similarity index 89% rename from rero_ils/modules/contributions/cli.py rename to rero_ils/modules/entities/cli.py index 415d5ee262..7a3a60d47a 100644 --- a/rero_ils/modules/contributions/cli.py +++ b/rero_ils/modules/entities/cli.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO -# Copyright (C) 2019-2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -23,35 +23,40 @@ import click from flask.cli import with_appcontext -from .sync import SyncAgent -from ..documents.tasks import \ +from rero_ils.modules.documents.tasks import \ replace_idby_contribution as task_replace_idby_contribution -from ..documents.tasks import \ +from rero_ils.modules.documents.tasks import \ replace_idby_subjects as task_replace_idby_subjects +from .sync import SyncAgent + @click.group() -def contribution(): - """Contribution management commands.""" +def entity(): + """Entity management commands.""" -def do_replace_idby(name, replace_class, verbose, debug, details, **kwargs): +def _do_replace_idby(name, replace_class, verbose, debug, details, **kwargs): """Find and replace identifiedBy.""" click.secho(f'Find and replace identifiedBy {name}.', fg='green') found, exists, deleted, no_data, no_mef = replace_class( - verbose=verbose, details=details, debug=debug, **kwargs) + verbose=verbose, + details=details, + debug=debug, + **kwargs + ) click.echo(f'Found: {found} | Exists: {exists} | Deleted: {deleted} | ' f'No Data: {no_data} | No MEF: {no_mef}') -@contribution.command() +@entity.command() @click.option('-v', '--verbose', is_flag=True, default=False) @click.option('-d', '--debug', is_flag=True, default=False) @click.option('-D', '--details', is_flag=True, default=False) @with_appcontext def replace_idby_contribution(verbose, debug, details): """Find and replace identifiedBy contributions.""" - do_replace_idby( + _do_replace_idby( name='contribution', replace_class=task_replace_idby_contribution, verbose=verbose, @@ -60,14 +65,14 @@ def replace_idby_contribution(verbose, debug, details): ) -@contribution.command() +@entity.command() @click.option('-v', '--verbose', is_flag=True, default=False) @click.option('-d', '--debug', is_flag=True, default=False) @click.option('-D', '--details', is_flag=True, default=False) @with_appcontext def replace_idby_subjects(verbose, debug, details): """Find and replace identifiedBy subjects.""" - do_replace_idby( + _do_replace_idby( name='subjects', replace_class=task_replace_idby_subjects, verbose=verbose, @@ -76,14 +81,14 @@ def replace_idby_subjects(verbose, debug, details): ) -@contribution.command() +@entity.command() @click.option('-v', '--verbose', is_flag=True, default=False) @click.option('-d', '--debug', is_flag=True, default=False) @click.option('-D', '--details', is_flag=True, default=False) @with_appcontext def replace_idby_subjects_imported(verbose, debug, details): """Find and replace identifiedBy subjects imported.""" - do_replace_idby( + _do_replace_idby( name='subjects_imported', replace_class=task_replace_idby_subjects, verbose=verbose, @@ -93,7 +98,7 @@ def replace_idby_subjects_imported(verbose, debug, details): ) -@contribution.command() +@entity.command() @click.option('-q', '--query', default='*') @click.option('-n', '--dry-run', is_flag=True, default=False) @click.option('-d', '--from-last-date', is_flag=True, default=False) @@ -113,7 +118,7 @@ def sync(query, dry_run, from_last_date, verbose, log_dir, from_date, a.sync(query, from_date) else: a.start_sync() - pids, total = a.get_contributions_pids(query, from_date) + pids, total = a.get_entities_pids(query, from_date) if in_memory: pids = list(pids) n_updated = 0 @@ -133,7 +138,7 @@ def sync(query, dry_run, from_last_date, verbose, log_dir, from_date, click.secho(f'ERROR: MEF pids: {err_pids}', fg='red') -@contribution.command() +@entity.command() @click.option('-q', '--query', default='*') @click.option('-n', '--dry-run', is_flag=True, default=False) @click.option('-v', '--verbose', count=True, default=0) @@ -146,7 +151,7 @@ def clean(query, dry_run, verbose, log_dir): a.remove_unused(query) else: a.start_clean() - pids, total = a.get_contributions_pids(query) + pids, total = a.get_entities_pids(query) n_removed = 0 err_pids = [] with click.progressbar(pids, length=total) as bar: @@ -162,7 +167,7 @@ def clean(query, dry_run, verbose, log_dir): click.secho(f'ERROR: MEF pids: {err_pids}', fg='red') -@contribution.command() +@entity.command() @click.option('-c', '--clear', is_flag=True, default=False) @with_appcontext def sync_errors(clear): diff --git a/rero_ils/modules/contributions/jsonschemas/__init__.py b/rero_ils/modules/entities/jsonschemas/__init__.py similarity index 100% rename from rero_ils/modules/contributions/jsonschemas/__init__.py rename to rero_ils/modules/entities/jsonschemas/__init__.py diff --git a/rero_ils/modules/contributions/jsonschemas/contributions/contribution-v0.0.1.json b/rero_ils/modules/entities/jsonschemas/entities/entity-v0.0.1.json similarity index 95% rename from rero_ils/modules/contributions/jsonschemas/contributions/contribution-v0.0.1.json rename to rero_ils/modules/entities/jsonschemas/entities/entity-v0.0.1.json index bb9d8f0997..9b7decfdec 100644 --- a/rero_ils/modules/contributions/jsonschemas/contributions/contribution-v0.0.1.json +++ b/rero_ils/modules/entities/jsonschemas/entities/entity-v0.0.1.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "title": "Mef contribution", - "description": "JSON schema for a mef contribution", + "title": "Mef entity", + "description": "JSON schema for a mef entity", "additionalProperties": false, "required": [ "$schema", diff --git a/rero_ils/modules/contributions/listener.py b/rero_ils/modules/entities/listener.py similarity index 62% rename from rero_ils/modules/contributions/listener.py rename to rero_ils/modules/entities/listener.py index dfeb48e3db..c6877604b2 100644 --- a/rero_ils/modules/contributions/listener.py +++ b/rero_ils/modules/entities/listener.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,13 +16,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Signals connector for Contributions.""" +"""Signals connector for `Entity` records.""" -from .api import Contribution, ContributionsSearch +from .api import EntitiesSearch, Entity -def enrich_contributions_data(sender, json=None, record=None, index=None, - doc_type=None, arguments=None, **dummy_kwargs): +def enrich_entities_data(sender, json=None, record=None, index=None, + doc_type=None, arguments=None, **dummy_kwargs): """Signal sent before a record is indexed. :param json: The dumped record dictionary which can be modified. @@ -29,8 +30,7 @@ def enrich_contributions_data(sender, json=None, record=None, index=None, :param index: The index in which the record will be indexed. :param doc_type: The doc_type for the record. """ - if index.split('-')[0] == ContributionsSearch.Meta.index: - contribution = record - if not isinstance(record, Contribution): - contribution = Contribution.get_record_by_pid(record.get('pid')) - json['organisations'] = contribution.organisation_pids + if index.split('-')[0] == EntitiesSearch.Meta.index: + if not isinstance(record, Entity): + record = Entity.get_record_by_pid(record.get('pid')) + json['organisations'] = record.organisation_pids diff --git a/rero_ils/modules/contributions/mappings/__init__.py b/rero_ils/modules/entities/mappings/__init__.py similarity index 100% rename from rero_ils/modules/contributions/mappings/__init__.py rename to rero_ils/modules/entities/mappings/__init__.py diff --git a/rero_ils/modules/contributions/mappings/v7/__init__.py b/rero_ils/modules/entities/mappings/v7/__init__.py similarity index 100% rename from rero_ils/modules/contributions/mappings/v7/__init__.py rename to rero_ils/modules/entities/mappings/v7/__init__.py diff --git a/rero_ils/modules/contributions/mappings/v7/contributions/contribution-v0.0.1.json b/rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json similarity index 100% rename from rero_ils/modules/contributions/mappings/v7/contributions/contribution-v0.0.1.json rename to rero_ils/modules/entities/mappings/v7/entities/entity-v0.0.1.json diff --git a/rero_ils/modules/contributions/models.py b/rero_ils/modules/entities/models.py similarity index 70% rename from rero_ils/modules/contributions/models.py rename to rero_ils/modules/entities/models.py index 5244dcd730..d38af0579b 100644 --- a/rero_ils/modules/contributions/models.py +++ b/rero_ils/modules/entities/models.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -24,10 +25,10 @@ from invenio_records.models import RecordMetadataBase -class ContributionIdentifier(RecordIdentifier): - """Sequence generator for Document identifiers.""" +class EntityIdentifier(RecordIdentifier): + """Sequence generator for `Entity` identifiers.""" - __tablename__ = 'contribution_id' + __tablename__ = 'entity_id' __mapper_args__ = {'concrete': True} recid = db.Column( @@ -37,21 +38,21 @@ class ContributionIdentifier(RecordIdentifier): ) -class ContributionMetadata(db.Model, RecordMetadataBase): - """Contribution record metadata.""" +class EntityMetadata(db.Model, RecordMetadataBase): + """Entity record metadata.""" - __tablename__ = 'contribution_metadata' + __tablename__ = 'entity_metadata' -class ContributionType: - """Class holding all availabe contribution types.""" +class EntityType: + """Class holding all available entity types.""" ORGANISATION = 'bf:Organisation' PERSON = 'bf:Person' -class ContributionUpdateAction: - """Class holding all availabe agent record creation actions.""" +class EntityUpdateAction: + """Class holding all available agent record creation actions.""" REPLACE = 'replace' UPTODATE = 'uptodate' diff --git a/rero_ils/modules/contributions/permissions.py b/rero_ils/modules/entities/permissions.py similarity index 72% rename from rero_ils/modules/contributions/permissions.py rename to rero_ils/modules/entities/permissions.py index 7779b2e59e..179606fdc4 100644 --- a/rero_ils/modules/contributions/permissions.py +++ b/rero_ils/modules/entities/permissions.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO -# Copyright (C) 2019-2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,17 +16,17 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Permissions for contributions.""" +"""Permissions for `Entity` records.""" from invenio_records_permissions.generators import AnyUser from rero_ils.modules.permissions import RecordPermissionPolicy -class ContributionPermissionPolicy(RecordPermissionPolicy): - """Contribution Permission Policy used by the CRUD operations. +class EntityPermissionPolicy(RecordPermissionPolicy): + """Entity Permission Policy used by the CRUD operations. - Only search and read is allowed for all users. Other operations is denied - far all. + Only search and read is allowed for all users. + Other operations are denied far anybody. """ can_search = [AnyUser()] diff --git a/rero_ils/modules/contributions/sync.py b/rero_ils/modules/entities/sync.py similarity index 82% rename from rero_ils/modules/contributions/sync.py rename to rero_ils/modules/entities/sync.py index 590789ebcb..7ddbe18387 100644 --- a/rero_ils/modules/contributions/sync.py +++ b/rero_ils/modules/entities/sync.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019 RERO -# Copyright (C) 2020 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -31,9 +31,8 @@ from elasticsearch_dsl import Q from flask import current_app -from rero_ils.modules.contributions.api import Contribution, \ - ContributionsSearch from rero_ils.modules.documents.api import Document, DocumentsSearch +from rero_ils.modules.entities.api import EntitiesSearch, Entity from rero_ils.modules.utils import get_timestamp, set_timestamp @@ -112,7 +111,7 @@ def _init_logger(self, verbose, log_dir): self.logger = logging.getLogger(__name__) self.log_dir = log_dir - def agent_are_different(self, agent1, agent2): + def _agent_are_different(self, agent1, agent2): """Check if two agent are different. The comparison is done only on the common fields. @@ -147,45 +146,42 @@ def remove_fields(agent): return True return False - def _get_latest(self, source, pid): + @staticmethod + def _get_latest(source, pid): """Query the MEF server to retrieve the last MEF for a given agent id. - :param source - string: the agent source such as idref, gnd. - :param pid - string: the identifier of the agent. - :returns: a dict of the MEF record. + :param source: (string) the agent source such as `idref`, `gnd` + :param pid: (string) the agent identifier. + :returns: dictionary representing the MEF record. :rtype: dictionary. """ mef_url = current_app.config.get('RERO_ILS_MEF_AGENTS_URL') url = f'{mef_url}/mef/latest/{source}:{pid}' return requests.get(url).json() - def update_agents_in_document( - self, - doc_pid, - pids_to_replace - ): + def _update_agents_in_document(self, doc_pid, pids_to_replace): """Updates the contribution and subjects in document. - :param doc_pid - string: document pid - :param source - string: agent source i.e. gnd, rero, mef - :param old_agent_pid - string: old agent pid from the database - :param new_agent_pid - string: new agent pid from the mef server - :param new_mef_pid - string: new MEF pid + :param doc_pid: (string) document pid + :param pids_to_replace: (dict) the list of object where replace the + agents. Dictionary keys are `source` ; dictionary values are + tuple of (old_agent_pid, new_agent_pid) + >> {'gnd': ('agent_old1', 'agent_new1')} """ # get the document from the DB doc = Document.get_record_by_pid(doc_pid) - - # build the $ ref urls + # build the $ref urls mef_url = current_app.config.get('RERO_ILS_MEF_AGENTS_URL') # get all agents from the document over all agent fields: # contribution and subjects agents = [ - subject for subject in doc.get('subjects', []) + subject + for subject in doc.get('subjects', []) if subject.get('$ref') - ] - agents += [ - contrib['entity'] for contrib in doc.get('contribution', []) + ] + [ + contrib['entity'] + for contrib in doc.get('contribution', []) if contrib.get('entity', {}).get('$ref') ] if not agents: @@ -208,34 +204,35 @@ def update_agents_in_document( if not self.dry_run: doc.replace(doc, dbcommit=True, reindex=True) - def get_documents_pids_from_mef(self, pid): + @staticmethod + def _get_documents_pids_from_mef(pid): """Retrieve all the linked documents to a MEF record. - :param pid - string: a MEF identifier. + :param pid: (string) a MEF identifier. :returns: a list of identifiers. :rtype: list of strings. """ # the MEF link can be in contribution or subjects es_query = DocumentsSearch() - filters = Q('term', contribution__entity__pid=pid) - filters |= Q('term', subjects__pid=pid) + filters = Q('term', contribution__entity__pid=pid) |\ + Q('term', subjects__pid=pid) es_query = es_query.filter('bool', must=[filters]).source('pid') - # can be a list as the should not be too big + # can be a list as it should not be too big return [d.pid for d in es_query.params(scroll='30m').scan()] - def get_contributions_pids(self, query='*', from_date=None): + def get_entities_pids(self, query='*', from_date=None): """Get contributions identifiers. - :param query - string: a query to select the MEF record to be updated. - :param from_date - string: only the MEF updated on the MEF server after - the given date will be considered. + :param query: (string) a query to select the MEF record to be updated. + :param from_date: (string) only the MEF records updated on the MEF + server after the given date will be considered. :returns: the list of the contribution identifiers. :rtype: list of strings. """ logging.basicConfig(filename='myfile.log', level=logging.DEBUG) mef_url = current_app.config.get('RERO_ILS_MEF_AGENTS_URL') url = f'{mef_url}/mef/updated' - es_query = ContributionsSearch().filter('query_string', query=query) + es_query = EntitiesSearch().filter('query_string', query=query) total = es_query.count() if not from_date and self.from_date: from_date = self.from_date @@ -245,9 +242,9 @@ def get_contributions_pids(self, query='*', from_date=None): def get_mef_pids(es_query, chunk_size=1000): """Get the identifiers from elasticsearch. - :param es_query: string - the elasticsearch query to limit the + :param es_query: (string) the elasticsearch query to limit the results - :param chunk_size: integer - the maximum number of pid per chunk + :param chunk_size: (integer) the maximum number of pid per chunk :returns: iterator over all pids The scroll is done using the slice scroll feature: @@ -312,21 +309,21 @@ def get_updated_mef(pids, chunk_size): def sync_record(self, pid): """Sync a MEF record. - :param pid - string: the MEF identifier. + :param pid: (string) the MEF identifier. :returns: the number of updated document, true if the MEF record - has been update, true if an error occurs. - :rtype: integer, boolean, boolean. + has been update, true if an error occurs. + :rtype: integer, boolean, x. """ doc_updated = set() updated = error = False try: # get contribution in db - agent = Contribution.get_record_by_pid(pid) + agent = Entity.get_record_by_pid(pid) if not agent: raise Exception(f'ERROR MEF {pid} does not exists in db.') self.logger.debug(f'Processing MEF(pid: {pid})') # iterate over all agent sources: rero, gnd, idref - doc_pids = self.get_documents_pids_from_mef(agent.pid) + doc_pids = self._get_documents_pids_from_mef(agent.pid) pids_to_replace = {} for source in agent['sources']: mef = self._get_latest(source, agent[source]["pid"]) @@ -345,7 +342,7 @@ def sync_record(self, pid): pids_to_replace[source] = (old_agent_pid, new_agent_pid) # can be mef pid, source pid or metadata - if self.agent_are_different(dict(agent), mef): + if self._agent_are_different(dict(agent), mef): # need a copy as we want to keep the MEF record # untouched for the next agent new_mef_data = deepcopy(mef) @@ -358,21 +355,8 @@ def sync_record(self, pid): f'MEF pid has changed from {old_mef_pid} to ' f'{new_mef_pid} for {source} (pid:{old_agent_pid})' ) - # if the MEF record does not exists create it - new_agent = Contribution.get_record_by_pid(new_mef_pid) - - if not new_agent: - if not self.dry_run: - Contribution.create( - data=new_mef_data, - dbcommit=True, - reindex=True - ) - self.logger.info( - f'Create a new MEF record(pid: {new_mef_pid})') - else: - # update the new MEF - # recursion + if Entity.get_record_by_pid(new_mef_pid): + # update the new MEF - recursion self.logger.info( f'MEF(pid: {agent.pid}) recursion with' f' (pid:{new_mef_pid})') @@ -382,22 +366,30 @@ def sync_record(self, pid): doc_updated.update(new_doc_updated) updated = updated or new_updated error = error or new_error - # something changed - # update the content + else: + # if the MEF record does not exist create it + if not self.dry_run: + Entity.create( + data=new_mef_data, + dbcommit=True, + reindex=True + ) + self.logger.info( + f'Create a new MEF record(pid: {new_mef_pid})') + # something changed, update the content self.logger.info( - f'MEF(pid: {agent.pid}) content has been ' - f'updated') + f'MEF(pid: {agent.pid}) content has been updated') if not self.dry_run: if old_mef_pid == new_mef_pid: - Contribution.get_record(agent.id).replace( + Entity.get_record(agent.id).replace( new_mef_data, dbcommit=True, reindex=True) else: # as we have only the last mef but not the old one # we need get it from the MEF server # this is important as it can still be used by # other agents - Contribution.get_record_by_pid( - pid).update_online(dbcommit=True, reindex=True) + Entity.get_record_by_pid(pid)\ + .update_online(dbcommit=True, reindex=True) updated = True if updated: @@ -406,13 +398,13 @@ def sync_record(self, pid): f'MEF(pid: {agent.pid}) try to update ' f'documents: {doc_pids}') for doc_pid in doc_pids: - self.update_agents_in_document( + self._update_agents_in_document( doc_pid=doc_pid, pids_to_replace=pids_to_replace ) doc_updated = set(doc_pids) except Exception as e: - self.logger.error(f'ERROR: MEF(pid:{pid}) -> {e}') + self.logger.error(f'ERROR: MEF(pid:{pid}) -> {str(e)}') error = True # uncomment to debug # raise @@ -434,10 +426,10 @@ def end_sync(self, n_doc_updated, n_mef_updated, mef_errors): f'mef updated: {n_mef_updated}.') if self.dry_run: return - data = get_timestamp('sync_agents') - errors = [] - if data: + if data := get_timestamp('sync_agents'): errors = data.get('errors', []) + else: + errors = [] errors += mef_errors set_timestamp( 'sync_agents', n_doc_updated=n_doc_updated, @@ -447,15 +439,18 @@ def end_sync(self, n_doc_updated, n_mef_updated, mef_errors): def sync(self, query="*", from_date=None, in_memory=False): """Updated the MEF records and the linked documents. - :param query - string: a query to select the MEF record to be updated. - :param from_date - string: only the MEF updated on the MEF server after - the given date will be considered. + :param query: (string) a query to select the MEF record to be updated. + :param from_date: (string) only the MEF records updated on the MEF + server after the given date will be considered. + :param in_memory: (boolean) is the record could be stored in memory + instead of using generator. Use to avoid ElasticSearch timeout + problem in case of big data set. :returns: the number of updated documents, the number of updated MEF - records, the list of MEF pids that generate an error. + records, the list of MEF pids that generate an error. :rtype: integer, integer, list of strings. """ self.start_sync() - pids, _ = self.get_contributions_pids(query, from_date=from_date) + pids, _ = self.get_entities_pids(query, from_date=from_date) if in_memory: pids = list(pids) # number of document updated @@ -474,17 +469,17 @@ def sync(self, query="*", from_date=None, in_memory=False): return len(doc_updated), n_mef_updated, mef_errors def remove_unused_record(self, pid): - """Removes MEF record if it is not linked to documents. + """Removes MEF record if it is not linked to any documents. - :param pid - string: MEF identifier. + :param pid: (string) MEF identifier. :returns: true if the record has been deleted, true if an error occurs. :rtype: boolean, boolean """ try: - doc_pids = self.get_documents_pids_from_mef(pid) + doc_pids = SyncAgent._get_documents_pids_from_mef(pid) if len(doc_pids) == 0: # get the contribution for the database - contrib = Contribution.get_record_by_pid(pid) + contrib = Entity.get_record_by_pid(pid) if not self.dry_run: # remove from the database and the index: no tombstone contrib.delete(True, True, True) @@ -501,8 +496,7 @@ def remove_unused_record(self, pid): @classmethod def get_errors(cls): """Get all the MEF pids that causes an error.""" - data = get_timestamp('sync_agents') - return data.get('errors', []) + return get_timestamp('sync_agents').get('errors', []) @classmethod def clear_errors(cls): @@ -522,17 +516,17 @@ def start_clean(self): self.logger.info('--------- Starting cleaning ---------') def remove_unused(self, query="*"): - """Removes MEF records that are not linked to documents. + """Removes MEF records that are not linked to any documents. - :param query - string: query to limit the record candidates. + :param query: (string) query to limit the record candidates. :returns: the number of deleted records, the list of pid that - causes an error. + causes an error. :rtype: integer, list of strings. """ self.start_clean() n_removed = 0 err_pids = [] - pids, _ = self.get_contributions_pids(query) + pids, _ = self.get_entities_pids(query) for pid in pids: removed, error = self.remove_unused_record(pid) if removed: diff --git a/rero_ils/modules/contributions/tasks.py b/rero_ils/modules/entities/tasks.py similarity index 83% rename from rero_ils/modules/contributions/tasks.py rename to rero_ils/modules/entities/tasks.py index 2f0bb6aa9b..c2ca58e11a 100644 --- a/rero_ils/modules/contributions/tasks.py +++ b/rero_ils/modules/entities/tasks.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,7 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Celery tasks to mef records.""" +"""Celery tasks for `Entity` records.""" from __future__ import absolute_import, print_function @@ -23,7 +24,7 @@ from celery import shared_task from flask import current_app -from .api import Contribution +from .api import Entity from .sync import SyncAgent @@ -36,7 +37,7 @@ def delete_records(records, verbose=False): :return: count of records """ for record in records: - status = Contribution.delete( + status = Entity.delete( record, force=False, dbcommit=True, @@ -51,12 +52,12 @@ def delete_records(records, verbose=False): @shared_task(ignore_result=True) def sync_agents(from_last_date=True, verbose=0, dry_run=False): - """Synchonize the agents within the MEF server. + """Synchronize the agents within the MEF server. - :param from_last_date: boolean - if True try to consider agent modified + :param from_last_date: (boolean) if True try to consider agent modified after the last run date time - :param verbose: bool or integer - verbose level - :param dry_run: bool - if true the data are not modified + :param verbose: (boolean|integer) verbose level + :param dry_run: (boolean) if true the data are not modified """ agent = SyncAgent( from_last_date=from_last_date, verbose=verbose, dry_run=dry_run) diff --git a/rero_ils/modules/contributions/templates/rero_ils/_contribution_by_source.html b/rero_ils/modules/entities/templates/rero_ils/_entity_by_source.html similarity index 92% rename from rero_ils/modules/contributions/templates/rero_ils/_contribution_by_source.html rename to rero_ils/modules/entities/templates/rero_ils/_entity_by_source.html index d34b84c81a..3eb2fe35ce 100644 --- a/rero_ils/modules/contributions/templates/rero_ils/_contribution_by_source.html +++ b/rero_ils/modules/entities/templates/rero_ils/_entity_by_source.html @@ -1,7 +1,8 @@ {# -*- coding: utf-8 -*- RERO ILS - Copyright (C) 2019-2022 RERO + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -43,7 +44,7 @@

{% with data=record[agency], source_name='agency', source=agency %} - {% include('rero_ils/_contribution_by_source_data.html') %} + {% include('rero_ils/_entity_by_source_data.html') %} {% endwith %}
diff --git a/rero_ils/modules/contributions/templates/rero_ils/_contribution_by_source_data.html b/rero_ils/modules/entities/templates/rero_ils/_entity_by_source_data.html similarity index 91% rename from rero_ils/modules/contributions/templates/rero_ils/_contribution_by_source_data.html rename to rero_ils/modules/entities/templates/rero_ils/_entity_by_source_data.html index 887285f8b2..0086768e61 100644 --- a/rero_ils/modules/contributions/templates/rero_ils/_contribution_by_source_data.html +++ b/rero_ils/modules/entities/templates/rero_ils/_entity_by_source_data.html @@ -1,7 +1,8 @@ {# -*- coding: utf-8 -*- RERO ILS - Copyright (C) 2019-2022 RERO + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -17,7 +18,7 @@ #} -{% from 'rero_ils/macros/contribution.html' import dl, dl_bool, dl_permalink_by_source %} +{% from 'rero_ils/macros/entity.html' import dl, dl_bool, dl_permalink_by_source %} {{ dl(_('Birth date'), data.date_of_birth) }} {{ dl(_('Death date'), data.date_of_death) }} diff --git a/rero_ils/modules/contributions/templates/rero_ils/_contribution_unified.html b/rero_ils/modules/entities/templates/rero_ils/_entity_unified.html similarity index 89% rename from rero_ils/modules/contributions/templates/rero_ils/_contribution_unified.html rename to rero_ils/modules/entities/templates/rero_ils/_entity_unified.html index 2b59b4d6fe..cf6305d56c 100644 --- a/rero_ils/modules/contributions/templates/rero_ils/_contribution_unified.html +++ b/rero_ils/modules/entities/templates/rero_ils/_entity_unified.html @@ -1,7 +1,8 @@ {# -*- coding: utf-8 -*- RERO ILS - Copyright (C) 2019-2022 RERO + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -17,9 +18,9 @@ #} -{% from 'rero_ils/macros/contribution.html' import dl, dl_bool, dl_permalink %} +{% from 'rero_ils/macros/entity.html' import dl, dl_bool, dl_permalink %} {% set record = record %} -{% set data = record|contribution_merge_data_values %} +{% set data = record | entity_merge_data_values %}
{{ dl(_('Birth date'), data.date_of_birth) }} diff --git a/rero_ils/modules/contributions/templates/rero_ils/detailed_view_contribution.html b/rero_ils/modules/entities/templates/rero_ils/detailed_view_entity.html similarity index 93% rename from rero_ils/modules/contributions/templates/rero_ils/detailed_view_contribution.html rename to rero_ils/modules/entities/templates/rero_ils/detailed_view_entity.html index 4e1fea8536..5541cbe30d 100644 --- a/rero_ils/modules/contributions/templates/rero_ils/detailed_view_contribution.html +++ b/rero_ils/modules/entities/templates/rero_ils/detailed_view_entity.html @@ -29,7 +29,7 @@ {% endif %}
-

{{ record | contribution_label(current_i18n.language) }}

+

{{ record | entity_label(current_i18n.language) }}

MEF ID: {{ record.pid }}
@@ -56,10 +56,10 @@

{{ record | contribution_label(current_i18n.language) }}

- {% include('rero_ils/_contribution_unified.html') %} + {% include('rero_ils/_entity_unified.html') %}
- {% include('rero_ils/_contribution_by_source.html') %} + {% include('rero_ils/_entity_by_source.html') %}
diff --git a/rero_ils/modules/contributions/templates/rero_ils/macros/contribution.html b/rero_ils/modules/entities/templates/rero_ils/macros/entity.html similarity index 92% rename from rero_ils/modules/contributions/templates/rero_ils/macros/contribution.html rename to rero_ils/modules/entities/templates/rero_ils/macros/entity.html index 08b31d0945..9bd81f1392 100644 --- a/rero_ils/modules/contributions/templates/rero_ils/macros/contribution.html +++ b/rero_ils/modules/entities/templates/rero_ils/macros/entity.html @@ -1,7 +1,8 @@ {# -*- coding: utf-8 -*- RERO ILS - Copyright (C) 2019-2022 RERO + Copyright (C) 2019-2023 RERO + Copyright (C) 2019-2023 UCLouvain This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -30,7 +31,7 @@ {{ value }} {% endif %} {% else %} -
    +
      {% for element in value %}
    • {% if prefix %} @@ -65,8 +66,8 @@ {{ _(name) }}:
      -
        - {% for source in config.RERO_ILS_CONTRIBUTIONS_SOURCES %} +
          + {% for source in config.RERO_ILS_AGENTS_SOURCES %} {% if record[source] %}
        • {% if source != 'rero' %} diff --git a/rero_ils/modules/contributions/utils.py b/rero_ils/modules/entities/utils.py similarity index 71% rename from rero_ils/modules/contributions/utils.py rename to rero_ils/modules/entities/utils.py index 5094f7e04a..74d76513bc 100644 --- a/rero_ils/modules/contributions/utils.py +++ b/rero_ils/modules/entities/utils.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,25 +16,24 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Contributions utilities.""" +"""Entities utilities.""" from __future__ import absolute_import, print_function from flask import current_app -def get_contribution_localized_value(contribution, key, language): - """Get the 1st localized value for given key among MEF source list. +def get_entity_localized_value(entity, key, language): + """Get the first localized value for given key among MEF source list. - :param contribution: Contribution data. + :param entity: Entity data. :param key: Key to find a translated form. :param language: Language to use. :returns: Value from key in language if found otherwise the value of key. """ - order = current_app.config.get( - 'RERO_ILS_CONTRIBUTIONS_LABEL_ORDER', []) + order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', []) source_order = order.get(language, order.get(order['fallback'], [])) for source in source_order: - if value := contribution.get(source, {}).get(key): + if value := entity.get(source, {}).get(key): return value - return contribution.get(key) + return entity.get(key) diff --git a/rero_ils/modules/contributions/views.py b/rero_ils/modules/entities/views.py similarity index 67% rename from rero_ils/modules/contributions/views.py rename to rero_ils/modules/entities/views.py index cd59f0d14a..6354410027 100644 --- a/rero_ils/modules/contributions/views.py +++ b/rero_ils/modules/entities/views.py @@ -25,14 +25,14 @@ from flask_babelex import gettext as translate from invenio_records_ui.signals import record_viewed -from .api import Contribution -from .models import ContributionType +from .api import Entity +from .models import EntityType from ..documents.api import DocumentsSearch from ..organisations.api import Organisation from ...theme.views import url_active blueprint = Blueprint( - 'contributions', + 'entities', __name__, url_prefix='/', template_folder='templates', @@ -40,79 +40,106 @@ ) api_blueprint = Blueprint( - 'api_contributions', + 'api_entities', __name__ ) -def contribution_proxy(viewcode, pid, contribution_type): - """Proxy for contributions. +def entity_proxy(viewcode, pid, entity_type): + """Proxy for entities. :param viewcode: viewcode of html request :param pid: pid of contribution - :param contribution_type: type of contribution - :returns: contribution template + :param entity_type: type of the entity + :returns: entity template """ - contribution = Contribution.get_record_by_pid(pid) - if not contribution or contribution.get('type') != contribution_type: + entity = Entity.get_record_by_pid(pid) + if not entity or entity.get('type') != entity_type: abort(404, 'Record not found') - return contribution_view_method( - pid=contribution.persistent_identifier, - record=contribution, - template='rero_ils/detailed_view_contribution.html', + return entity_view_method( + pid=entity.persistent_identifier, + record=entity, + template='rero_ils/detailed_view_entity.html', viewcode=viewcode ) -@blueprint.route('/persons/', methods=['GET']) -def persons_proxy(viewcode, pid): - """Proxy person for contribution.""" - return contribution_proxy(viewcode, pid, ContributionType.PERSON) - - -@blueprint.route('/corporate-bodies/', methods=['GET']) -def corporate_bodies_proxy(viewcode, pid): - """Proxy corporate bodies for contribution.""" - return contribution_proxy(viewcode, pid, 'bf:Organisation') - - -def contribution_view_method(pid, record, template=None, **kwargs): +def entity_view_method(pid, record, template=None, **kwargs): """Display default view. Sends record_viewed signal and renders template. + :param pid: PID object. + :param record: the `Entity` record, + :param template: the template to use to render the entity """ record_viewed.send( current_app._get_current_object(), pid=pid, record=record) # Get contribution persons documents - search = DocumentsSearch().filter( - 'term', contribution__agent__pid=pid.pid_value - ) + search = DocumentsSearch()\ + .filter('term', contribution__entity__pid=pid.pid_value) viewcode = kwargs['viewcode'] - if (viewcode != current_app.config.get( - 'RERO_ILS_SEARCH_GLOBAL_VIEW_CODE' - )): + if viewcode != current_app.config.get('RERO_ILS_SEARCH_GLOBAL_VIEW_CODE'): org_pid = Organisation.get_record_by_viewcode(viewcode)['pid'] search = search \ .filter('term', holdings__organisation__organisation_pid=org_pid) search = search \ .params(preserve_order=True)\ - .sort({'sort_title': {"order": "asc"}}) + .sort({'sort_title': {'order': 'asc'}}) record['documents'] = list(search.scan()) - return render_template( - template, - record=record, - viewcode=viewcode + return render_template(template, record=record, viewcode=viewcode) + + +@blueprint.route('/persons/', methods=['GET']) +def persons_proxy(viewcode, pid): + """Proxy person for entity.""" + return entity_proxy(viewcode, pid, EntityType.PERSON) + + +@blueprint.route('/corporate-bodies/', methods=['GET']) +def corporate_bodies_proxy(viewcode, pid): + """Proxy corporate bodies for entity.""" + return entity_proxy(viewcode, pid, EntityType.ORGANISATION) + + +@api_blueprint.route('/mef/', defaults={'path': ''}) +@api_blueprint.route('/mef/') +def mef_proxy(path): + """Proxy to mef server.""" + resp = requests.request( + method=request.method, + url=request.url.replace( + request.base_url.replace(path, ''), + f'{current_app.config.get("RERO_ILS_MEF_AGENTS_URL")}/mef/' + ), + headers={ + key: value for (key, value) in request.headers if key != 'Host' + }, + data=request.get_data(), + cookies=request.cookies, + allow_redirects=True ) + excluded_headers = ['content-encoding', 'content-length', + 'transfer-encoding', 'connection'] + headers = [ + (name, value) + for (name, value) in resp.raw.headers.items() + if name.lower() not in excluded_headers + ] + response = Response(resp.content, resp.status_code, headers) + if response.status_code != requests.codes.ok: + abort(response.status_code) + return response +# TEMPLATE JINJA FILTERS ====================================================== @blueprint.app_template_filter() -def contribution_merge_data_values(data): +def entity_merge_data_values(data): """Create merged data for values.""" - sources = current_app.config.get('RERO_ILS_CONTRIBUTIONS_SOURCES', []) + sources = current_app.config.get('RERO_ILS_AGENTS_SOURCES', []) result = {} for source in sources: if data.get(source): @@ -137,14 +164,12 @@ def contribution_merge_data_values(data): @blueprint.app_template_filter() -def contribution_label(data, language): +def entity_label(data, language): """Create contribution label.""" - order = current_app.config.get('RERO_ILS_CONTRIBUTIONS_LABEL_ORDER', []) + order = current_app.config.get('RERO_ILS_AGENTS_LABEL_ORDER', []) source_order = order.get(language, order.get(order['fallback'], [])) - for source in source_order: - label = data.get(source, {}).get('authorized_access_point', None) - if label: + if label := data.get(source, {}).get('authorized_access_point', None): return label return '-' @@ -157,66 +182,29 @@ def translat_unified(data, prefix=''): :param prefix: prefix to add to keys :returns: dictionary with translated keys """ - translated_data = {} - for key, value in data.items(): - translated_data[translate('{prefix}{key}'.format( - prefix=prefix, key=key))] = value - return translated_data + return { + translate(f'{prefix}{key}'): value + for key, value + in data.items() + } @blueprint.app_template_filter() @api_blueprint.app_template_filter() def translat(data, prefix='', seperator=', '): """Translate data.""" - translated = None if data: if isinstance(data, list): - translated = [] - for item in data: - translated.append(translate('{prefix}{item}'.format( - prefix=prefix, item=item))) - translated = seperator.join(translated) + translated = [translate(f'{prefix}{item}') for item in data] + return seperator.join(translated) elif isinstance(data, str): - translated = translate('{prefix}{data}'.format( - prefix=prefix, data=data)) - return translated + return translate(f'{prefix}{data}') @blueprint.app_template_filter('biographicaUrl') def biographical_url(biographicals): """Add link url on text if http detected.""" - output = dict() - for biographical in biographicals: - output[url_active(biographical, '_blank')] = \ - biographicals[biographical] - return output - - -@api_blueprint.route('/mef/', defaults={'path': ''}) -@api_blueprint.route('/mef/') -def mef_proxy(path): - """Proxy to mef server.""" - resp = requests.request( - method=request.method, - url=request.url.replace( - request.base_url.replace(path, ''), - f'{current_app.config.get("RERO_ILS_MEF_AGENTS_URL")}/mef/' - ), - headers={ - key: value for (key, value) in request.headers if key != 'Host' - }, - data=request.get_data(), - cookies=request.cookies, - allow_redirects=True - ) - excluded_headers = ['content-encoding', 'content-length', - 'transfer-encoding', 'connection'] - headers = [ - (name, value) for (name, value) in resp.raw.headers.items() - if name.lower() not in excluded_headers - ] - - response = Response(resp.content, resp.status_code, headers) - if response.status_code != requests.codes.ok: - abort(response.status_code) - return response + return { + url_active(biographical, '_blank'): biographicals[biographical] + for biographical in biographicals + } diff --git a/rero_ils/modules/ext.py b/rero_ils/modules/ext.py index 131b0f6866..329a65c0fb 100644 --- a/rero_ils/modules/ext.py +++ b/rero_ils/modules/ext.py @@ -56,9 +56,9 @@ from rero_ils.modules.acquisition.budgets.listener import \ budget_is_active_changed from rero_ils.modules.collections.listener import enrich_collection_data -from rero_ils.modules.contributions.listener import enrich_contributions_data from rero_ils.modules.documents.listener import enrich_document_data from rero_ils.modules.ebooks.receivers import publish_harvested_records +from rero_ils.modules.entities.listener import enrich_entities_data from rero_ils.modules.holdings.listener import enrich_holding_data, \ update_items_locations_and_types from rero_ils.modules.ill_requests.listener import enrich_ill_request_data @@ -292,7 +292,7 @@ def register_signals(self, app): before_record_index.connect(enrich_collection_data, sender=app) before_record_index.connect(enrich_loan_data, sender=app) before_record_index.connect(enrich_document_data, sender=app) - before_record_index.connect(enrich_contributions_data, sender=app) + before_record_index.connect(enrich_entities_data, sender=app) before_record_index.connect(enrich_item_data, sender=app) before_record_index.connect(enrich_patron_data, sender=app) before_record_index.connect(enrich_location_data, sender=app) diff --git a/rero_ils/modules/indexer_utils.py b/rero_ils/modules/indexer_utils.py index ad3fd565da..e818c9821e 100644 --- a/rero_ils/modules/indexer_utils.py +++ b/rero_ils/modules/indexer_utils.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019-2022 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -40,8 +41,8 @@ def record_to_index(record): # authorities specific transformation if re.search(r'/mef/', schema): - schema = re.sub(r'/mef/', '/contributions/', schema) - schema = re.sub(r'mef-contribution', 'contribution', schema) + schema = re.sub(r'/mef/', '/entities/', schema) + schema = re.sub(r'mef-contribution', 'entity', schema) index, doc_type = schema_to_index(schema, index_names=index_names) if index and doc_type: diff --git a/rero_ils/modules/notifications/subclasses/acq_order.py b/rero_ils/modules/notifications/subclasses/acq_order.py index c6ee2f2e5e..bf5ca0bcb3 100644 --- a/rero_ils/modules/notifications/subclasses/acq_order.py +++ b/rero_ils/modules/notifications/subclasses/acq_order.py @@ -148,10 +148,9 @@ def get_notification_context(cls, notifications=None): the required data to build the template. :return: a ``dict`` containing all required data to build the template. """ - context = {} notifications = notifications or [] if not notifications: - return context + return {} notification = notifications[0] order = notification.order diff --git a/rero_ils/modules/notifications/subclasses/at_desk.py b/rero_ils/modules/notifications/subclasses/at_desk.py index 38a9db17c8..838b24c32e 100644 --- a/rero_ils/modules/notifications/subclasses/at_desk.py +++ b/rero_ils/modules/notifications/subclasses/at_desk.py @@ -21,7 +21,7 @@ from __future__ import absolute_import, print_function from rero_ils.filter import format_date_filter -from rero_ils.modules.documents.dumpers import document_title +from rero_ils.modules.documents.dumpers import document_title_dumper from rero_ils.modules.items.dumpers import ItemNotificationDumper from rero_ils.modules.loans.models import LoanState from rero_ils.modules.locations.api import Location @@ -112,7 +112,8 @@ def get_notification_context(cls, notifications=None): ) # merge doc and item metadata preserving document key item_data = notification.item.dumps(dumper=item_dumper) - doc_data = notification.document.dumps(dumper=document_title) + doc_data = notification.document.dumps( + dumper=document_title_dumper) doc_data = {**item_data, **doc_data} # pickup location name --> !! pickup is on notif.request_loan, not # on notif.loan diff --git a/rero_ils/modules/notifications/subclasses/availability.py b/rero_ils/modules/notifications/subclasses/availability.py index 22bbba7064..ae5ce45e67 100644 --- a/rero_ils/modules/notifications/subclasses/availability.py +++ b/rero_ils/modules/notifications/subclasses/availability.py @@ -22,7 +22,7 @@ import ciso8601 -from rero_ils.modules.documents.dumpers import document_title +from rero_ils.modules.documents.dumpers import document_title_dumper from rero_ils.modules.items.dumpers import ItemNotificationDumper from rero_ils.modules.libraries.dumpers import \ LibraryCirculationNotificationDumper @@ -83,19 +83,19 @@ def get_notification_context(cls, notifications=None): include_address = notifications[0].get_communication_channel() == \ NotificationChannel.MAIL # Dump basic informations - context.update({ + context |= { 'include_patron_address': include_address, 'patron': patron.dumps(dumper=PatronNotificationDumper()), 'library': library.dumps( dumper=LibraryCirculationNotificationDumper()), 'loans': [], 'delay': 0 - }) + } # Availability notification could be sent with a delay. We need to find # this delay into the library notifications settings. for setting in library.get('notification_settings', []): if setting['type'] == NotificationType.AVAILABILITY: - context.update({'delay': setting.get('delay', 0)}) + context['delay'] = setting.get('delay', 0) # Add metadata for any ``notification.loan`` of the notifications list item_dumper = ItemNotificationDumper() for notification in notifications: @@ -112,7 +112,8 @@ def get_notification_context(cls, notifications=None): lib = notification.transaction_library # merge doc and item metadata preserving document key item_data = notification.item.dumps(dumper=item_dumper) - doc_data = notification.document.dumps(dumper=document_title) + doc_data = notification.document.dumps( + dumper=document_title_dumper) doc_data = {**item_data, **doc_data} if loc and lib: context['loans'].append({ diff --git a/rero_ils/modules/notifications/subclasses/booking.py b/rero_ils/modules/notifications/subclasses/booking.py index f6b010ac21..66ac168ab6 100644 --- a/rero_ils/modules/notifications/subclasses/booking.py +++ b/rero_ils/modules/notifications/subclasses/booking.py @@ -23,7 +23,7 @@ import hashlib from rero_ils.filter import format_date_filter -from rero_ils.modules.documents.dumpers import document_title +from rero_ils.modules.documents.dumpers import document_title_dumper from rero_ils.modules.items.dumpers import ItemNotificationDumper from rero_ils.modules.loans.api import LoanState from rero_ils.modules.locations.api import Location @@ -92,7 +92,8 @@ def get_notification_context(cls, notifications=None): ) # merge doc and item metadata preserving document key item_data = notification.item.dumps(dumper=item_dumper) - doc_data = notification.document.dumps(dumper=document_title) + doc_data = notification.document.dumps( + dumper=document_title_dumper) doc_data = {**item_data, **doc_data} # pickup location name --> !! pickup is on notif.request_loan, not # on notif.loan diff --git a/rero_ils/modules/notifications/subclasses/internal.py b/rero_ils/modules/notifications/subclasses/internal.py index 532dc72133..ed560d1e27 100644 --- a/rero_ils/modules/notifications/subclasses/internal.py +++ b/rero_ils/modules/notifications/subclasses/internal.py @@ -69,7 +69,7 @@ def can_be_cancelled(self): @property def aggregation_key(self): """Get the aggregation key for this notification.""" - # Internal notifications must be send to a library. No need to + # Internal notifications must be sent to a library. No need to # take care of the requested patron for these notifications. parts = [ self.get_template_path(), @@ -90,6 +90,5 @@ def get_recipients_to(self): """Get notification recipient email addresses.""" # Internal notification will be sent to the library, not to the # patron related to the loan. - recipient = self.library.get_email(self.type) - if recipient: + if recipient := self.library.get_email(self.type): return [recipient] diff --git a/rero_ils/modules/notifications/subclasses/recall.py b/rero_ils/modules/notifications/subclasses/recall.py index 592095ec59..ad272cdaaf 100644 --- a/rero_ils/modules/notifications/subclasses/recall.py +++ b/rero_ils/modules/notifications/subclasses/recall.py @@ -22,7 +22,7 @@ import ciso8601 -from rero_ils.modules.documents.dumpers import document_title +from rero_ils.modules.documents.dumpers import document_title_dumper from rero_ils.modules.items.models import ItemStatus from rero_ils.modules.libraries.dumpers import \ LibraryCirculationNotificationDumper @@ -77,13 +77,13 @@ def get_notification_context(cls, notifications=None): include_address = notifications[0].get_communication_channel == \ NotificationChannel.MAIL # Dump basic informations - context.update({ + context |= { 'include_patron_address': include_address, 'patron': patron.dumps(dumper=PatronNotificationDumper()), 'library': library.dumps( dumper=LibraryCirculationNotificationDumper()), 'loans': [] - }) + } # Add metadata for any ``notification.loan`` of the notifications list for notification in notifications: end_date = notification.loan.get('end_date') @@ -91,7 +91,8 @@ def get_notification_context(cls, notifications=None): end_date = ciso8601.parse_datetime(end_date) end_date = end_date.strftime("%d.%m.%Y") context['loans'].append({ - 'document': notification.document.dumps(dumper=document_title), + 'document': notification.document.dumps( + dumper=document_title_dumper), 'end_date': end_date }) return context diff --git a/rero_ils/modules/notifications/subclasses/reminder.py b/rero_ils/modules/notifications/subclasses/reminder.py index d08f991532..65eea6360a 100644 --- a/rero_ils/modules/notifications/subclasses/reminder.py +++ b/rero_ils/modules/notifications/subclasses/reminder.py @@ -25,7 +25,7 @@ from rero_ils.modules.circ_policies.api import DUE_SOON_REMINDER_TYPE, \ OVERDUE_REMINDER_TYPE -from rero_ils.modules.documents.dumpers import document_title +from rero_ils.modules.documents.dumpers import document_title_dumper from rero_ils.modules.libraries.dumpers import \ LibraryCirculationNotificationDumper from rero_ils.modules.loans.utils import get_circ_policy @@ -154,13 +154,13 @@ def get_notification_context(cls, notifications=None): include_address = notifications[0].get_communication_channel == \ NotificationChannel.MAIL # Dump basic informations - context.update({ + context |= { 'include_patron_address': include_address, 'patron': patron.dumps(dumper=PatronNotificationDumper()), 'library': library.dumps( dumper=LibraryCirculationNotificationDumper()), 'loans': [] - }) + } # Add metadata for any ``notification.loan`` of the notifications list item_dumper = ItemNotificationDumper() language = language_iso639_2to1(notifications[0].get_language_to_use()) @@ -175,7 +175,8 @@ def get_notification_context(cls, notifications=None): end_date = end_date.strftime("%d.%m.%Y") # merge doc and item metadata preserving document key item_data = notification.item.dumps(dumper=item_dumper) - doc_data = notification.document.dumps(dumper=document_title) + doc_data = notification.document.dumps( + dumper=document_title_dumper) doc_data = {**item_data, **doc_data} context['loans'].append({ 'document': doc_data, diff --git a/rero_ils/modules/notifications/subclasses/request.py b/rero_ils/modules/notifications/subclasses/request.py index 19cfe8dcd0..519a0bd45b 100644 --- a/rero_ils/modules/notifications/subclasses/request.py +++ b/rero_ils/modules/notifications/subclasses/request.py @@ -21,7 +21,7 @@ from __future__ import absolute_import, print_function from rero_ils.filter import format_date_filter -from rero_ils.modules.documents.dumpers import document_title +from rero_ils.modules.documents.dumpers import document_title_dumper from rero_ils.modules.items.dumpers import ItemNotificationDumper from rero_ils.modules.loans.api import LoanState from rero_ils.modules.patrons.dumpers import PatronNotificationDumper @@ -68,7 +68,8 @@ def get_notification_context(cls, notifications=None): ) # merge doc and item metadata preserving document key item_data = notification.item.dumps(dumper=item_dumper) - doc_data = notification.document.dumps(dumper=document_title) + doc_data = notification.document.dumps( + dumper=document_title_dumper) doc_data = {**item_data, **doc_data} # pickup location name pickup_location = notification.pickup_location diff --git a/rero_ils/modules/notifications/subclasses/transit.py b/rero_ils/modules/notifications/subclasses/transit.py index 16aa23865f..ef5415b4a1 100644 --- a/rero_ils/modules/notifications/subclasses/transit.py +++ b/rero_ils/modules/notifications/subclasses/transit.py @@ -21,7 +21,7 @@ from __future__ import absolute_import, print_function from rero_ils.filter import format_date_filter -from rero_ils.modules.documents.dumpers import document_title +from rero_ils.modules.documents.dumpers import document_title_dumper from rero_ils.modules.items.dumpers import ItemNotificationDumper from rero_ils.modules.libraries.dumpers import \ LibraryCirculationNotificationDumper @@ -50,8 +50,7 @@ def get_template_path(self): def get_recipients_to(self): """Get notification recipient email addresses.""" # Transit notification will be sent to the loan transaction library. - recipient = self.transaction_library.get_email(self.type) - if recipient: + if recipient := self.transaction_library.get_email(self.type): return [recipient] @classmethod @@ -70,7 +69,8 @@ def get_notification_context(cls, notifications=None): ) # merge doc and item metadata preserving document key item_data = notification.item.dumps(dumper=item_dumper) - doc_data = notification.document.dumps(dumper=document_title) + doc_data = notification.document.dumps( + dumper=document_title_dumper) doc_data = {**item_data, **doc_data} loan_context = { diff --git a/rero_ils/modules/patrons/views.py b/rero_ils/modules/patrons/views.py index 54fe17d702..ae3ab05e81 100644 --- a/rero_ils/modules/patrons/views.py +++ b/rero_ils/modules/patrons/views.py @@ -127,12 +127,9 @@ def logged_user(): 'language': current_i18n.locale.language, 'globalView': config.get('RERO_ILS_SEARCH_GLOBAL_VIEW_CODE'), 'baseUrl': get_base_url(), - 'contributionAgentTypes': config.get( - 'RERO_ILS_CONTRIBUTIONS_AGENT_TYPES', {}), - 'contributionsLabelOrder': config.get( - 'RERO_ILS_CONTRIBUTIONS_LABEL_ORDER', {}), - 'contributionSources': config.get( - 'RERO_ILS_CONTRIBUTIONS_SOURCES', []), + 'agentAgentTypes': config.get('RERO_ILS_AGENTS_AGENT_TYPES', {}), + 'agentLabelOrder': config.get('RERO_ILS_AGENTS_LABEL_ORDER', {}), + 'agentSources': config.get('RERO_ILS_AGENTS_SOURCES', []), 'operationLogs': config.get('RERO_ILS_ENABLE_OPERATION_LOG', []), 'userProfile': { 'readOnly': config.get( diff --git a/rero_ils/modules/utils.py b/rero_ils/modules/utils.py index 16ddfd0703..113159dc73 100644 --- a/rero_ils/modules/utils.py +++ b/rero_ils/modules/utils.py @@ -390,7 +390,7 @@ def get_schema_for_resource(resource): USAGE: schema = get_schema_for_resource('ptrn') - shcema = get_schema_for_resource(Patron) + schema = get_schema_for_resource(Patron) """ if not isinstance(resource, str): resource = resource.provider.pid_type @@ -556,7 +556,7 @@ def set_timestamp(name, **kwargs): """Set timestamp in current cache. Allows to timestamp functionality and monitoring of the changed - timestamps externaly via url requests. + timestamps externally via url requests. :param name: name of time stamp. :returns: time of time stamp diff --git a/rero_ils/query.py b/rero_ils/query.py index 1f3435da29..c68147d71e 100644 --- a/rero_ils/query.py +++ b/rero_ils/query.py @@ -180,7 +180,7 @@ def search_factory_for_holdings_and_items(view, search): return search -def contribution_view_search_factory(self, search, query_parser=None): +def entity_view_search_factory(self, search, query_parser=None): """Search factory with view code parameter.""" view = request.args.get( 'view', current_app.config.get('RERO_ILS_SEARCH_GLOBAL_VIEW_CODE')) diff --git a/scripts/setup b/scripts/setup index 43f6e0b14f..bc877518c9 100755 --- a/scripts/setup +++ b/scripts/setup @@ -78,7 +78,7 @@ invert_warning_option() { DEPLOYMENT=false -LOADCONTRIBUTIONS=false +LOADENTITIES=false CREATE_ITEMS_HOLDINGS_SMALL=false CREATE_ITEMS_HOLDINGS_BIG=false STOP_EXECUTION=true @@ -101,7 +101,7 @@ if [[ -z "${VIRTUAL_ENV}" ]]; then fi # options may be followed by one colon to indicate they have a required argument -if ! options=$(getopt -o dCsbclptmwkD:i: -l deployment,contributions,create_items_holdings_small,create_items_holdings_big,lazy,pursue,time,es-mapping,warnings,enqueue,data_path:,index_parallel: -- "$@") +if ! options=$(getopt -o dCsbclptmwkD:i: -l deployment,entities,create_items_holdings_small,create_items_holdings_big,lazy,pursue,time,es-mapping,warnings,enqueue,data_path:,index_parallel: -- "$@") then # something went wrong, getopt will put out an error message for us exit 1 @@ -116,8 +116,8 @@ do -L|--load_extra_files) LOADEXTRAFILES=true ;; - -C|--contributions) - LOADCONTRIBUTIONS=true + -E|--entities) + LOADENTITIES=true ;; -s|--create_items_holdings_small) CREATE_ITEMS_HOLDINGS_SMALL=true @@ -333,34 +333,34 @@ eval ${PREFIX} invenio reroils index reindex -t illr --yes-i-know # SIZE=big # SIZE=small # export RERO_ILS_MEF_AGENTS_URL=https://mef.rero.ch/api/agents # invenio reroils utils marc21tojson -t rero ${DATA_PATH}/documents_${SIZE}.xml ${DATA_PATH}/documents_${SIZE}.json ${DATA_PATH}/documents_${SIZE}_errors.xml -v -r -# Save the contribution after setup for later use. -# invenio reroils utils export -t cont -o ${DATA_PATH}/contributions_${SIZE}.json -v +# Save the entities after setup for later use. +# invenio reroils utils export -t ent -o ${DATA_PATH}/entities_${SIZE}.json -v if ${DEPLOYMENT} then DOCUMENTS=${DATA_PATH}/documents_big.json ITEMS=${DATA_PATH}/items_big.json HOLDINGS=${DATA_PATH}/holdings_big.json - CONTRIBUTIONS=${DATA_PATH}/contributions_big.json + ENTITIES=${DATA_PATH}/entities_big.json else DOCUMENTS=${DATA_PATH}/documents_small.json ITEMS=${DATA_PATH}/items_small.json HOLDINGS=${DATA_PATH}/holdings_small.json - CONTRIBUTIONS=${DATA_PATH}/contributions_small.json + ENTITIES=${DATA_PATH}/entities_small.json fi if ${LOADCONTRIBUTIONS} then - info_msg "- CONTRIBUTIONS: ${CONTRIBUTIONS} ${CREATE_LAZY} ${DONT_STOP}" - eval ${PREFIX} invenio reroils fixtures create --pid_type cont --schema 'https://bib.rero.ch/schemas/contributions/contribution-v0.0.1.json' ${CONTRIBUTIONS} --append ${CREATE_LAZY} ${DONT_STOP} - info_msg "Indexing Contributions:" - eval ${PREFIX} invenio reroils index reindex -t cont --yes-i-know + info_msg "- ENTITIES: ${ENTITIES} ${CREATE_LAZY} ${DONT_STOP}" + eval ${PREFIX} invenio reroils fixtures create --pid_type ent --schema 'https://bib.rero.ch/schemas/entities/entity-v0.0.1.json' ${ENTITIES} --append ${CREATE_LAZY} ${DONT_STOP} + info_msg "Indexing Entities:" + eval ${PREFIX} invenio reroils index reindex -t ent --yes-i-know if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio reroils index run -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio reroils index run --raise-on-error if [ ${INDEX_PARALLEL} -gt 0 ]; then - eval ${PREFIX} invenio reroils index reindex_missing -t cont -v + eval ${PREFIX} invenio reroils index reindex_missing -t ent -v fi # else # info_msg "- Contributions from MEF: ${DOCUMENTS} ${CREATE_LAZY} ${ENQUEUE}" @@ -431,15 +431,15 @@ if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio reroils index reindex_missing -t doc -v fi -# index contributions -# We have to reindex contributions to get the oraganisations pids indexed correctly. -eval ${PREFIX} invenio reroils index reindex -t cont --yes-i-know +# index entities +# We have to reindex entities to get the organisations pids indexed correctly. +eval ${PREFIX} invenio reroils index reindex -t ent --yes-i-know if [ ${INDEX_PARALLEL} -gt 0 ]; then eval ${PREFIX} invenio reroils index run -d -c ${INDEX_PARALLEL} --raise-on-error fi eval ${PREFIX} invenio reroils index run --raise-on-error if [ ${INDEX_PARALLEL} -gt 0 ]; then - eval ${PREFIX} invenio reroils index reindex_missing -t cont -v + eval ${PREFIX} invenio reroils index reindex_missing -t ent -v fi info_msg "- Local fields ${DATA_PATH}/local_fields.json ${CREATE_LAZY} ${DONT_STOP}" diff --git a/setup.py b/setup.py index b84a179540..df1df89194 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ def run(self): ], 'invenio_base.blueprints': [ 'collections = rero_ils.modules.collections.views:blueprint', - 'contributions = rero_ils.modules.contributions.views:blueprint', + 'entities = rero_ils.modules.entities.views:blueprint', 'documents = rero_ils.modules.documents.views:blueprint', 'holdings = rero_ils.modules.holdings.views:blueprint', 'ill_requests = rero_ils.modules.ill_requests.views:blueprint', @@ -107,7 +107,7 @@ def run(self): 'acq_receipts = rero_ils.modules.acquisition.acq_receipts.views:api_blueprint', 'api_documents = rero_ils.modules.documents.views:api_blueprint', 'circ_policies = rero_ils.modules.circ_policies.views:blueprint', - 'contributions = rero_ils.modules.contributions.views:api_blueprint', + 'entities = rero_ils.modules.entities.views:api_blueprint', 'holdings = rero_ils.modules.holdings.api_views:api_blueprint', 'item_types = rero_ils.modules.item_types.views:blueprint', 'items = rero_ils.modules.items.views:api_blueprint', @@ -149,7 +149,7 @@ def run(self): ], 'flask.commands': [ 'apiharvester = rero_ils.modules.apiharvester.cli:apiharvester', - 'contribution = rero_ils.modules.contributions.cli:contribution', + 'entity = rero_ils.modules.entities.cli:entity', 'monitoring = rero_ils.modules.monitoring.cli:monitoring', 'notifications = rero_ils.modules.notifications.cli:notifications', 'oaiharvester = rero_ils.modules.ebooks.cli:oaiharvester', @@ -179,7 +179,7 @@ def run(self): 'libraries = rero_ils.modules.libraries.models', 'local_fields = rero_ils.modules.local_fields.models', 'locations = rero_ils.modules.locations.models', - 'mef = rero_ils.modules.contributions.models', + 'entity = rero_ils.modules.entities.models', 'notifications = rero_ils.modules.notifications.models', 'organisations = rero_ils.modules.organisations.models', 'patron_transaction_events = rero_ils.modules.patron_transaction_events.models', @@ -201,7 +201,7 @@ def run(self): 'budget_id = rero_ils.modules.acquisition.budgets.api:budget_id_minter', 'circ_policy_id = rero_ils.modules.circ_policies.api:circ_policy_id_minter', 'collection_id = rero_ils.modules.collections.api:collection_id_minter', - 'contribution_id = rero_ils.modules.contributions.api:contribution_id_minter', + 'entity_id = rero_ils.modules.entities.api:entity_id_minter', 'document_id = rero_ils.modules.documents.api:document_id_minter', 'holding_id = rero_ils.modules.holdings.api:holding_id_minter', 'ill_request_id = rero_ils.modules.ill_requests.api:ill_request_id_minter', @@ -231,7 +231,7 @@ def run(self): 'circ_policy_id = rero_ils.modules.circ_policies.api:circ_policy_id_fetcher', 'collection_id = rero_ils.modules.collections.api:collection_id_fetcher', 'document_id = rero_ils.modules.documents.api:document_id_fetcher', - 'contribution_id = rero_ils.modules.contributions.api:contribution_id_fetcher', + 'entity_id = rero_ils.modules.entities.api:entity_id_fetcher', 'holding_id = rero_ils.modules.holdings.api:holding_id_fetcher', 'ill_request_id = rero_ils.modules.ill_requests.api:ill_request_id_fetcher', 'item_id = rero_ils.modules.items.api:item_id_fetcher', @@ -261,7 +261,7 @@ def run(self): 'circ_policies = rero_ils.modules.circ_policies.jsonschemas', 'collections = rero_ils.modules.collections.jsonschemas', 'common = rero_ils.jsonschemas', - 'contributions = rero_ils.modules.contributions.jsonschemas', + 'entities = rero_ils.modules.entities.jsonschemas', 'documents = rero_ils.modules.documents.jsonschemas', 'holdings = rero_ils.modules.holdings.jsonschemas', 'ill_requests = rero_ils.modules.ill_requests.jsonschemas', @@ -293,7 +293,7 @@ def run(self): 'budgets = rero_ils.modules.acquisition.budgets.mappings', 'circ_policies = rero_ils.modules.circ_policies.mappings', 'collections = rero_ils.modules.collections.mappings', - 'contributions = rero_ils.modules.contributions.mappings', + 'entities = rero_ils.modules.entities.mappings', 'documents = rero_ils.modules.documents.mappings', 'holdings = rero_ils.modules.holdings.mappings', 'ill_requests = rero_ils.modules.ill_requests.mappings', @@ -339,7 +339,7 @@ def run(self): 'acq_receipt_lines = rero_ils.modules.acquisition.acq_receipt_lines.jsonresolver', 'budgets = rero_ils.modules.acquisition.budgets.jsonresolver', 'collections = rero_ils.modules.collections.jsonresolver', - 'contributions = rero_ils.modules.contributions.jsonresolver', + 'entities = rero_ils.modules.entities.jsonresolver', 'documents = rero_ils.modules.documents.jsonresolver', 'holdings = rero_ils.modules.holdings.jsonresolver', 'ill_requests = rero_ils.modules.ill_requests.jsonresolver', diff --git a/tests/api/test_dumpers.py b/tests/api/acquisition/test_acquisition_dumpers.py similarity index 53% rename from tests/api/test_dumpers.py rename to tests/api/acquisition/test_acquisition_dumpers.py index 129517740d..83dd4b47d2 100644 --- a/tests/api/test_dumpers.py +++ b/tests/api/acquisition/test_acquisition_dumpers.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2021 RERO -# Copyright (C) 2021 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -16,13 +16,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Tests dumpers from RERO-ILS projects.""" +"""Test library dumpers.""" from rero_ils.modules.acquisition.acq_orders.dumpers import \ AcqOrderNotificationDumper -from rero_ils.modules.acquisition.dumpers import document_acquisition_dumper -from rero_ils.modules.documents.dumpers import document_title -from rero_ils.modules.libraries.dumpers import \ - LibraryAcquisitionNotificationDumper def test_acquisition_dumpers( @@ -41,33 +37,3 @@ def test_acquisition_dumpers( assert dump_data['library']['shipping_informations'] assert dump_data['library']['billing_informations'] assert dump_data['vendor'] - - -def test_library_dumpers(lib_martigny, lib_saxon): - """Test library dumpers.""" - - dump_data = lib_martigny.dumps( - dumper=LibraryAcquisitionNotificationDumper()) - assert dump_data['shipping_informations'] - assert dump_data['billing_informations'] - - dump_data = lib_saxon.dumps( - dumper=LibraryAcquisitionNotificationDumper()) - assert dump_data['shipping_informations'] - assert 'billing_informations' not in dump_data - - -def test_document_dumpers(document): - """Test document dumpers.""" - dump_data = document.dumps( - dumper=document_title - ) - assert dump_data['pid'] - assert dump_data['title_text'] - - dump_data = document.dumps( - dumper=document_acquisition_dumper - ) - assert dump_data['pid'] - assert dump_data['title_text'] - assert dump_data['identifiers'] diff --git a/tests/api/documents/test_documents_dumpers.py b/tests/api/documents/test_documents_dumpers.py new file mode 100644 index 0000000000..9091dad16e --- /dev/null +++ b/tests/api/documents/test_documents_dumpers.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Test document dumpers.""" +import pytest + +from rero_ils.modules.acquisition.dumpers import document_acquisition_dumper +from rero_ils.modules.commons.exceptions import RecordNotFound +from rero_ils.modules.documents.api import Document +from rero_ils.modules.documents.dumpers import document_replace_refs_dumper, \ + document_title_dumper +from rero_ils.modules.documents.models import DocumentSubjectType + + +def test_document_dumpers(document, document_data): + """Test document dumpers.""" + dump_data = document.dumps(dumper=document_title_dumper) + assert dump_data['pid'] + assert dump_data['title_text'] + + dump_data = document.dumps(dumper=document_acquisition_dumper) + assert dump_data['pid'] + assert dump_data['title_text'] + assert dump_data['identifiers'] + + document['contribution'] = [{'entity': {'$ref': 'n/a', 'pid': 'n/a'}}] + with pytest.raises(RecordNotFound): + document.dumps(dumper=document_replace_refs_dumper) + document['contribution'] = document_data['contribution'] + document['subjects'] = [{ + '$ref': 'n/a', + 'pid': 'n/a', + 'type': DocumentSubjectType.PERSON + }] + with pytest.raises(RecordNotFound): + document.dumps(dumper=document_replace_refs_dumper) + + +@pytest.mark.skip(reason="Dumper() not implement 'load()' method") +def test_multi_dumpers(document_data): + """Test MultiDumper.""" + # TODO :: Try to fix this test. + document_title_dumper.load(document_data, Document) diff --git a/tests/api/documents/test_documents_rest.py b/tests/api/documents/test_documents_rest.py index 10a70ef432..97a7d50ae6 100644 --- a/tests/api/documents/test_documents_rest.py +++ b/tests/api/documents/test_documents_rest.py @@ -405,7 +405,7 @@ def test_documents_post_put_delete( def test_documents_get_resolve_rero_json( - client, document_ref, contribution_person_data, rero_json_header, + client, document_ref, entity_person_data, rero_json_header, ): """Test record get with resolve and mimetype rero+json.""" api_url = url_for('invenio_records_rest.doc_item', pid_value='doc2', @@ -414,7 +414,7 @@ def test_documents_get_resolve_rero_json( assert res.status_code == 200 metadata = get_json(res).get('metadata', {}) pid = metadata['contribution'][0]['entity']['pid'] - assert pid == contribution_person_data['pid'] + assert pid == entity_person_data['pid'] def test_document_can_request_view( @@ -488,8 +488,8 @@ def test_document_boosting(client, ebook_1, ebook_4): @mock.patch('requests.get') def test_documents_resolve( - mock_contributions_mef_get, client, mef_agents_url, - loc_public_martigny, document_ref, contribution_person_response_data + mock_contributions_mef_get, client, mef_agents_url,loc_public_martigny, document_ref, + entity_person_response_data ): """Test document detailed view with items filter.""" res = client.get(url_for( @@ -499,7 +499,7 @@ def test_documents_resolve( assert res.json['metadata']['contribution'] == [{ 'entity': { '$ref': f'{mef_agents_url}/rero/A017671081', - 'pid': 'cont_pers', + 'pid': 'ent_pers', 'type': 'bf:Person' }, 'role': ['aut'] @@ -507,7 +507,7 @@ def test_documents_resolve( assert res.status_code == 200 mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data + json_data=entity_person_response_data ) res = client.get(url_for( 'invenio_records_rest.doc_item', diff --git a/tests/api/documents/test_documents_tasks.py b/tests/api/documents/test_documents_tasks.py index f7d7fff221..920132c766 100644 --- a/tests/api/documents/test_documents_tasks.py +++ b/tests/api/documents/test_documents_tasks.py @@ -24,18 +24,18 @@ import mock from utils import mock_response -from rero_ils.modules.contributions.api import Contribution from rero_ils.modules.documents.api import Document, DocumentsSearch from rero_ils.modules.documents.tasks import replace_idby_contribution, \ replace_idby_subjects from rero_ils.modules.documents.utils_mef import \ ReplaceMefIdentifiedByContribution, ReplaceMefIdentifiedBySubjects +from rero_ils.modules.entities.api import Entity @mock.patch('requests.get') def test_replace_idby_contribution(mock_contributions_mef_get, app, document_data, - contribution_person_response_data): + entity_person_response_data): """Test replace identifiedBy in contribution.""" assert replace_idby_contribution() == (0, 0, 0, 0, 0) @@ -48,7 +48,7 @@ def test_replace_idby_contribution(mock_contributions_mef_get, app, replace.process() assert replace.counts_len == (0, 0, 0, 0, 1) - without_idref_gnd = deepcopy(contribution_person_response_data) + without_idref_gnd = deepcopy(entity_person_response_data) without_idref_gnd['hits']['hits'][0]['metadata'].pop('idref') without_idref_gnd['hits']['hits'][0]['metadata'].pop('gnd') mock_contributions_mef_get.return_value = mock_response( @@ -56,7 +56,7 @@ def test_replace_idby_contribution(mock_contributions_mef_get, app, ) assert replace_idby_contribution() == (0, 0, 0, 1, 0) - without_idref_gnd = deepcopy(contribution_person_response_data) + without_idref_gnd = deepcopy(entity_person_response_data) without_idref_gnd['hits']['hits'][0]['metadata']['deleted'] = '2022' mock_contributions_mef_get.return_value = mock_response( json_data=without_idref_gnd @@ -64,21 +64,21 @@ def test_replace_idby_contribution(mock_contributions_mef_get, app, assert replace_idby_contribution() == (0, 0, 1, 0, 0) mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data + json_data=entity_person_response_data ) assert replace_idby_contribution() == (1, 0, 0, 0, 0) # clean up doc.delete(dbcommit=True, delindex=True, force=True) - for id in Contribution.get_all_ids(): - cont = Contribution.get_record(id) + for id in Entity.get_all_ids(): + cont = Entity.get_record(id) cont.delete(dbcommit=True, delindex=True, force=True) @mock.patch('requests.get') def test_replace_idby_subjects(mock_contributions_mef_get, app, document_data, - contribution_person_response_data): + entity_person_response_data): """Test replace identifiedBy in subjects.""" assert replace_idby_subjects() == (0, 0, 0, 0, 0) @@ -88,7 +88,7 @@ def test_replace_idby_subjects(mock_contributions_mef_get, app, replace.process() assert replace.counts_len == (0, 0, 0, 0, 1) - without_idref_gnd = deepcopy(contribution_person_response_data) + without_idref_gnd = deepcopy(entity_person_response_data) without_idref_gnd['hits']['hits'][0]['metadata'].pop('idref') without_idref_gnd['hits']['hits'][0]['metadata'].pop('gnd') mock_contributions_mef_get.return_value = mock_response( @@ -96,7 +96,7 @@ def test_replace_idby_subjects(mock_contributions_mef_get, app, ) assert replace_idby_subjects() == (0, 0, 0, 1, 0) - without_idref_gnd = deepcopy(contribution_person_response_data) + without_idref_gnd = deepcopy(entity_person_response_data) without_idref_gnd['hits']['hits'][0]['metadata']['deleted'] = '2022' mock_contributions_mef_get.return_value = mock_response( json_data=without_idref_gnd @@ -104,12 +104,12 @@ def test_replace_idby_subjects(mock_contributions_mef_get, app, assert replace_idby_subjects() == (0, 0, 1, 0, 0) mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data + json_data=entity_person_response_data ) assert replace_idby_subjects() == (1, 0, 0, 0, 0) # clean up doc.delete(dbcommit=True, delindex=True, force=True) - for id in Contribution.get_all_ids(): - cont = Contribution.get_record(id) + for id in Entity.get_all_ids(): + cont = Entity.get_record(id) cont.delete(dbcommit=True, delindex=True, force=True) diff --git a/tests/api/contributions/test_contributions_permissions.py b/tests/api/entities/test_entities_permissions.py similarity index 68% rename from tests/api/contributions/test_contributions_permissions.py rename to tests/api/entities/test_entities_permissions.py index 812ea6e44f..eda3df34ef 100644 --- a/tests/api/contributions/test_contributions_permissions.py +++ b/tests/api/entities/test_entities_permissions.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2022 RERO -# Copyright (C) 2022 UCLouvain +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -22,22 +22,21 @@ from invenio_accounts.testutils import login_user_via_session from utils import check_permission, get_json -from rero_ils.modules.contributions.permissions import \ - ContributionPermissionPolicy +from rero_ils.modules.entities.permissions import EntityPermissionPolicy -def test_contribution_permissions_api(client, patron_martigny, - contribution_person, - librarian_martigny): - """Test organisations permissions api.""" +def test_entity_permissions_api(client, patron_martigny, + entity_person, + librarian_martigny): + """Test entities permissions api.""" prs_permissions_url = url_for( 'api_blueprint.permissions', - route_name='contributions' + route_name='entities' ) prs_real_permission_url = url_for( 'api_blueprint.permissions', - route_name='contributions', - record_pid=contribution_person.pid + route_name='entities', + record_pid=entity_person.pid ) # Not logged @@ -61,15 +60,15 @@ def test_contribution_permissions_api(client, patron_martigny, assert not data['delete']['can'] -def test_contribution_permissions(patron_martigny, - librarian_martigny, - system_librarian_martigny): - """Test contribution permissions class.""" - permission_policy = ContributionPermissionPolicy +def test_entity_permissions(patron_martigny, + librarian_martigny, + system_librarian_martigny): + """Test entity permissions class.""" + permission_policy = EntityPermissionPolicy # Anonymous user - # - Allow search/read actions on any contribution - # - Deny create/update/delete actions on any contribution + # - Allow search/read actions on any entity + # - Deny create/update/delete actions on any entity identity_changed.send( current_app._get_current_object(), identity=AnonymousIdentity() ) @@ -81,8 +80,8 @@ def test_contribution_permissions(patron_martigny, 'delete': False }, {}) # Patron user - # - Allow search/read actions on any contribution - # - Deny create/update/delete actions on any contribution + # - Allow search/read actions on any entity + # - Deny create/update/delete actions on any entity login_user(patron_martigny.user) check_permission(permission_policy, { 'search': True, @@ -92,8 +91,8 @@ def test_contribution_permissions(patron_martigny, 'delete': False }, {}) # Full permission user - # - Allow search/read actions on any contribution - # - Deny create/update/delete actions on any contribution + # - Allow search/read actions on any entity + # - Deny create/update/delete actions on any entity login_user(system_librarian_martigny.user) check_permission(permission_policy, { 'search': True, diff --git a/tests/api/contributions/test_contributions_rest.py b/tests/api/entities/test_entities_rest.py similarity index 51% rename from tests/api/contributions/test_contributions_rest.py rename to tests/api/entities/test_entities_rest.py index c782a88b63..945ba3f0d6 100644 --- a/tests/api/contributions/test_contributions_rest.py +++ b/tests/api/entities/test_entities_rest.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,33 +16,25 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Tests REST API contributions.""" - -# import json -# from utils import get_json, to_relative_url +"""Tests `Entity` resource REST API.""" from flask import url_for from utils import get_json, postdata, to_relative_url -from rero_ils.modules.contributions.models import ContributionType +from rero_ils.modules.entities.models import EntityType -def test_contributions_permissions(client, contribution_person, json_header): +def test_entities_permissions(client, entity_person, json_header): """Test record retrieval.""" - item_url = url_for('invenio_records_rest.cont_item', pid_value='cont_pers') - + item_url = url_for('invenio_records_rest.ent_item', pid_value='ent_pers') res = client.get(item_url) assert res.status_code == 200 - res, _ = postdata( - client, - 'invenio_records_rest.cont_list', - {} - ) + res, _ = postdata(client, 'invenio_records_rest.ent_list', {}) assert res.status_code == 401 - res = client.put( - url_for('invenio_records_rest.cont_item', pid_value='cont_pers'), + client.put( + url_for('invenio_records_rest.ent_item', pid_value='ent_pers'), data={}, headers=json_header ) @@ -50,19 +43,17 @@ def test_contributions_permissions(client, contribution_person, json_header): assert res.status_code == 401 -def test_contributions_get(client, contribution_person): +def test_entities_get(client, entity_person): """Test record retrieval.""" - item_url = url_for('invenio_records_rest.cont_item', pid_value='cont_pers') + item_url = url_for('invenio_records_rest.ent_item', pid_value='ent_pers') res = client.get(item_url) assert res.status_code == 200 - - assert res.headers['ETag'] == '"{}"'.format( - contribution_person.revision_id) + assert res.headers['ETag'] == f'"{entity_person.revision_id}"' data = get_json(res) - assert contribution_person.dumps() == data['metadata'] - assert contribution_person.dumps() == data['metadata'] + assert entity_person.dumps() == data['metadata'] + assert entity_person.dumps() == data['metadata'] # Check metadata for k in ['created', 'updated', 'metadata', 'links']: @@ -72,17 +63,14 @@ def test_contributions_get(client, contribution_person): res = client.get(to_relative_url(data['links']['self'])) assert res.status_code == 200 assert data == get_json(res) - assert contribution_person.dumps() == data['metadata'] + assert entity_person.dumps() == data['metadata'] - list_url = url_for('invenio_records_rest.cont_list', pid='cont_pers') + list_url = url_for('invenio_records_rest.ent_list', pid='ent_pers') res = client.get(list_url) assert res.status_code == 200 data = get_json(res) - contribution_person = contribution_person.replace_refs() - contribution_person['organisations'] = \ - contribution_person.organisation_pids - contribution_person['type'] = ContributionType.PERSON - contribution_person['type'] = ContributionType.PERSON - - assert data['hits']['hits'][0]['metadata'] == \ - contribution_person.replace_refs() + entity_person = entity_person.replace_refs() + entity_person['organisations'] = entity_person.organisation_pids + entity_person['type'] = EntityType.PERSON + entity_person['type'] = EntityType.PERSON + assert data['hits']['hits'][0]['metadata'] == entity_person.replace_refs() diff --git a/tests/api/libraries/test_libraries_dumpers.py b/tests/api/libraries/test_libraries_dumpers.py new file mode 100644 index 0000000000..8d45083b26 --- /dev/null +++ b/tests/api/libraries/test_libraries_dumpers.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Test library dumpers.""" + +from rero_ils.modules.libraries.dumpers import \ + LibraryAcquisitionNotificationDumper + + +def test_library_dumpers(lib_martigny, lib_saxon): + """Test library dumpers.""" + + dump_data = lib_martigny.dumps( + dumper=LibraryAcquisitionNotificationDumper()) + assert dump_data['shipping_informations'] + assert dump_data['billing_informations'] + + dump_data = lib_saxon.dumps( + dumper=LibraryAcquisitionNotificationDumper()) + assert dump_data['shipping_informations'] + assert 'billing_informations' not in dump_data diff --git a/tests/api/sru/test_sru_rest.py b/tests/api/sru/test_sru_rest.py index 7af2d1a9d1..ab0286b780 100644 --- a/tests/api/sru/test_sru_rest.py +++ b/tests/api/sru/test_sru_rest.py @@ -32,7 +32,7 @@ def test_sru_explain(client): assert 'sru:explainResponse' in xml_dict -def test_sru_documents(client, document_ref, contribution_person_data): +def test_sru_documents(client, document_ref, entity_person_data): """Test sru documents rest api.""" api_url = url_for('api_sru.documents', version='1.1', operation='searchRetrieve', diff --git a/tests/api/test_monitoring_rest.py b/tests/api/test_monitoring_rest.py index 33128ab26b..b71f01c8cf 100644 --- a/tests/api/test_monitoring_rest.py +++ b/tests/api/test_monitoring_rest.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -26,8 +27,7 @@ from invenio_db import db from utils import flush_index, get_json -from rero_ils.modules.contributions.api import Contribution, \ - ContributionsSearch +from rero_ils.modules.entities.api import EntitiesSearch, Entity from rero_ils.modules.utils import get_timestamp, set_timestamp @@ -47,7 +47,7 @@ def test_monitoring_es_db_counts(client): 'budg': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'budgets'}, 'cipo': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'circ_policies'}, 'coll': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'collections'}, - 'cont': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'contributions'}, + 'ent': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'entities'}, 'doc': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'documents'}, 'hold': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'holdings'}, 'illr': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'ill_requests'}, @@ -73,43 +73,41 @@ def test_monitoring_es_db_counts(client): } -def test_monitoring_check_es_db_counts(app, client, contribution_person_data, +def test_monitoring_check_es_db_counts(app, client, entity_person_data, system_librarian_martigny): """Test monitoring check_es_db_counts.""" res = client.get(url_for('api_monitoring.check_es_db_counts', delay=0)) assert res.status_code == 200 assert get_json(res) == {'data': {'status': 'green'}} - pers = Contribution.create( - data=contribution_person_data, + pers = Entity.create( + data=entity_person_data, delete_pid=False, dbcommit=True, reindex=False) - flush_index(ContributionsSearch.Meta.index) + flush_index(EntitiesSearch.Meta.index) res = client.get(url_for('api_monitoring.check_es_db_counts', delay=0)) assert res.status_code == 200 assert get_json(res) == { 'data': {'status': 'red'}, 'errors': [{ 'code': 'DB_ES_COUNTER_MISMATCH', - 'details': 'There are 1 items from cont missing in ES.', + 'details': 'There are 1 items from ent missing in ES.', 'id': 'DB_ES_COUNTER_MISMATCH', 'links': { 'about': 'http://localhost/monitoring/check_es_db_counts', - 'cont': 'http://localhost/monitoring/missing_pids/cont' + 'ent': 'http://localhost/monitoring/missing_pids/ent' }, 'title': "DB items counts don't match ES items count." }] } # this view is only accessible by monitoring - res = client.get(url_for('api_monitoring.missing_pids', doc_type='cont')) + res = client.get(url_for('api_monitoring.missing_pids', doc_type='ent')) assert res.status_code == 401 login_user_via_session(client, system_librarian_martigny.user) - res = client.get( - url_for('api_monitoring.missing_pids', doc_type='cont') - ) + res = client.get(url_for('api_monitoring.missing_pids', doc_type='ent')) assert res.status_code == 403 # give user superuser admin rights @@ -120,15 +118,13 @@ def test_monitoring_check_es_db_counts(app, client, contribution_person_data, ) ) db.session.commit() - res = client.get( - url_for('api_monitoring.missing_pids', doc_type='cont', delay=0) - ) + res = client.get(url_for( + 'api_monitoring.missing_pids', doc_type='ent', delay=0)) assert res.status_code == 200 - assert get_json(res) == { 'data': { 'DB': [], - 'ES': ['http://localhost/contributions/cont_pers'], + 'ES': ['http://localhost/entities/ent_pers'], 'ES duplicate': [] } } diff --git a/tests/data/data.json b/tests/data/data.json index 8b3494230f..6309b7b6b1 100644 --- a/tests/data/data.json +++ b/tests/data/data.json @@ -1489,8 +1489,8 @@ } ] }, - "cont_pers": { - "$schema": "https://bib.rero.ch/schemas/contributions/contribution-v0.0.1.json", + "ent_pers": { + "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", "gnd": { "$schema": "https://mef.test.rero.ch/schemas/gnd/gnd-contribution-v0.0.1.json", "bf:Agent": "bf:Person", @@ -1537,7 +1537,7 @@ "authorized_access_point": "Loy, Georg, 1885-19..", "preferred_name": "Loy, Georg" }, - "pid": "cont_pers", + "pid": "ent_pers", "type": "bf:Person", "sources": [ "gnd", @@ -1546,8 +1546,8 @@ ], "viaf_pid": "70119347" }, - "cont_org": { - "$schema": "https://bib.rero.ch/schemas/contributions/contribution-v0.0.1.json", + "ent_org": { + "$schema": "https://bib.rero.ch/schemas/entities/entity-v0.0.1.json", "gnd": { "$schema": "https://mef.rero.ch/schemas/gnd/gnd-contribution-v0.0.1.json", "bf:Agent": "bf:Organisation", @@ -1570,7 +1570,7 @@ "pid": "A027711299", "preferred_name": "Convegno internazionale di Italianistica" }, - "pid": "cont_org", + "pid": "ent_org", "type": "bf:Organisation", "sources": [ "rero", @@ -1578,7 +1578,7 @@ ], "viaf_pid": "3117153063217219320007" }, - "cont_pers2": { + "ent_pers2": { "$schema": "https://mef.rero.ch/schemas/mef/mef-v0.0.1.json", "type": "bf:Person", "gnd": { @@ -1621,7 +1621,7 @@ "pid": "029314100", "preferred_name": "Nebehay, Christian Michael" }, - "pid": "cont_pers2", + "pid": "ent_pers2", "rero": { "$schema": "https://mef.rero.ch/schemas/agents_rero/rero-agent-v0.0.1.json", "authorized_access_point": "Nebehay, Christian Michael", diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index 428dc7e36f..370cd0078e 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -26,9 +26,8 @@ import pytest from utils import flush_index, mock_response -from rero_ils.modules.contributions.api import Contribution, \ - ContributionsSearch from rero_ils.modules.documents.api import Document, DocumentsSearch +from rero_ils.modules.entities.api import EntitiesSearch, Entity from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch from rero_ils.modules.local_fields.api import LocalField, LocalFieldsSearch @@ -246,31 +245,30 @@ def journal(app, journal_data): @pytest.fixture(scope="module") -def contribution_person_data(data): +def entity_person_data(data): """Load mef contribution person data.""" - return deepcopy(data.get('cont_pers')) + return deepcopy(data.get('ent_pers')) @pytest.fixture(scope="function") -def contribution_person_data_tmp(app, data): +def entity_person_data_tmp(app, data): """Load mef contribution data person scope function.""" - contribution_person = deepcopy(data.get('cont_pers')) - sources = app.config.get('RERO_ILS_CONTRIBUTIONS_SOURCES', []) - for source in sources: - if source in contribution_person: - contribution_person[source].pop('$schema', None) - return contribution_person + entity_person = deepcopy(data.get('ent_pers')) + for source in app.config.get('RERO_ILS_AGENTS_SOURCES', []): + if source in entity_person: + entity_person[source].pop('$schema', None) + return entity_person @pytest.fixture(scope="module") -def contribution_person_response_data(contribution_person_data): +def entity_person_response_data(entity_person_data): """Load mef contribution person response data.""" return { 'hits': { 'hits': [ { - 'id': contribution_person_data['pid'], - 'metadata': contribution_person_data + 'id': entity_person_data['pid'], + 'metadata': entity_person_data } ] } @@ -278,73 +276,72 @@ def contribution_person_response_data(contribution_person_data): @pytest.fixture(scope="module") -def contribution_person(app, contribution_person_data): +def entity_person(app, entity_person_data): """Load contribution person record.""" - cont = Contribution.create( - data=contribution_person_data, + cont = Entity.create( + data=entity_person_data, delete_pid=False, dbcommit=True, reindex=True) - flush_index(ContributionsSearch.Meta.index) + flush_index(EntitiesSearch.Meta.index) return cont @pytest.fixture(scope="module") -def contribution_organisation_data(data): +def entity_organisation_data(data): """Load mef contribution organisation data.""" - return deepcopy(data.get('cont_org')) + return deepcopy(data.get('ent_org')) @pytest.fixture(scope="function") -def contribution_organisation_data_tmp(data): +def entity_organisation_data_tmp(data): """Load mef contribution data organisation scope function.""" return deepcopy(data.get('cont_oeg')) @pytest.fixture(scope="module") -def contribution_organisation_response_data(contribution_organisation_data): +def entity_organisation_response_data(entity_organisation_data): """Load mef contribution organisation response data.""" - json_data = { + return { 'hits': { 'hits': [ { - 'id': contribution_organisation_data['pid'], - 'metadata': contribution_organisation_data + 'id': entity_organisation_data['pid'], + 'metadata': entity_organisation_data } ] } } - return json_data @pytest.fixture(scope="module") -def contribution_organisation(app, contribution_organisation_data): +def entity_organisation(app, entity_organisation_data): """Create mef contribution organisation record.""" - org = Contribution.create( - data=contribution_organisation_data, + org = Entity.create( + data=entity_organisation_data, delete_pid=False, dbcommit=True, reindex=True) - flush_index(ContributionsSearch.Meta.index) + flush_index(EntitiesSearch.Meta.index) return org @pytest.fixture(scope="module") def person2_data(data): """Load mef person data.""" - return deepcopy(data.get('cont_pers2')) + return deepcopy(data.get('ent_pers2')) @pytest.fixture(scope="function") def person2_data_tmp(data): """Load mef person data scope function.""" - return deepcopy(data.get('cont_pers2')) + return deepcopy(data.get('ent_pers2')) @pytest.fixture(scope="module") def person2_response_data(person2_data): """Load mef person response data.""" - json_data = { + return { 'hits': { 'hits': [ { @@ -354,28 +351,27 @@ def person2_response_data(person2_data): ] } } - return json_data @pytest.fixture(scope="module") def person2(app, person2_data): """Create mef person record.""" - pers = Contribution.create( + pers = Entity.create( data=person2_data, delete_pid=False, dbcommit=True, reindex=True) - flush_index(ContributionsSearch.Meta.index) + flush_index(EntitiesSearch.Meta.index) return pers @pytest.fixture(scope="module") @mock.patch('requests.get') def document_ref(mock_contributions_mef_get, - app, document_data_ref, contribution_person_response_data): + app, document_data_ref, entity_person_response_data): """Load document with mef records reference.""" mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data + json_data=entity_person_response_data ) doc = Document.create( data=document_data_ref, diff --git a/tests/ui/contributions/test_contributions_mapping.py b/tests/ui/contributions/test_contributions_mapping.py deleted file mode 100644 index 28548224b7..0000000000 --- a/tests/ui/contributions/test_contributions_mapping.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2019 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Mef contributions Record tests.""" - -from utils import get_mapping - -from rero_ils.modules.contributions.api import Contribution, \ - ContributionsSearch - - -def test_contribution_es_mapping(es_clear, db, contribution_person_data_tmp): - """Test mef elasticsearch mapping.""" - search = ContributionsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Contribution.create( - contribution_person_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) - - -def test_contributions_search_mapping(app, contribution_person): - """Test Mef contributions search mapping.""" - search = ContributionsSearch() - - count = search.query( - 'query_string', query='philosophische Fakultät').count() - assert count == 1 - - count = search.query( - 'match', - **{'gnd.preferred_name': 'Loy'}).\ - count() - assert count == 1 - - count = search.query( - 'match', - **{'gnd.variant_name': 'Madeiros'}).\ - count() - assert count == 1 diff --git a/tests/ui/documents/test_documents_api.py b/tests/ui/documents/test_documents_api.py index e4cf1d24e7..ef3849ff88 100644 --- a/tests/ui/documents/test_documents_api.py +++ b/tests/ui/documents/test_documents_api.py @@ -28,15 +28,29 @@ from utils import flush_index, mock_response from rero_ils.modules.api import IlsRecordError -from rero_ils.modules.contributions.api import Contribution, \ - ContributionsSearch from rero_ils.modules.documents.api import Document, DocumentsSearch, \ document_id_fetcher from rero_ils.modules.documents.models import DocumentIdentifier from rero_ils.modules.ebooks.tasks import create_records +from rero_ils.modules.entities.api import EntitiesSearch, Entity from rero_ils.modules.tasks import process_bulk_queue +def test_document_properties(document): + """Test document properties.""" + # As no item/holding are loaded into this test, `is_available` should + # return false/raise an error. + assert not Document.is_available(document.pid, 'global') + + with mock.patch( + 'rero_ils.modules.holdings.api.Holding.' + 'get_holdings_pid_by_document_pid', + mock.MagicMock(return_value=['not_exists']) + ): + with pytest.raises(ValueError): + Document.is_available(document.pid, 'global', raise_exception=True) + + def test_document_create(db, document_data_tmp): """Test document creation.""" ptty = Document.create(document_data_tmp, delete_pid=True) @@ -61,35 +75,34 @@ def test_document_create(db, document_data_tmp): @mock.patch('requests.get') def test_document_create_with_mef( mock_contributions_mef_get, app, document_data_ref, document_data, - contribution_person_data, contribution_person_response_data): + entity_person_data, entity_person_response_data): """Load document with mef records reference.""" mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data + json_data=entity_person_response_data ) - assert ContributionsSearch().count() == 0 + assert EntitiesSearch().count() == 0 doc = Document.create( data=deepcopy(document_data_ref), delete_pid=False, dbcommit=False, reindex=False) doc.reindex() flush_index(DocumentsSearch.Meta.index) doc = Document.get_record_by_pid(doc.get('pid')) - assert doc['contribution'][0]['entity']['pid'] == \ - contribution_person_data['pid'] - hit = DocumentsSearch().execute().to_dict()[ - 'hits']['hits'][0] - assert hit['_source']['contribution'][0]['entity'][ - 'pid'] == contribution_person_data['pid'] - assert hit['_source']['contribution'][0]['entity'][ - 'primary_source'] == 'rero' - assert ContributionsSearch().count() == 1 - contrib = Contribution.get_record_by_pid(contribution_person_data['pid']) + assert doc['contribution'][0]['entity']['pid'] == entity_person_data['pid'] + hit = DocumentsSearch().get_record_by_pid(doc.pid).to_dict() + from pprint import pprint + pprint(hit) + + assert hit['contribution'][0]['entity']['pid'] == entity_person_data['pid'] + assert hit['contribution'][0]['entity']['primary_source'] == 'rero' + assert EntitiesSearch().count() == 1 + contrib = Entity.get_record_by_pid(entity_person_data['pid']) contrib.delete_from_index() doc.delete_from_index() db.session.rollback() assert not Document.get_record_by_pid(doc.get('pid')) - assert not Contribution.get_record_by_pid(contribution_person_data['pid']) - assert ContributionsSearch().count() == 0 + assert not Entity.get_record_by_pid(entity_person_data['pid']) + assert EntitiesSearch().count() == 0 with pytest.raises(ValidationError): doc = Document.create( @@ -97,8 +110,8 @@ def test_document_create_with_mef( delete_pid=False, dbcommit=True, reindex=True) assert not Document.get_record_by_pid(doc.get('pid')) - assert not Contribution.get_record_by_pid(contribution_person_data['pid']) - assert ContributionsSearch().count() == 0 + assert not Entity.get_record_by_pid(entity_person_data['pid']) + assert EntitiesSearch().count() == 0 data = deepcopy(document_data_ref) contrib = data.pop('contribution') doc = Document.create( @@ -112,15 +125,15 @@ def test_document_create_with_mef( doc.pop('type') doc.update(doc, commit=True, dbcommit=True, reindex=True) assert Document.get_record_by_pid(doc.get('pid')) - assert not Contribution.get_record_by_pid(contribution_person_data['pid']) - assert ContributionsSearch().count() == 0 + assert not Entity.get_record_by_pid(entity_person_data['pid']) + assert EntitiesSearch().count() == 0 data = deepcopy(document_data_ref) doc.update(data, commit=True, dbcommit=False, reindex=False) doc.reindex() assert Document.get_record_by_pid(doc.get('pid')) - assert Contribution.get_record_by_pid(contribution_person_data['pid']) - assert ContributionsSearch().count() == 1 + assert Entity.get_record_by_pid(entity_person_data['pid']) + assert EntitiesSearch().count() == 1 doc.delete_from_index() db.session.rollback() diff --git a/tests/ui/documents/test_documents_filter.py b/tests/ui/documents/test_documents_filter.py index 29b96ed185..55163d770f 100644 --- a/tests/ui/documents/test_documents_filter.py +++ b/tests/ui/documents/test_documents_filter.py @@ -16,8 +16,8 @@ # along with this program. If not, see . """Document filters tests.""" +import mock -from rero_ils.modules.documents.api import Document from rero_ils.modules.documents.models import DocumentSubjectType from rero_ils.modules.documents.views import cartographic_attributes, \ contribution_format, identified_by, main_title_text, note_general, \ @@ -394,11 +394,24 @@ def test_work_access_point(): assert results == work_access_point(wap) -def test_contribution_format(db, document_data): +def test_contribution_format(db, document, entity_organisation): """Test contribution format.""" result = 'Nebehay, Christian Michael' - doc = Document.create(document_data, delete_pid=True) - assert contribution_format(doc.pid, 'en', 'global').startswith(result) + assert contribution_format(document.pid, 'en', 'global').startswith(result) + + magic_mock = mock.MagicMock(return_value={ + 'contribution': [{ + 'entity': { + 'pid': entity_organisation.pid, + } + }] + }) + with mock.patch( + 'rero_ils.modules.documents.api.Document.dumps', + magic_mock + ): + link_part = f'/corporate-bodies/{entity_organisation.pid}' + assert link_part in contribution_format(document.pid, 'en', 'global') def test_identifiedby_format(): diff --git a/tests/ui/documents/test_documents_mapping.py b/tests/ui/documents/test_documents_mapping.py index ffc2aed968..10ee86c948 100644 --- a/tests/ui/documents/test_documents_mapping.py +++ b/tests/ui/documents/test_documents_mapping.py @@ -29,14 +29,14 @@ @mock.patch('requests.get') def test_document_es_mapping(mock_contributions_mef_get, es, db, org_martigny, document_data_ref, item_lib_martigny, - contribution_person_response_data): + entity_person_response_data): """Test document elasticsearch mapping.""" search = DocumentsSearch() mapping = get_mapping(search.Meta.index) assert mapping data = deepcopy(document_data_ref) mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data + json_data=entity_person_response_data ) Document.create( data, diff --git a/tests/ui/contributions/test_contributions_api.py b/tests/ui/entities/test_entities_api.py similarity index 61% rename from tests/ui/contributions/test_contributions_api.py rename to tests/ui/entities/test_entities_api.py index c3276e4389..7a1b2295a3 100644 --- a/tests/ui/contributions/test_contributions_api.py +++ b/tests/ui/entities/test_entities_api.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,7 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""CircPolicy Record tests.""" +"""Entities Record tests.""" from __future__ import absolute_import, print_function @@ -25,37 +26,34 @@ import mock from utils import flush_index, mock_response -from rero_ils.modules.contributions.api import Contribution, \ - ContributionsSearch, contribution_id_fetcher -from rero_ils.modules.contributions.sync import SyncAgent from rero_ils.modules.documents.api import Document, DocumentsSearch +from rero_ils.modules.entities.api import EntitiesSearch, Entity, \ + entity_id_fetcher +from rero_ils.modules.entities.models import EntityType +from rero_ils.modules.entities.sync import SyncAgent -def test_contribution_create(app, contribution_person_data_tmp, caplog): - """Test MEF contribution creation.""" - pers = Contribution.get_record_by_pid('1') +def test_entity_create(app, entity_person_data_tmp, caplog): + """Test MEF entity creation.""" + pers = Entity.get_record_by_pid('1') assert not pers - pers = Contribution.create( - contribution_person_data_tmp, + pers = Entity.create( + entity_person_data_tmp, dbcommit=True, delete_pid=True ) - assert pers == contribution_person_data_tmp + assert pers == entity_person_data_tmp assert pers.get('pid') == '1' - pers = Contribution.get_record_by_pid('1') - assert pers == contribution_person_data_tmp + pers = Entity.get_record_by_pid('1') + assert pers == entity_person_data_tmp - fetched_pid = contribution_id_fetcher(pers.id, pers) + fetched_pid = entity_id_fetcher(pers.id, pers) assert fetched_pid.pid_value == '1' - assert fetched_pid.pid_type == 'cont' - contribution_person_data_tmp['viaf_pid'] = '1234' - pers = Contribution.create( - contribution_person_data_tmp, - dbcommit=True, - delete_pid=True - ) - pers = Contribution.get_record_by_pid('2') + assert fetched_pid.pid_type == 'ent' + entity_person_data_tmp['viaf_pid'] = '1234' + Entity.create(entity_person_data_tmp, dbcommit=True, delete_pid=True) + pers = Entity.get_record_by_pid('2') assert pers.get('viaf_pid') == '1234' assert pers.organisation_pids == [] @@ -64,54 +62,55 @@ def test_contribution_create(app, contribution_person_data_tmp, caplog): # test the messages from current_app.logger assert caplog.records[0].name == 'elasticsearch' assert caplog.record_tuples[1] == ( - 'invenio', 30, 'Can not delete from index Contribution: 2' + 'invenio', 30, 'Can not delete from index Entity: 2' ) @mock.patch('requests.get') -def test_contribution_mef_create(mock_contributions_mef_get, app, - mef_agents_url, - contribution_person_data_tmp, - contribution_person_response_data): +def test_entity_mef_create( + mock_contributions_mef_get, app, mef_agents_url, + entity_person_data_tmp, entity_person_response_data +): """Test MEF contribution creation.""" - count = Contribution.count() + count = Entity.count() mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data + json_data=entity_person_response_data ) - pers_mef, online = Contribution.get_record_by_ref( + pers_mef, online = Entity.get_record_by_ref( f'{mef_agents_url}/rero/A017671081') - flush_index(ContributionsSearch.Meta.index) - assert pers_mef == contribution_person_data_tmp + flush_index(EntitiesSearch.Meta.index) + assert pers_mef == entity_person_data_tmp assert online - assert Contribution.count() == count + 1 + assert Entity.count() == count + 1 pers_mef.pop('idref') pers_mef['sources'] = ['gnd'] pers_mef.replace(pers_mef, dbcommit=True) - pers_db, online = Contribution.get_record_by_ref( + pers_db, online = Entity.get_record_by_ref( f'{mef_agents_url}/gnd/13343771X') assert pers_db['sources'] == ['gnd'] assert not online # remove created contribution - Contribution.get_record_by_pid(contribution_person_data_tmp['pid']).delete( + Entity.get_record_by_pid(entity_person_data_tmp['pid']).delete( True, True, True) -@mock.patch('rero_ils.modules.contributions.api.requests.get') -def test_sync_contribution(mock_get, app, mef_agents_url, - contribution_person_data_tmp, document_data_ref): +@mock.patch('rero_ils.modules.entities.api.requests.get') +def test_sync_contribution( + mock_get, app, mef_agents_url, entity_person_data_tmp, document_data_ref +): """Test MEF agent synchronization.""" # === setup log_path = tempfile.mkdtemp() sync = SyncAgent(log_dir=log_path) assert sync - pers = Contribution.create( - contribution_person_data_tmp, + pers = Entity.create( + entity_person_data_tmp, dbcommit=True, reindex=True, delete_pid=True ) - flush_index(ContributionsSearch.Meta.index) + flush_index(EntitiesSearch.Meta.index) idref_pid = pers['idref']['pid'] document_data_ref['contribution'][0]['entity']['$ref'] = \ @@ -126,15 +125,14 @@ def test_sync_contribution(mock_get, app, mef_agents_url, flush_index(DocumentsSearch.Meta.index) # === nothing to update - sync._get_latest = mock.MagicMock( - return_value=contribution_person_data_tmp) - # nothing touched as it is up to date + sync._get_latest = mock.MagicMock(return_value=entity_person_data_tmp) + # nothing touched as it is up-to-date assert (0, 0, set()) == sync.sync(f'{pers.pid}') # nothing removed assert (0, []) == sync.remove_unused(f'{pers.pid}') # === MEF metadata has been changed - data = deepcopy(contribution_person_data_tmp) + data = deepcopy(entity_person_data_tmp) data['idref']['authorized_access_point'] = 'foo' sync._get_latest = mock.MagicMock(return_value=data) mock_resp = dict(hits=dict(hits=[dict( @@ -151,7 +149,7 @@ def test_sync_contribution(mock_get, app, mef_agents_url, flush_index(DocumentsSearch.Meta.index) # contribution and document should be changed - assert Contribution.get_record_by_pid( + assert Entity.get_record_by_pid( pers.pid)['idref']['authorized_access_point'] == 'foo' assert DocumentsSearch().query( 'term', contribution__entity__authorized_access_point_fr='foo').count() @@ -159,7 +157,7 @@ def test_sync_contribution(mock_get, app, mef_agents_url, assert (0, []) == sync.remove_unused(f'{pers.pid}') # === a new MEF exists with the same content - data = deepcopy(contribution_person_data_tmp) + data = deepcopy(entity_person_data_tmp) # MEF pid has changed data['pid'] = 'foo_mef' # mock MEF services @@ -171,12 +169,12 @@ def test_sync_contribution(mock_get, app, mef_agents_url, mock_get.return_value = mock_response(json_data=mock_resp) # synchronization the same document has been updated 3 times, one MEF - # record has been udpated, no errors + # record has been updated, no errors assert (1, 1, set()) == sync.sync(f'{pers.pid}') flush_index(DocumentsSearch.Meta.index) # new contribution has been created - assert Contribution.get_record_by_pid('foo_mef') - assert Contribution.get_record_by_ref( + assert Entity.get_record_by_pid('foo_mef') + assert Entity.get_record_by_ref( f'{mef_agents_url}/idref/{idref_pid}')[0] db_agent = Document.get_record_by_pid( doc.pid).get('contribution')[0]['entity'] @@ -184,10 +182,10 @@ def test_sync_contribution(mock_get, app, mef_agents_url, # the old MEF has been removed assert (1, []) == sync.remove_unused(f'{pers.pid}') # should not exists anymore - assert not Contribution.get_record_by_pid(pers.pid) + assert not Entity.get_record_by_pid(pers.pid) # === Update the MEF links content - data = deepcopy(contribution_person_data_tmp) + data = deepcopy(entity_person_data_tmp) # MEF pid has changed data['pid'] = 'foo_mef' # IDREF pid has changed @@ -205,7 +203,7 @@ def test_sync_contribution(mock_get, app, mef_agents_url, assert (1, 1, set()) == sync.sync(f'{data["pid"]}') flush_index(DocumentsSearch.Meta.index) # new contribution has been created - assert Contribution.get_record_by_pid('foo_mef') + assert Entity.get_record_by_pid('foo_mef') # document has been updated with the new MEF and IDREF pid assert DocumentsSearch().query( 'term', contribution__entity__pid='foo_mef').count() @@ -223,4 +221,40 @@ def test_sync_contribution(mock_get, app, mef_agents_url, # the MEF record can be removed assert (1, []) == sync.remove_unused() # should not exists anymore - assert not Contribution.get_record_by_pid('foo_mef') + assert not Entity.get_record_by_pid('foo_mef') + + +def test_entity_properties( + entity_person, item_lib_martigny, document, document_data +): + """Test entity properties.""" + item = item_lib_martigny + + assert document.pid not in entity_person.documents_pids() + assert document.id not in entity_person.documents_ids() + assert item.organisation_pid not in entity_person.organisation_pids + document['contribution'] = [{ + 'entity': { + '$ref': 'https://mef.rero.ch/api/agents/idref/223977268', + 'type': EntityType.PERSON + }, + 'role': ['cre'] + }] + document.update(document, dbcommit=True, reindex=True) + assert document.pid in entity_person.documents_pids() + assert str(document.id) in entity_person.documents_ids() + assert item.organisation_pid in entity_person.organisation_pids + + assert entity_person == Entity.get_entity('mef', entity_person.pid) + assert entity_person == Entity.get_entity('viaf', '70119347') + + sources_pids = entity_person.source_pids() + assert sources_pids['idref'] == '223977268' + assert sources_pids['gnd'] == '13343771X' + assert sources_pids['rero'] == 'A017671081' + + document.index_contributions() + document.index_contributions(True) + + # Reset fixture + document.update(document_data, dbcommit=True, reindex=True) diff --git a/tests/ui/contributions/test_contributions_filter.py b/tests/ui/entities/test_entities_filter.py similarity index 81% rename from tests/ui/contributions/test_contributions_filter.py rename to tests/ui/entities/test_entities_filter.py index bed6e8e2b1..cac9ab565d 100644 --- a/tests/ui/contributions/test_contributions_filter.py +++ b/tests/ui/entities/test_entities_filter.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -17,27 +18,27 @@ """Jinja2 filters tests.""" -from rero_ils.modules.contributions.views import contribution_label, \ - contribution_merge_data_values +from rero_ils.modules.entities.views import entity_label, \ + entity_merge_data_values -def test_contribution_label(app, contribution_person_data): - """Test contributions merge data.""" - app.config['RERO_ILS_CONTRIBUTIONS_LABEL_ORDER'] = { +def test_entity_label(app, entity_person_data): + """Test entity label.""" + app.config['RERO_ILS_AGENTS_LABEL_ORDER'] = { 'fallback': 'fr', 'fr': ['rero', 'idref', 'gnd'], 'de': ['gnd', 'rero', 'idref'], } - label = contribution_label(contribution_person_data, 'fr') + label = entity_label(entity_person_data, 'fr') assert label == 'Loy, Georg, 1885-19..' - label = contribution_label(contribution_person_data, 'it') + label = entity_label(entity_person_data, 'it') assert label == 'Loy, Georg, 1885-19..' -def test_contribution_merge_data_values(app, contribution_person_data): - """Test contributions merge data.""" - app.config['RERO_ILS_CONTRIBUTIONS_SOURCES'] = ['idref', 'gnd', 'rero'] - data = contribution_merge_data_values(contribution_person_data) +def test_entity_merge_data_values(app, entity_person_data): + """Test entities merge data.""" + app.config['RERO_ILS_AGENTS_SOURCES'] = ['idref', 'gnd', 'rero'] + data = entity_merge_data_values(entity_person_data) assert data == { '$schema': { 'https://mef.test.rero.ch/schemas/gnd/' diff --git a/tests/ui/entities/test_entities_mapping.py b/tests/ui/entities/test_entities_mapping.py new file mode 100644 index 0000000000..95508452c2 --- /dev/null +++ b/tests/ui/entities/test_entities_mapping.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Mef entities record tests.""" + +from utils import get_mapping + +from rero_ils.modules.entities.api import EntitiesSearch, Entity + + +def test_entity_es_mapping(es_clear, db, entity_person_data_tmp): + """Test entity elasticsearch mapping.""" + search = EntitiesSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Entity.create( + entity_person_data_tmp, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_entities_search_mapping(app, entity_person): + """Test Mef entities search mapping.""" + assert EntitiesSearch()\ + .query('query_string', query='philosophische Fakultät')\ + .count() == 1 + assert EntitiesSearch()\ + .query('match', **{'gnd.preferred_name': 'Loy'})\ + .count() == 1 + assert EntitiesSearch()\ + .query('match', **{'gnd.variant_name': 'Madeiros'})\ + .count() == 1 diff --git a/tests/ui/contributions/test_contributions_ui.py b/tests/ui/entities/test_entities_ui.py similarity index 60% rename from tests/ui/contributions/test_contributions_ui.py rename to tests/ui/entities/test_entities_ui.py index 66bdeb22d8..e8fa2cd13f 100644 --- a/tests/ui/contributions/test_contributions_ui.py +++ b/tests/ui/entities/test_entities_ui.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019 RERO +# Copyright (C) 2019-2023 RERO +# Copyright (C) 2019-2023 UCLouvain # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,23 +16,22 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Tests UI view for patrons.""" +"""Tests UI view for entities.""" from flask import url_for -def test_contributions_person_detailed_view(client, contribution_person): - """Test contribution person detailed view.""" +def test_entity_person_detailed_view(client, entity_person): + """Test entity person detailed view.""" res = client.get(url_for( - 'contributions.persons_proxy', - viewcode='global', pid=contribution_person.pid)) + 'entities.persons_proxy', + viewcode='global', pid=entity_person.pid)) assert res.status_code == 200 -def test_contributions_organisation_detailed_view( - client, contribution_organisation): - """Test contribution organisation detailed view.""" +def test_entity_organisation_detailed_view(client, entity_organisation): + """Test entity organisation detailed view.""" res = client.get(url_for( - 'contributions.corporate_bodies_proxy', - viewcode='global', pid='cont_org')) + 'entities.corporate_bodies_proxy', + viewcode='global', pid='ent_org')) assert res.status_code == 200 diff --git a/tests/ui/test_indexer_utils.py b/tests/ui/test_indexer_utils.py index 3c43afd647..23641e6fc1 100644 --- a/tests/ui/test_indexer_utils.py +++ b/tests/ui/test_indexer_utils.py @@ -18,9 +18,38 @@ """API tests for indexer utilities.""" import pytest from elasticsearch import NotFoundError +from mock import mock +from utils import flush_index from rero_ils.modules.documents.api import DocumentsSearch from rero_ils.modules.indexer_utils import record_to_index +from rero_ils.modules.libraries.api import LibrariesIndexer, LibrariesSearch + + +def test_record_indexing(app, lib_martigny): + """Test record indexing process.""" + + # TEST#1 :: Test indexing without $ref replacement + app.config['INDEXER_REPLACE_REFS'] = False + lib_martigny.reindex() + flush_index(LibrariesSearch.Meta.index) + record = LibrariesSearch().get_record_by_pid(lib_martigny.pid) + assert '$ref' in record.organisation.to_dict() + + # TEST#2 :: Raise exception during indexing process + with mock.patch( + 'rero_ils.modules.api.IlsRecordsIndexer._index_action', + side_effect=Exception('Test!') + ): + indexer = LibrariesIndexer() + indexer.bulk_index([lib_martigny.id]) + res = indexer.process_bulk_queue() + assert res[1] == (0, 0) + + # RESET INDEX + app.config['INDEXER_REPLACE_REFS'] = True + lib_martigny.reindex() + flush_index(LibrariesSearch.Meta.index) def test_record_to_index(app): @@ -40,7 +69,7 @@ def test_record_to_index(app): assert record_to_index({ '$schema': 'https://mef.rero.ch/schemas/' 'mef/mef-contribution-v0.0.1.json' - }) == ('contributions-contribution-v0.0.1', '_doc') + }) == ('entities-entity-v0.0.1', '_doc') # for others assert record_to_index({ diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 161922e863..48d85fc825 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -211,11 +211,11 @@ def patron_martigny_data_tmp_with_id(patron_martigny_data_tmp): @pytest.fixture() -def contributions_schema(monkeypatch): - """Patron Jsonschema for records.""" +def entities_schema(monkeypatch): + """Entity Jsonschema for records.""" schema_in_bytes = resource_string( - 'rero_ils.modules.contributions.jsonschemas', - '/contributions/contribution-v0.0.1.json' + 'rero_ils.modules.entities.jsonschemas', + '/entities/entity-v0.0.1.json' ) return get_schema(monkeypatch, schema_in_bytes) @@ -292,7 +292,7 @@ def mef_record_with_idref_rero(): """Mef record with idref rero.""" return { '$schema': 'https://ils.rero.ch/schemas/' - 'contributions/contribution-v0.0.1.json', + 'entities/entity-v0.0.1.json', 'idref': { '$schema': 'https://mef.rero.ch/schemas/' 'agents_idref/idref-agent-v0.0.1.json', diff --git a/tests/unit/documents/test_documents_dojson_marc21.py b/tests/unit/documents/test_documents_dojson_marc21.py index 75c22e44f3..05f35474fa 100644 --- a/tests/unit/documents/test_documents_dojson_marc21.py +++ b/tests/unit/documents/test_documents_dojson_marc21.py @@ -442,7 +442,7 @@ def test_contribution_to_marc21(app, mef_agents_url, marc21_record, }] } with mock.patch( - 'rero_ils.modules.contributions.api.Contribution.get_contribution', + 'rero_ils.modules.entities.api.Entity.get_entity', side_effect=[mef_record_with_idref_rero, mef_record_with_idref_gnd, mef_record_with_idref_gnd_rero] ): diff --git a/tests/unit/documents/test_subjects.py b/tests/unit/documents/test_subjects.py index 70c65d685b..84324235d8 100644 --- a/tests/unit/documents/test_subjects.py +++ b/tests/unit/documents/test_subjects.py @@ -21,9 +21,9 @@ import pytest from utils import mock_response -from rero_ils.modules.contributions.api import Contribution from rero_ils.modules.documents.commons import SubjectFactory from rero_ils.modules.documents.models import DocumentSubjectType +from rero_ils.modules.entities.api import Entity from rero_ils.modules.utils import get_ref_for_pid @@ -101,13 +101,13 @@ def test_document_local_subjects(): @mock.patch('requests.get') -def test_document_referenced_subject(mock_contributions_mef_get, - mef_agents_url, - contribution_person_response_data, - contribution_person): +def test_document_referenced_subject( + mock_contributions_mef_get, mef_agents_url, + entity_person_response_data, entity_person +): """Test referenced document subjects.""" mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data) + json_data=entity_person_response_data) # REFERENCED SUBJECTS - SUCCESS data = { @@ -121,7 +121,7 @@ def test_document_referenced_subject(mock_contributions_mef_get, # REFERENCED SUBJECTS - ERRORS data = { - '$dummy_ref': get_ref_for_pid(Contribution, contribution_person.pid), + '$dummy_ref': get_ref_for_pid(Entity, entity_person.pid), 'type': DocumentSubjectType.PERSON } with pytest.raises(AttributeError): diff --git a/tests/unit/test_cli_fixtures.py b/tests/unit/test_cli_fixtures.py index b2406e6791..26b992512b 100644 --- a/tests/unit/test_cli_fixtures.py +++ b/tests/unit/test_cli_fixtures.py @@ -51,11 +51,11 @@ def test_count(app, script_info): @mock.patch('requests.get') def test_create(mock_contributions_mef_get, app, script_info, - contribution_person_response_data): + entity_person_response_data): """Test create cli.""" json_file_name = join(dirname(__file__), '../data/documents.json') mock_contributions_mef_get.return_value = mock_response( - json_data=contribution_person_response_data + json_data=entity_person_response_data ) runner = CliRunner() diff --git a/tests/unit/test_contributions_jsonschema.py b/tests/unit/test_contributions_jsonschema.py index 372906a51c..3543c8485f 100644 --- a/tests/unit/test_contributions_jsonschema.py +++ b/tests/unit/test_contributions_jsonschema.py @@ -24,40 +24,40 @@ from jsonschema.exceptions import ValidationError -def test_required(contributions_schema, contribution_person_data_tmp): +def test_required(entities_schema, entity_person_data_tmp): '''Test required for patron jsonschemas.''' - validate(contribution_person_data_tmp, contributions_schema) + validate(entity_person_data_tmp, entities_schema) with pytest.raises(ValidationError): - validate({}, contributions_schema) - validate(contribution_person_data_tmp, contributions_schema) + validate({}, entities_schema) + validate(entity_person_data_tmp, entities_schema) with pytest.raises(ValidationError): validate({ - 'pid': 'cont_pers', + 'pid': 'ent_pers', 'viaf_pid': '56597999', 'sources': [ 'rero', 'gnd' - ]}, contributions_schema) - validate(contribution_person_data_tmp, contributions_schema) + ]}, entities_schema) + validate(entity_person_data_tmp, entities_schema) with pytest.raises(ValidationError): validate({ - '$schema': 'https://bib.rero.ch/schemas/contributions/' - 'contribution-v0.0.1.json', + '$schema': 'https://bib.rero.ch/schemas/entities/' + 'entity-v0.0.1.json', 'viaf_pid': '56597999', 'sources': [ 'rero', 'gnd' - ]}, contributions_schema) - validate(contribution_person_data_tmp, contributions_schema) + ]}, entities_schema) + validate(entity_person_data_tmp, entities_schema) with pytest.raises(ValidationError): validate({ - '$schema': 'https://bib.rero.ch/schemas/contributions/' - 'contribution-v0.0.1.json', - 'pid': 'cont_pers', + '$schema': 'https://bib.rero.ch/schemas/entities/' + 'entity-v0.0.1.json', + 'pid': 'ent_pers', 'viaf_pid': '56597999' - }, contributions_schema) - validate(contribution_person_data_tmp, contributions_schema) + }, entities_schema) + validate(entity_person_data_tmp, entities_schema)