From b05eb74fab3a118dddce552198a5fce51cb482a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johnny=20Mari=C3=A9thoz?= Date: Wed, 10 Jul 2019 17:04:19 +0200 Subject: [PATCH] search: Replace AND default operator by OR. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds a new display_score url parameters `display_score=1`. * Restore OR default search operator, closes #384. * Improves Elasticsearch mappings and JSONSchema for documents: * two JSONSchema: one minimal for havested records and one for internal record. * unique elasticsearch mapping * Put cantook and internal records in the same Elasticsearch index. * Improves search page layout. * Improves serializers permissions, closes #89. Co-Authored-by: Johnny Mariéthoz Co-Authored-by: Bertrand Zuchuat --- rero_ils/config.py | 16 +- rero_ils/modules/documents/api.py | 9 +- ....0.1.json => document-minimal-v0.0.1.json} | 75 ++--- .../documents/document-v0.0.1.json | 31 ++ .../{document-v0.0.1.json => document.json} | 2 +- .../mappings/v6/documents/ebook-v0.0.1.json | 268 ------------------ rero_ils/modules/ebooks/receivers.py | 3 +- rero_ils/modules/ebooks/search.py | 36 --- rero_ils/modules/ebooks/tasks.py | 14 +- rero_ils/modules/indexer_utils.py | 50 ++++ rero_ils/modules/serializers.py | 8 +- rero_ils/permissions.py | 12 +- rero_ils/query.py | 13 +- rero_ils/templates/rero_ils/search.html | 2 +- tests/api/test_documents_rest.py | 19 +- tests/data/data.json | 20 +- tests/ui/documents/test_documents_api.py | 19 ++ tests/ui/test_indexer_utils.py | 32 +++ .../autocomplete/autocomplete.component.html | 1 + .../autocomplete/autocomplete.component.ts | 5 + ui/src/app/records/records.service.ts | 9 +- .../public-documents-brief-view.component.ts | 17 +- .../documents-search.component.ts | 6 + ui/src/app/records/search/search.component.ts | 18 +- ui/src/assets/i18n/de.json | 5 +- ui/src/assets/i18n/en.json | 7 +- ui/src/assets/i18n/en_US.json | 1 + ui/src/assets/i18n/fr.json | 5 +- ui/src/assets/i18n/it.json | 5 +- 29 files changed, 315 insertions(+), 393 deletions(-) rename rero_ils/modules/documents/jsonschemas/documents/{ebook-v0.0.1.json => document-minimal-v0.0.1.json} (92%) rename rero_ils/modules/documents/mappings/v6/documents/{document-v0.0.1.json => document.json} (99%) delete mode 100644 rero_ils/modules/documents/mappings/v6/documents/ebook-v0.0.1.json delete mode 100644 rero_ils/modules/ebooks/search.py create mode 100644 rero_ils/modules/indexer_utils.py create mode 100644 tests/ui/test_indexer_utils.py diff --git a/rero_ils/config.py b/rero_ils/config.py index 2f2b2db7d9..ce8c6a9941 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -72,7 +72,8 @@ can_create_organisation_records_factory, \ can_delete_organisation_records_factory, \ can_update_organisation_records_factory, \ - librarian_delete_permission_factory, librarian_permission_factory + librarian_delete_permission_factory, librarian_permission_factory, \ + librarian_update_permission_factory def _(x): @@ -321,7 +322,7 @@ def _(x): RECORDS_REST_DEFAULT_READ_PERMISSION_FACTORY = librarian_permission_factory """Default read permission factory: check if the record exists.""" -RECORDS_REST_DEFAULT_UPDATE_PERMISSION_FACTORY = librarian_permission_factory +RECORDS_REST_DEFAULT_UPDATE_PERMISSION_FACTORY = librarian_update_permission_factory """Default update permission factory: reject any request.""" RECORDS_REST_DEFAULT_DELETE_PERMISSION_FACTORY = librarian_delete_permission_factory @@ -363,7 +364,7 @@ def _(x): # ), default_media_type='application/json', max_result_window=5000000, - search_factory_imp='rero_ils.query:and_search_factory', + search_factory_imp='rero_ils.query:search_factory', read_permission_factory_imp=allow_all, list_permission_factory_imp=allow_all ), @@ -522,7 +523,7 @@ def _(x): item_route='/organisations/', default_media_type='application/json', max_result_window=10000, - search_factory_imp='rero_ils.query:and_search_factory', + search_factory_imp='rero_ils.query:search_factory', create_permission_factory_imp=deny_all, update_permission_factory_imp=deny_all, delete_permission_factory_imp=deny_all, @@ -615,7 +616,7 @@ def _(x): item_route='/persons/', default_media_type='application/json', max_result_window=10000, - search_factory_imp='rero_ils.query:and_search_factory', + search_factory_imp='rero_ils.query:search_factory', read_permission_factory_imp=allow_all, list_permission_factory_imp=allow_all, create_permission_factory_imp=deny_all, @@ -812,8 +813,8 @@ def _(x): # Elasticsearch fields boosting by index RERO_ILS_QUERY_BOOSTING = { 'documents': { - 'title.*': 2, - 'titlesProper.*': 2, + 'title.*': 3, + 'titlesProper.*': 3, 'authors.name': 2, 'authors.name_*': 2, 'publicationYearText': 2, @@ -932,6 +933,7 @@ def _(x): # Misc INDEXER_REPLACE_REFS = True +INDEXER_RECORD_TO_INDEX = 'rero_ils.modules.indexer_utils.record_to_index' SEARCH_UI_SEARCH_API = '/api/documents/' diff --git a/rero_ils/modules/documents/api.py b/rero_ils/modules/documents/api.py index 479c27b129..d1b74b5aa4 100644 --- a/rero_ils/modules/documents/api.py +++ b/rero_ils/modules/documents/api.py @@ -64,11 +64,16 @@ class Document(IlsRecord): fetcher = document_id_fetcher provider = DocumentProvider + @property + def harvested(self): + """Is this record harvested from an external service.""" + return bool(self.get('identifiers', {}).get('harvestedID')) + @property def can_edit(self): """Return a boolean for can_edit resource.""" # TODO: Make this condition on data - return 'ebook' != self.get('type') + return not self.harvested def get_number_of_items(self): """Get number of items for document.""" @@ -106,4 +111,6 @@ def reasons_not_to_delete(self): links = self.get_links_to_me() if links: cannot_delete['links'] = links + if self.harvested: + cannot_delete['others'] = dict(harvested=True) return cannot_delete diff --git a/rero_ils/modules/documents/jsonschemas/documents/ebook-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json similarity index 92% rename from rero_ils/modules/documents/jsonschemas/documents/ebook-v0.0.1.json rename to rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json index f513202fe4..97fd021c84 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/ebook-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json @@ -1,12 +1,13 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Schema for ebooks document", + "title": "Schema for document", "type": "object", "required": [ "$schema", "pid", "type", - "title" + "title", + "languages" ], "additionalProperties": false, "properties": { @@ -14,7 +15,8 @@ "title": "Schema", "description": "Schema to validate document against.", "type": "string", - "minLength": 7 + "minLength": 7, + "default": "https://ils.rero.ch/schema/documents/document-minimal-v0.0.1.json" }, "pid": { "title": "Document ID", @@ -76,6 +78,7 @@ "title": "Language", "description": "Required. Language of the resource, primary or not.", "type": "string", + "default": "fre", "validationMessage": "Required. Language of the resource, primary or not.", "enum": [ "fre", @@ -89,8 +92,7 @@ "heb", "jpn", "por", - "rus", - "und" + "rus" ] } } @@ -107,37 +109,29 @@ "validationMessage": "Should be in the ISO 639 format, with 3 characters, ie eng for English." } }, - "electronic_location": { - "title": "Electronic Location", - "description": "Information needed to locate and access an electronic resource.", - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": [ - "uri" - ], - "uri": { - "title": "Uniform Resource Identifier", - "description": "Uniform Resource Identifier (URI), which provides standard syntax for locating an object using existing Internet protocols.", - "type": "string", - "format": "uri" - } - } - }, "authors": { "title": "Authors", "description": "Author(s) of the resource. Can be either persons or organisations.", "type": "array", - "minItems": 1, + "minItems": 0, "items": { "type": "object", "required": [ - "name", "type" ], "additionalProperties": false, "properties": { + "$ref": { + "title": "MEF person ref", + "type": "string", + "pattern": "^https://mef.rero.ch/api/mef/.*?$" + }, + "pid": { + "title": "pid", + "description": "Corresponding pid of the MEF record.", + "type": "string", + "minLength": 1 + }, "name": { "title": "Name", "description": "Person's or organisation's name.", @@ -311,12 +305,12 @@ "pattern": "^97[8|9][0-9]{10}$", "validationMessage": "Should be a valid ISBN-13 without dashes." }, - "oai": { - "title": "OAI", + "harvestedID": { + "title": "Identifier in the harvested source.", "description": "OAI identifiers of the harvested source.", "type": "string", - "pattern": "^oai:.*", - "validationMessage": "Should be a valid ISBN-13 without dashes." + "pattern": "^[a-z]+:.*", + "validationMessage": "Should start with a chars: prefix i.e. cantook:." } } }, @@ -330,16 +324,29 @@ "minLength": 1 } }, - "available": { - "title": "Document availability", - "type": "boolean", - "default": false - }, "cover_art": { "title": "Cover art", "description": "Vendor cover art URL.", "type": "string", "format": "uri" + }, + "electronic_location": { + "title": "Electronic Location", + "description": "Information needed to locate and access an electronic resource.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "uri" + ], + "uri": { + "title": "Uniform Resource Identifier", + "description": "Uniform Resource Identifier (URI), which provides standard syntax for locating an object using existing Internet protocols.", + "type": "string", + "format": "uri" + } + } } } } diff --git a/rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json index e7295c908d..f5d53b686a 100644 --- a/rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json +++ b/rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json @@ -304,6 +304,13 @@ "type": "string", "pattern": "^97[8|9][0-9]{10}$", "validationMessage": "Should be a valid ISBN-13 without dashes." + }, + "harvestedID": { + "title": "Identifier in the harvested source.", + "description": "OAI identifiers of the harvested source.", + "type": "string", + "pattern": "^[a-z]+:.*", + "validationMessage": "Should start with a chars: prefix i.e. cantook:." } } }, @@ -316,6 +323,30 @@ "type": "string", "minLength": 1 } + }, + "cover_art": { + "title": "Cover art", + "description": "Vendor cover art URL.", + "type": "string", + "format": "uri" + }, + "electronic_location": { + "title": "Electronic Location", + "description": "Information needed to locate and access an electronic resource.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "uri" + ], + "uri": { + "title": "Uniform Resource Identifier", + "description": "Uniform Resource Identifier (URI), which provides standard syntax for locating an object using existing Internet protocols.", + "type": "string", + "format": "uri" + } + } } } } diff --git a/rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json b/rero_ils/modules/documents/mappings/v6/documents/document.json similarity index 99% rename from rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json rename to rero_ils/modules/documents/mappings/v6/documents/document.json index 58130b12c3..5829f244b9 100644 --- a/rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json +++ b/rero_ils/modules/documents/mappings/v6/documents/document.json @@ -24,7 +24,7 @@ } }, "mappings": { - "document-v0.0.1": { + "document": { "date_detection": false, "numeric_detection": false, "properties": { diff --git a/rero_ils/modules/documents/mappings/v6/documents/ebook-v0.0.1.json b/rero_ils/modules/documents/mappings/v6/documents/ebook-v0.0.1.json deleted file mode 100644 index 0153fe1230..0000000000 --- a/rero_ils/modules/documents/mappings/v6/documents/ebook-v0.0.1.json +++ /dev/null @@ -1,268 +0,0 @@ -{ - "settings": { - "number_of_shards": 1, - "number_of_replicas": 0, - "max_result_window": 20000 - }, - "mappings": { - "ebook-v0.0.1": { - "date_detection": false, - "numeric_detection": false, - "properties": { - "$schema": { - "type": "keyword" - }, - "pid": { - "type": "keyword" - }, - "title": { - "type": "text", - "analyzer": "global_lowercase_asciifolding", - "fields": { - "eng": { - "type": "text", - "analyzer": "english" - }, - "fre": { - "type": "text", - "analyzer": "french" - }, - "ger": { - "type": "text", - "analyzer": "german" - }, - "ita": { - "type": "text", - "analyzer": "italian" - } - } - }, - "titlesProper": { - "type": "text", - "analyzer": "global_lowercase_asciifolding", - "fields": { - "eng": { - "type": "text", - "analyzer": "english" - }, - "fre": { - "type": "text", - "analyzer": "french" - }, - "ger": { - "type": "text", - "analyzer": "german" - }, - "ita": { - "type": "text", - "analyzer": "italian" - } - } - }, - "type": { - "type": "keyword" - }, - "languages": { - "type": "object", - "properties": { - "language": { - "type": "keyword" - } - } - }, - "is_part_of": { - "type": "text", - "analyzer": "global_lowercase_asciifolding", - "fields": { - "eng": { - "type": "text", - "analyzer": "english" - }, - "fre": { - "type": "text", - "analyzer": "french" - }, - "ger": { - "type": "text", - "analyzer": "german" - }, - "ita": { - "type": "text", - "analyzer": "italian" - } - } - }, - "translatedFrom": { - "type": "keyword" - }, - "authors": { - "type": "object", - "properties": { - "name": { - "type": "text", - "analyzer": "global_lowercase_asciifolding", - "copy_to": [ - "facet_authors_en", - "facet_authors_fr", - "facet_authors_de", - "facet_authors_it" - ] - }, - "type": { - "type": "keyword" - }, - "date": { - "type": "keyword" - }, - "qualifier": { - "type": "keyword" - } - } - }, - "item_status": { - "type": "keyword" - }, - "facet_authors_en": { - "type": "keyword" - }, - "facet_authors_fr": { - "type": "keyword" - }, - "facet_authors_de": { - "type": "keyword" - }, - "facet_authors_it": { - "type": "keyword" - }, - "publishers": { - "type": "object", - "properties": { - "name": { - "type": "text", - "analyzer": "global_lowercase_asciifolding" - }, - "place": { - "type": "text" - } - } - }, - "freeFormedPublicationDate": { - "type": "keyword" - }, - "extent": { - "type": "text", - "analyzer": "global_lowercase_asciifolding" - }, - "publicationYear": { - "type": "date", - "format": "yyyy", - "copy_to": "publicationYearText" - }, - "publicationYearText": { - "type": "keyword" - }, - "otherMaterialCharacteristics": { - "type": "keyword" - }, - "formats": { - "type": "keyword" - }, - "electronic_location": { - "properties": { - "uri": { - "type": "keyword" - } - } - }, - "additionalMaterials": { - "type": "keyword" - }, - "series": { - "type": "object", - "properties": { - "name": { - "type": "text", - "analyzer": "global_lowercase_asciifolding", - "fields": { - "eng": { - "type": "text", - "analyzer": "english" - }, - "fre": { - "type": "text", - "analyzer": "french" - }, - "ger": { - "type": "text", - "analyzer": "german" - }, - "ita": { - "type": "text", - "analyzer": "italian" - } - } - }, - "number": { - "type": "keyword" - } - } - }, - "notes": { - "type": "text" - }, - "abstracts": { - "type": "text", - "analyzer": "global_lowercase_asciifolding", - "fields": { - "eng": { - "type": "text", - "analyzer": "english" - }, - "fre": { - "type": "text", - "analyzer": "french" - }, - "ger": { - "type": "text", - "analyzer": "german" - }, - "ita": { - "type": "text", - "analyzer": "italian" - } - } - }, - "identifiers": { - "type": "object", - "properties": { - "reroID": { - "type": "keyword" - }, - "bnfID": { - "type": "keyword" - }, - "isbn": { - "type": "keyword" - }, - "oai": { - "type": "keyword" - } - } - }, - "subjects": { - "type": "text", - "copy_to": "facet_subjects" - }, - "facet_subjects": { - "type": "keyword" - }, - "_created": { - "type": "date" - }, - "_updated": { - "type": "date" - } - } - } - } -} diff --git a/rero_ils/modules/ebooks/receivers.py b/rero_ils/modules/ebooks/receivers.py index 4652ad9d26..c7fcbac607 100644 --- a/rero_ils/modules/ebooks/receivers.py +++ b/rero_ils/modules/ebooks/receivers.py @@ -45,7 +45,8 @@ def publish_harvested_records(sender=None, records=[], *args, **kwargs): continue rec = create_record(record.xml) rec = marc21.do(rec) - rec.setdefault('identifiers', {})['oai'] = record.header.identifier + rec.setdefault( + 'identifiers', {})['harvestedID'] = record.header.identifier converted_records.append(rec) if records: current_app.logger.info( diff --git a/rero_ils/modules/ebooks/search.py b/rero_ils/modules/ebooks/search.py deleted file mode 100644 index 44584eae14..0000000000 --- a/rero_ils/modules/ebooks/search.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of RERO ILS. -# Copyright (C) 2017 RERO. -# -# RERO ILS is free software; you can redistribute it -# and/or modify it under the terms of the GNU General Public License as -# published by the Free Software Foundation; either version 2 of the -# License, or (at your option) any later version. -# -# RERO ILS is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with RERO ILS; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, -# MA 02111-1307, USA. -# -# In applying this license, RERO does not -# waive the privileges and immunities granted to it by virtue of its status -# as an Intergovernmental Organization or submit itself to any jurisdiction. - -"""Search utilities.""" - -from invenio_search.api import RecordsSearch - - -class EbookSearch(RecordsSearch): - """RecordsSearch for documents.""" - - class Meta: - """Search only on documents index.""" - - index = 'documents' diff --git a/rero_ils/modules/ebooks/tasks.py b/rero_ils/modules/ebooks/tasks.py index 4c3c4f74bd..bf667859ed 100644 --- a/rero_ils/modules/ebooks/tasks.py +++ b/rero_ils/modules/ebooks/tasks.py @@ -29,8 +29,7 @@ from celery import shared_task from flask import current_app -from .search import EbookSearch -from ..documents.api import Document +from ..documents.api import Document, DocumentsSearch @shared_task(ignore_result=True) @@ -40,12 +39,15 @@ def create_records(records): n_created = 0 for record in records: record['$schema'] = \ - 'https://ils.rero.ch/schema/documents/ebook-v0.0.1.json' + 'https://ils.rero.ch/schema/documents/document-minimal-v0.0.1.json' # check if already harvested - oai_id = record.get('identifiers').get('oai') - query = EbookSearch().filter('term', identifiers__oai=oai_id)\ - .source(includes=['pid']) + harvestedID = record.get('identifiers').get('harvestedID') + query = DocumentsSearch().filter( + 'term', + identifiers__harvestedID=harvestedID + ).source(includes=['pid']) + # update the record try: pid = [r.pid for r in query.scan()].pop() diff --git a/rero_ils/modules/indexer_utils.py b/rero_ils/modules/indexer_utils.py new file mode 100644 index 0000000000..6d5eefa2e8 --- /dev/null +++ b/rero_ils/modules/indexer_utils.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Utility functions for indexer data processing.""" + +import re + +from flask import current_app +from invenio_search import current_search +from invenio_search.utils import schema_to_index + + +def record_to_index(record): + """Get index/doc_type given a record. + + It tries to extract from `record['$schema']` the index and doc_type. + If it fails, return the default values. + + :param record: The record object. + :returns: Tuple (index, doc_type). + """ + index_names = current_search.mappings.keys() + schema = record.get('$schema', '') + if isinstance(schema, dict): + schema = schema.get('$ref', '') + + # put all document in the same index + if re.search(r'/documents/', schema): + schema = re.sub(r'-.*\.json', '.json', schema) + index, doc_type = schema_to_index(schema, index_names=index_names) + + if index and doc_type: + return index, doc_type + else: + return (current_app.config['INDEXER_DEFAULT_INDEX'], + current_app.config['INDEXER_DEFAULT_DOC_TYPE']) diff --git a/rero_ils/modules/serializers.py b/rero_ils/modules/serializers.py index 5b754f3604..16282aad32 100644 --- a/rero_ils/modules/serializers.py +++ b/rero_ils/modules/serializers.py @@ -38,6 +38,7 @@ class RecordSchemaJSONV1(_RecordSchemaJSONV1): """ permissions = fields.Raw() + explanation = fields.Raw() class JSONSerializer(_JSONSerializer): @@ -80,13 +81,16 @@ def preprocess_search_hit(pid, record_hit, links_factory=None, **kwargs): 'cannot_update': {'permisson': 'permission denied'}, 'cannot_delete': {'permisson': 'permission denied'} } - return dict( + search_hit = dict( pid=pid, metadata=record_hit['_source'], links=links_factory(pid, record_hit=record_hit, **kwargs), revision=record_hit['_version'], permissions=permissions ) + if record_hit.get('_explanation'): + search_hit['explanation'] = record_hit.get('_explanation') + return search_hit @staticmethod def add_item_links_and_permissions(record, data, pid): @@ -100,7 +104,7 @@ def add_item_links_and_permissions(record, data, pid): for action in actions: permission = JSONSerializer.get_permission(action, pid.pid_type) if permission: - can = permission(record).can() + can = permission(record, credentials_only=True).can() if can: action_links[action] = url_for( 'invenio_records_rest.{pid_type}_item'.format( diff --git a/rero_ils/permissions.py b/rero_ils/permissions.py index 8600706528..b0dfad25cc 100644 --- a/rero_ils/permissions.py +++ b/rero_ils/permissions.py @@ -195,8 +195,18 @@ def librarian_permission_factory(record, *args, **kwargs): return librarian_permission -def librarian_delete_permission_factory(record, *args, **kwargs): +def librarian_update_permission_factory(record, *args, **kwargs): + """User has editor role and the record is editable.""" + if record.can_edit: + return librarian_permission + return type('Check', (), {'can': lambda x: False})() + + +def librarian_delete_permission_factory( + record, credentials_only=False, *args, **kwargs): """User can delete record.""" + if credentials_only: + return librarian_permission if record.can_delete: return librarian_permission return type('Check', (), {'can': lambda x: False})() diff --git a/rero_ils/query.py b/rero_ils/query.py index 6902d8fa9e..6597034eb9 100644 --- a/rero_ils/query.py +++ b/rero_ils/query.py @@ -36,14 +36,14 @@ def organisation_search_factory(self, search, query_parser=None): """Search factory.""" - search, urlkwargs = and_search_factory(self, search) + search, urlkwargs = search_factory(self, search) if current_patron: search = search.filter( 'term', organisation__pid=current_patron.get_organisation()['pid']) return (search, urlkwargs) -def and_search_factory(self, search, query_parser=None): +def search_factory(self, search, query_parser=None): """Parse query using elasticsearch DSL query. Terms defined by: RERO_ILS_QUERY_BOOSTING will be boosted @@ -57,12 +57,12 @@ def _default_parser(qstr=None, query_boosting=[]): """Default parser that uses the Q() from elasticsearch_dsl.""" if qstr: if not query_boosting: - return Q('query_string', query=qstr, default_operator='AND') + return Q('query_string', query=qstr) else: return Q('bool', should=[ Q('query_string', query=qstr, boost=2, - fields=query_boosting, default_operator="AND"), - Q('query_string', query=qstr, default_operator="AND") + fields=query_boosting), + Q('query_string', query=qstr) ]) return Q() @@ -78,6 +78,9 @@ def _boosting_parser(query_boosting, search_index): from invenio_records_rest.sorter import default_sorter_factory query_string = request.values.get('q') + display_score = request.values.get('display_score') + if display_score: + search = search.extra(explain=True) query_parser = query_parser or _default_parser search_index = search._index[0] diff --git a/rero_ils/templates/rero_ils/search.html b/rero_ils/templates/rero_ils/search.html index e261283c72..c17dbfc126 100644 --- a/rero_ils/templates/rero_ils/search.html +++ b/rero_ils/templates/rero_ils/search.html @@ -14,7 +14,7 @@
- +
diff --git a/tests/api/test_documents_rest.py b/tests/api/test_documents_rest.py index 45f07a9fab..4832b42098 100644 --- a/tests/api/test_documents_rest.py +++ b/tests/api/test_documents_rest.py @@ -97,6 +97,12 @@ def test_documents_get(client, document): url_for('api_documents.import_bnf_ean', ean='9782070541270')) assert res.status_code == 401 + list_url = url_for('invenio_records_rest.doc_list', q="Vincent Berthe") + res = client.get(list_url) + assert res.status_code == 200 + data = get_json(res) + assert data['hits']['total'] == 1 + @mock.patch('invenio_records_rest.views.verify_record_permission', mock.MagicMock(return_value=VerifyRecordPermissionPatch)) @@ -228,19 +234,6 @@ def test_documents_import_bnf_ean(client): } -def test_document_can_delete(client, item_lib_martigny, loan_pending_martigny, - document): - """Test can delete a document.""" - links = document.get_links_to_me() - assert 'items' in links - assert 'loans' in links - - assert not document.can_delete - - reasons = document.reasons_not_to_delete() - assert 'links' in reasons - - def test_document_can_request_view(client, item_lib_fully, loan_pending_martigny, document, patron_martigny_no_email, diff --git a/tests/data/data.json b/tests/data/data.json index 3d4153d9ca..f9642705b8 100644 --- a/tests/data/data.json +++ b/tests/data/data.json @@ -1000,7 +1000,7 @@ "freeFormedPublicationDate": "[ca 1889]" }, "ebook1": { - "$schema": "https://ils.rero.ch/schema/documents/ebook-v0.0.1.json", + "$schema": "https://ils.rero.ch/schema/documents/document-minimal-v0.0.1.json", "pid": "ebook1", "type": "ebook", "title": "La maison des oubli\u00e9s", @@ -1031,7 +1031,8 @@ "Le d\u00e9m\u00e9nagement dans ce manoir charmant, en haut de la colline, devait \u00eatre le point de d\u00e9part pour une nouvelle vie. Apr\u00e8s des ann\u00e9es pass\u00e9es dans la banlieue de Brighton, Ollie Harcourt ne pouvait r\u00eaver mieux qu'une existence paisible \u00e0 la campagne. Le reste de la famille suit d'un pas h\u00e9sitant, mais ne rechigne pas pour autant \u00e0 cette nouvelle aventure. Cependant, peu apr\u00e8s leur installation, des sc\u00e8nes \u00e9tranges se d\u00e9roulent dans la maison. Des ombres apparaissent, les animaux domestiques se comportent de mani\u00e8re bizarre et plusieurs accidents, plus d\u00e9routants les uns que les autres, ont lieu. Bient\u00f4t, Ollie n'a plus de doute : leur pr\u00e9sence n'est pas vraiment souhait\u00e9e. Quelqu'un semble m\u00eame pr\u00eat \u00e0 tout pour les expulser de l\u00e0... \u00e0 n'importe quel prix." ], "identifiers": { - "isbn": "9782823855890" + "isbn": "9782823855890", + "harvestedID": "cantook:ebook1" }, "subjects": [ "thriller", @@ -1039,9 +1040,12 @@ ] }, "ebook2": { - "$schema": "https://ils.rero.ch/schema/documents/ebook-v0.0.1.json", + "$schema": "https://ils.rero.ch/schema/documents/document-minimal-v0.0.1.json", "pid": "ebook2", "type": "ebook", + "identifiers": { + "harvestedID": "cantook:ebook2" + }, "title": "Kurkuma: Effekte auf den menschlichen K\u00f6rper-Der aktuelle Stand der Wissenschaft", "languages": [ { @@ -1071,8 +1075,11 @@ ] }, "ebook3": { - "$schema": "https://ils.rero.ch/schema/documents/ebook-v0.0.1.json", + "$schema": "https://ils.rero.ch/schema/documents/document-minimal-v0.0.1.json", "pid": "ebook3", + "identifiers": { + "harvestedID": "cantook:ebook3" + }, "type": "ebook", "title": "Harry Potter and the Chamber of Secrets", "languages": [ @@ -1103,8 +1110,11 @@ ] }, "ebook4": { - "$schema": "https://ils.rero.ch/schema/documents/ebook-v0.0.1.json", + "$schema": "https://ils.rero.ch/schema/documents/document-minimal-v0.0.1.json", "pid": "ebook4", + "identifiers": { + "harvestedID": "cantook:ebook4" + }, "type": "ebook", "title": "Nouveau titre", "languages": [ diff --git a/tests/ui/documents/test_documents_api.py b/tests/ui/documents/test_documents_api.py index 79afefe4f0..394f4b35c0 100644 --- a/tests/ui/documents/test_documents_api.py +++ b/tests/ui/documents/test_documents_api.py @@ -63,6 +63,25 @@ def test_document_can_delete(app, document_data_tmp): assert document.can_delete +def test_document_can_delete_harvested(app, ebook_1_data): + """Test can delete for harvested records.""" + document = Document.create(ebook_1_data, delete_pid=True) + assert not document.can_delete + + +def test_document_can_delete_with_loans( + client, item_lib_martigny, loan_pending_martigny, document): + """Test can delete a document.""" + links = document.get_links_to_me() + assert 'items' in links + assert 'loans' in links + + assert not document.can_delete + + reasons = document.reasons_not_to_delete() + assert 'links' in reasons + + @mock.patch('rero_ils.modules.documents.listener.requests_get') @mock.patch('rero_ils.modules.documents.jsonresolver_mef_person.requests_get') def test_document_person_resolve(mock_resolver_get, mock_listener_get, diff --git a/tests/ui/test_indexer_utils.py b/tests/ui/test_indexer_utils.py new file mode 100644 index 0000000000..96c2dcdfe6 --- /dev/null +++ b/tests/ui/test_indexer_utils.py @@ -0,0 +1,32 @@ +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""API tests for indexer utilities.""" + +from rero_ils.modules.indexer_utils import record_to_index + + +def test_record_to_index(app): + """Test the index name value from the JSONSchema.""" + + # for documents + assert record_to_index({ + '$schema': 'https://ils.rero.ch/schema/' + 'documents/document-minimal-v0.0.1.json' + }) == ('documents-document', 'document') + + # for others + assert record_to_index({ + '$schema': 'https://ils.rero.ch/schema/' + 'organisations/organisation-v0.0.1.json' + }) == ('organisations-organisation-v0.0.1', 'organisation-v0.0.1') diff --git a/ui/src/app/autocomplete/autocomplete.component.html b/ui/src/app/autocomplete/autocomplete.component.html index 05455ddc2a..67c6ec54f5 100644 --- a/ui/src/app/autocomplete/autocomplete.component.html +++ b/ui/src/app/autocomplete/autocomplete.component.html @@ -25,6 +25,7 @@ > +
diff --git a/ui/src/app/autocomplete/autocomplete.component.ts b/ui/src/app/autocomplete/autocomplete.component.ts index c51107fb1c..03cbdf11aa 100644 --- a/ui/src/app/autocomplete/autocomplete.component.ts +++ b/ui/src/app/autocomplete/autocomplete.component.ts @@ -29,6 +29,8 @@ export class AutocompleteComponent implements OnInit { @Input() placeholder: string; @Input() maxLengthSuggestion = 100; + @Input() + displayScore = undefined; constructor( private recordsService: RecordsService, @@ -46,6 +48,9 @@ export class AutocompleteComponent implements OnInit { ngOnInit() { this.route.queryParamMap.subscribe((params: any) => { const query = params.get('q'); + if (params.get('display_score')) { + this.displayScore = params.get('display_score'); + } if (query) { this.asyncSelected = { query: query, diff --git a/ui/src/app/records/records.service.ts b/ui/src/app/records/records.service.ts index 7480c8c0df..dc043e3772 100644 --- a/ui/src/app/records/records.service.ts +++ b/ui/src/app/records/records.service.ts @@ -40,7 +40,8 @@ export class RecordsService { query: string = '', mime_type: string = 'application/json', filters = [], - sort? + sort?, + displayScore? ) { let url = `/api/${record_type}/?page=${page}&size=${size}&q=${query}`; if (filters.length) { @@ -49,6 +50,9 @@ export class RecordsService { if (sort) { url = url + `&sort=${sort}`; } + if (displayScore) { + url = url + `&display_score=${displayScore}`; + } return this.http.get(url, this.httpOptions(mime_type)).pipe( catchError(e => { if (e.status === 404) { @@ -279,7 +283,8 @@ export class RecordsService { private othersMessages() { return { 'is_default': this.translateService.instant(_('The default record cannot be deleted')), - 'has_settings': this.translateService.instant(_('The record contains settings')) + 'has_settings': this.translateService.instant(_('The record contains settings')), + 'harvested': this.translateService.instant(_('The record has been harvested')) }; } diff --git a/ui/src/app/records/search/brief-view/public-documents-brief-view.component.ts b/ui/src/app/records/search/brief-view/public-documents-brief-view.component.ts index f0b03a5419..e8287b3243 100644 --- a/ui/src/app/records/search/brief-view/public-documents-brief-view.component.ts +++ b/ui/src/app/records/search/brief-view/public-documents-brief-view.component.ts @@ -60,9 +60,18 @@ import { _, } from '@app/core'; available not available +
+ + 
{{record.explanation|json}}
+
- `, + +`, styles: [` .thumb-brief img { @@ -76,6 +85,12 @@ import { _, } from '@app/core'; max-width: 48px; } } + +pre { + white-space: pre-wrap; + max-height: 300px; + font-size: 0.7em; +} `] }) export class PublicDocumentsBriefViewComponent implements BriefView { diff --git a/ui/src/app/records/search/public-search/documents-search.component.ts b/ui/src/app/records/search/public-search/documents-search.component.ts index 467995ecac..2dde8fd7ad 100644 --- a/ui/src/app/records/search/public-search/documents-search.component.ts +++ b/ui/src/app/records/search/public-search/documents-search.component.ts @@ -14,6 +14,7 @@ import { map } from 'rxjs/operators'; [currentPage]="currentPage" [aggFilters]="aggFilters" [showSearchInput]="false" + [displayScore]="displayScore" > `, @@ -25,6 +26,7 @@ export class DocumentsSearchComponent implements OnInit { public nPerPage = undefined; public currentPage = undefined; public aggFilters = undefined; + public displayScore = undefined; constructor( protected route: ActivatedRoute, @@ -53,6 +55,10 @@ export class DocumentsSearchComponent implements OnInit { this.currentPage = +urlQuery.get(key); break; } + case 'display_score': { + this.displayScore = +urlQuery.get(key); + break; + } default: { for (const value of urlQuery.getAll(key)) { const filterValue = `${key}=${value}`; diff --git a/ui/src/app/records/search/search.component.ts b/ui/src/app/records/search/search.component.ts index 4bd3daa5f9..82a0093be9 100644 --- a/ui/src/app/records/search/search.component.ts +++ b/ui/src/app/records/search/search.component.ts @@ -73,6 +73,18 @@ export class SearchComponent implements OnInit { } private _currentPage = 1; + @Input() + set displayScore(value) { + if (value !== undefined) { + this._displayScore = value; + this.getRecords(); + } + } + get displayScore() { + return this._displayScore; + } + private _displayScore = 0; + @Input() set aggFilters(value) { if (value !== undefined) { @@ -161,7 +173,8 @@ export class SearchComponent implements OnInit { this.query, this.searchMime, this.aggFilters, - sort + sort, + this.displayScore ).subscribe(data => { if (data === null) { this.notFound = true; @@ -220,6 +233,9 @@ export class SearchComponent implements OnInit { page: this.currentPage, q: this.query }; + if (this.displayScore) { + queryParams['display_score'] = this.displayScore; + } const filters = {}; for (const filter of this.aggFilters) { const [key, value] = filter.split('='); diff --git a/ui/src/assets/i18n/de.json b/ui/src/assets/i18n/de.json index 13120a560c..22196a948c 100644 --- a/ui/src/assets/i18n/de.json +++ b/ui/src/assets/i18n/de.json @@ -123,6 +123,7 @@ "has # patrons attached": "hat # Leser angefügt", "The default record cannot be deleted": "Der Standardsatz kann nicht gelöscht werden.", "The record contains settings": "Der Datensatz enthält Einstellungen", + "The record has been harvested": "Der Datensatz wurde gesammelt", "Edit": "Bearbeiten", "no item": "keine Exemplare", "Record deleted.": "Datensatz gelöscht.", @@ -169,7 +170,7 @@ "Request date": "Datum der Bestellung", "Please insert a name": "Bitte erfassen Sie einen Namen", "Days before due date": "Days before due date", - "Loading…": "Wird geladen...", + "Loading\u2026": "Wird geladen...", "Name": "Name", "Name is required.": "Name ist erforderlich.", "Name must be at least 2 characters long.": "Der Name muss mindestens 2 Zeichen lang sein.", @@ -231,7 +232,7 @@ "Please insert a code": "Bitte geben Sie einen Code ein", "Opening Hours": "Öffnungszeiten", "Exceptions (holidays, etc.)": "Ausnahmen (Feiertage, usw.)", - "Validating…": "Validierung...", + "Validating\u2026": "Validierung...", "Name must be at least 4 characters long.": "Der Name muss mindestens 4 Zeichen lang sein.", "Address must be at least 4 characters long.": "Die Adresse muss mindestens 4 Zeichen lang sein.", "Email format is not correct.": "Das E-Mail-Format ist nicht korrekt.", diff --git a/ui/src/assets/i18n/en.json b/ui/src/assets/i18n/en.json index 002583ec14..fe685cada9 100644 --- a/ui/src/assets/i18n/en.json +++ b/ui/src/assets/i18n/en.json @@ -123,6 +123,7 @@ "has # patrons attached": "has # patrons attached", "The default record cannot be deleted": "The default record cannot be deleted", "The record contains settings": "The record contains settings", + "The record has been harvested": "The record has been harvested", "Edit": "Edit", "no item": "no item", "Record deleted.": "Record deleted.", @@ -243,5 +244,7 @@ "Documents": "Documents", "Persons": "Persons", "results": "results", - "No result found.": "No result found." -} \ No newline at end of file + "No result found.": "No result found.", + "more…": "more…", + "less…": "less…" +} diff --git a/ui/src/assets/i18n/en_US.json b/ui/src/assets/i18n/en_US.json index 6096ba8ba9..e972ce5eda 100644 --- a/ui/src/assets/i18n/en_US.json +++ b/ui/src/assets/i18n/en_US.json @@ -123,6 +123,7 @@ "has # patrons attached": "has # patrons attached", "The default record cannot be deleted": "The default record cannot be deleted", "The record contains settings": "The record contains settings", + "The record has been harvested": "The record has been harvested", "Edit": "Edit", "no item": "no item", "Record deleted.": "Record deleted.", diff --git a/ui/src/assets/i18n/fr.json b/ui/src/assets/i18n/fr.json index 978f86d708..a9f66652a1 100644 --- a/ui/src/assets/i18n/fr.json +++ b/ui/src/assets/i18n/fr.json @@ -123,6 +123,7 @@ "has # patrons attached": "a # lecteurs attachés", "The default record cannot be deleted": "L'enregistrement par défaut ne peut pas être supprimé", "The record contains settings": "L'enregistrement contient des paramètres", + "The record has been harvested": "L'enregistrement a été moissonné.", "Edit": "Editer", "no item": "pas d'exemplaire", "Record deleted.": "La ressource a été effacée.", @@ -169,7 +170,7 @@ "Request date": "Date de la demande", "Please insert a name": "Veuillez introduire un nom", "Days before due date": "Nombre de jours avant l'échéance", - "Loading…": "Chargement en cours...", + "Loading\u2026": "Chargement en cours...", "Name": "Nom", "Name is required.": "Le nom est obligatoire.", "Name must be at least 2 characters long.": "Le nom doit comporter au moins 2 caractères.", @@ -231,7 +232,7 @@ "Please insert a code": "Veuillez introduire un code", "Opening Hours": "Heures d'ouverture", "Exceptions (holidays, etc.)": "Exceptions (vacances, etc.)", - "Validating…": "Validation en cours...", + "Validating\u2026": "Validation en cours...", "Name must be at least 4 characters long.": "Le nom doit comporter au moins 4 caractères.", "Address must be at least 4 characters long.": "L'adresse doit comporter au moins 4 caractères.", "Email format is not correct.": "Le format de l'email est incorrect.", diff --git a/ui/src/assets/i18n/it.json b/ui/src/assets/i18n/it.json index 9962312c53..24ff2a5f8d 100644 --- a/ui/src/assets/i18n/it.json +++ b/ui/src/assets/i18n/it.json @@ -123,6 +123,7 @@ "has # patrons attached": "has # patrons attached", "The default record cannot be deleted": "The default record cannot be deleted", "The record contains settings": "The record contains settings", + "The record has been harvested": "Il record è stato raccolto", "Edit": "Edit", "no item": "nessun'esemplare", "Record deleted.": "Record deleted.", @@ -169,7 +170,7 @@ "Request date": "Data della richiesta", "Please insert a name": "Please insert a name", "Days before due date": "Days before due date", - "Loading…": "Caricamento...", + "Loading\u2026": "Caricamento...", "Name": "Name", "Name is required.": "Name is required.", "Name must be at least 2 characters long.": "Name must be at least 2 characters long.", @@ -231,7 +232,7 @@ "Please insert a code": "Please insert a code", "Opening Hours": "Opening Hours", "Exceptions (holidays, etc.)": "Exceptions (holidays, etc.)", - "Validating…": "Validazione...", + "Validating\u2026": "Validazione...", "Name must be at least 4 characters long.": "Name must be at least 4 characters long.", "Address must be at least 4 characters long.": "Address must be at least 4 characters long.", "Email format is not correct.": "Email format is not correct.",