diff --git a/.gitignore b/.gitignore index 0d48520cbe..a2a803de83 100644 --- a/.gitignore +++ b/.gitignore @@ -77,6 +77,9 @@ celerybeat-schedule # Coverage xml file cov.xml -# these json files are generated during the setup +# these json files are generated during bootstrap or run-test rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json + +# ES mapping files +tmp/ diff --git a/MANIFEST.in b/MANIFEST.in index 153f524285..e626780c4d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -35,6 +35,9 @@ recursive-include rero_ils *.po *.pot *.mo recursive-include docker *.cfg *.conf *.crt *.ini *.key *.pem *.sh exclude rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json exclude rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json +# ES mappings +exclude mapping_after_setup.txt +exclude mapping_before_setup.txt # added by check_manifest.py include *.base diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py index 92f0519537..adfc1c64d5 100644 --- a/rero_ils/modules/cli.py +++ b/rero_ils/modules/cli.py @@ -1324,3 +1324,23 @@ def run_tests(self, tests): dependency_tests.run_tests(tests) sys.exit(dependency_tests.missing + dependency_tests.not_found) + + +@utils.command('dump_es_mappings') +@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) +@click.option('-o', '--outfile', 'outfile', type=click.File('w'), default=None) +@with_appcontext +def dump_es_mappings(verbose, outfile): + """Dumps ES mappings.""" + click.secho('Dump ES mappings:', fg='green') + aliases = current_search.client.indices.get_alias('*') + mappings = current_search.client.indices.get_mapping() + for alias in sorted(aliases): + if alias[0] != '.': + mapping = mappings.get(alias, {}).get('mappings') + click.echo('{alias}'.format(alias=alias)) + if verbose or not outfile: + print(json.dumps(mapping, indent=2)) + if outfile: + json.dump(mapping, outfile, indent=2) + outfile.write('\n') diff --git a/rero_ils/modules/documents/listener.py b/rero_ils/modules/documents/listener.py index c10d4d327f..ff9c9896ee 100644 --- a/rero_ils/modules/documents/listener.py +++ b/rero_ils/modules/documents/listener.py @@ -17,7 +17,6 @@ """Signals connector for Document.""" -from .views import create_publication_statement from ..documents.api import DocumentsSearch from ..holdings.api import Holding, HoldingsSearch from ..items.api import ItemsSearch @@ -78,17 +77,6 @@ def enrich_document_data(sender, json=None, record=None, index=None, if holdings: json['holdings'] = holdings - # provisionActivity - publisher_statements = [] - for provision_activity in record.get('provisionActivity', []): - publication_statement = create_publication_statement( - provision_activity - ).get('default') - if publication_statement: - publisher_statements.append(publication_statement) - if publisher_statements: - json['publisherStatement'] = publisher_statements - # MEF person ES index update authors = [] for author in json.get('authors', []): diff --git a/rero_ils/modules/documents/mappings/v6/documents/document.json b/rero_ils/modules/documents/mappings/v6/documents/document.json index 5be70bd240..241c4d38c7 100644 --- a/rero_ils/modules/documents/mappings/v6/documents/document.json +++ b/rero_ils/modules/documents/mappings/v6/documents/document.json @@ -194,8 +194,11 @@ "properties": { "_text": { "properties": { - "default": { + "value": { "type": "text" + }, + "language": { + "type": "keyword" } } }, @@ -245,9 +248,6 @@ "statement": { "type": "object", "properties": { - "type": { - "type": "text" - }, "label": { "type": "object", "properties": { @@ -259,6 +259,9 @@ "type": "keyword" } } + }, + "type": { + "type": "text" } } }, @@ -274,19 +277,21 @@ "type": "date", "format": "yyyy" }, + "type": { + "type": "text" + }, "_text": { "properties": { - "default": { + "value": { "type": "text" + }, + "language": { + "type": "keyword" } } } } }, - "publisherStatement": { - "type": "keyword", - "index": false - }, "extent": { "type": "text", "analyzer": "global_lowercase_asciifolding" @@ -435,6 +440,9 @@ "pid": { "type": "keyword" }, + "available": { + "type": "boolean" + }, "call_number": { "type": "keyword" }, diff --git a/rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html b/rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html index 8d2d4d32a5..4376c3cdd8 100644 --- a/rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html +++ b/rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html @@ -48,12 +48,12 @@

{{ record.title }}

{% if record.editionStatement %} - {{ dl_list(_('Edition'), record.editionStatement | edition_format) }} + {{ dl_list(_('Edition'), record.editionStatement|edition_format) }} {% endif %} {% for provision_activity in record.provisionActivity %} - {{ dl_dict(_(provision_activity.type), provision_activity|create_publication_statement ) }} + {{ dl_list(_(provision_activity.type), provision_activity|create_publication_statement ) }} {% endfor %} diff --git a/rero_ils/modules/documents/utils.py b/rero_ils/modules/documents/utils.py index 88fe17b8dc..f63bc51413 100644 --- a/rero_ils/modules/documents/utils.py +++ b/rero_ils/modules/documents/utils.py @@ -83,10 +83,14 @@ def publication_statement_text(provision_activity): statement_type = statement['type'] # date field: remove ';' and append + statement_text = [] for key, value in statement_with_language.items(): value = remove_trailing_punctuation(value) - statement_with_language[key] = value - return statement_with_language + if key == 'default': + statement_text.insert(0, {'value': value, 'language': key}) + else: + statement_text.append({'value': value, 'language': key}) + return statement_text def series_format_text(serie): @@ -101,7 +105,6 @@ def series_format_text(serie): def edition_format_text(edition): """Format edition for _text.""" - edition_with_language = {'default': ''} designations = edition.get('editionDesignation', []) responsibilities = edition.get('responsibility', []) designation_output = {} @@ -115,13 +118,17 @@ def edition_format_text(edition): value = responsibility.get('value', '') responsibility_output[language] = value + edition_text = [] for key, value in designation_output.items(): - output_value = remove_trailing_punctuation( + value = remove_trailing_punctuation( '{designation} / {responsibility}'.format( - designation=value, + designation=designation_output.get(key), responsibility=responsibility_output.get(key, ''), ) ) - edition_with_language[key] = output_value + if key == 'default': + edition_text.insert(0, {'value': value, 'language': key}) + else: + edition_text.append({'value': value, 'language': key}) - return edition_with_language + return edition_text diff --git a/rero_ils/modules/documents/views.py b/rero_ils/modules/documents/views.py index 5fc2c2dace..6226e4394a 100644 --- a/rero_ils/modules/documents/views.py +++ b/rero_ils/modules/documents/views.py @@ -295,10 +295,8 @@ def edition_format(editions): for edition in editions: languages = edition_format_text(edition) if languages: - output.append(languages['default']) - del languages['default'] - for key, value in languages.items(): - output.append(value) + for edition_text in languages: + output.append(edition_text.get('value')) return output @@ -431,4 +429,8 @@ def document_availability(document_pid): @blueprint.app_template_filter() def create_publication_statement(provision_activity): """Create publication statement from place, agent and date values.""" - return publication_statement_text(provision_activity) + output = [] + publication_texts = publication_statement_text(provision_activity) + for publication_text in publication_texts: + output.append(publication_text.get('value')) + return output diff --git a/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json b/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json index 2f6607f160..54cf93f678 100644 --- a/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json +++ b/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json @@ -21,6 +21,13 @@ "fee_type": { "type": "keyword" }, + "organisation": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, "notification": { "properties": { "pid": { diff --git a/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json b/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json index 49af392d13..642e90c718 100644 --- a/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json +++ b/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json @@ -50,6 +50,20 @@ } } }, + "library": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, + "organisation": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, "_created": { "type": "date" }, @@ -59,4 +73,4 @@ } } } -} \ No newline at end of file +} diff --git a/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json b/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json index 8eeac9270f..2e4ee2ce4c 100644 --- a/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json +++ b/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json @@ -52,6 +52,9 @@ }, "period": { "type": "keyword" + }, + "data": { + "type": "long" } } }, diff --git a/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json b/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json index 4deb4a2e60..1afa464c45 100644 --- a/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json +++ b/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json @@ -40,6 +40,9 @@ "end_date": { "type": "date" }, + "extension_count": { + "type": "long" + }, "state": { "type": "keyword" }, diff --git a/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json b/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json index 92a8885ac9..5c4a49f452 100644 --- a/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json +++ b/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json @@ -37,6 +37,13 @@ } } }, + "organisation": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, "_created": { "type": "date" }, @@ -46,4 +53,4 @@ } } } -} \ No newline at end of file +} diff --git a/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json b/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json index a84cb1552c..a467b0f14d 100644 --- a/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json +++ b/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json @@ -34,6 +34,13 @@ } } }, + "organisation": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, "_created": { "type": "date" }, diff --git a/rero_ils/modules/vendors/mappings/v6/vendors/vendor-v0.0.1.json b/rero_ils/modules/vendors/mappings/v6/vendors/vendor-v0.0.1.json index b2463d1c58..df576b7657 100644 --- a/rero_ils/modules/vendors/mappings/v6/vendors/vendor-v0.0.1.json +++ b/rero_ils/modules/vendors/mappings/v6/vendors/vendor-v0.0.1.json @@ -23,9 +23,18 @@ "vendor_name": { "type": "keyword" }, + "note": { + "type": "text" + }, + "website": { + "type": "text" + }, "currency": { "type": "keyword" }, + "communication_language": { + "type": "keyword" + }, "tva_rate": { "type": "keyword" }, @@ -36,6 +45,28 @@ } } }, + "default_contact": { + "properties": { + "city": { + "type": "keyword" + }, + "country": { + "type": "keyword" + }, + "email": { + "type": "keyword" + }, + "phone": { + "type": "keyword" + }, + "postal_code": { + "type": "keyword" + }, + "street": { + "type": "text" + } + } + }, "_created": { "type": "date" }, diff --git a/run-tests.sh b/run-tests.sh index f934469ca7..1f02ccff29 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -105,4 +105,4 @@ if [ "$1" = "external" ] fi success_msg "Perfect ${PROGRAM}! See you soon…" -exit 0 \ No newline at end of file +exit 0 diff --git a/scripts/setup b/scripts/setup index 930ea7fcff..14c45daead 100755 --- a/scripts/setup +++ b/scripts/setup @@ -75,12 +75,13 @@ STOP_EXECUTION=true CREATE_LAZY="" DONT_STOP="" PREFIX="" +ES_MAPPING=false # Displays program name msg "PROGRAM: ${PROGRAM}" # options may be followed by one colon to indicate they have a required argument -if ! options=$(getopt -o dsb -l -t deployment,create_items_holdings_small,create_items_holdings_big,lazy,pursue,time,data_path: -- "$@") +if ! options=$(getopt -o dsb -l -t deployment,create_items_holdings_small,create_items_holdings_big,lazy,pursue,data_path,time,es-mapping: -- "$@") then # something went wrong, getopt will put out an error message for us exit 1 @@ -97,6 +98,7 @@ do -p|--pursue) DONT_STOP="--dont-stop" ;; -D|--data_path) DATA_PATH=$2 ;; -t|--time) PREFIX="time" ;; # Get time for all commands + -m|--es-mapping) ES_MAPPING=true ;; (--) shift; break;; (-*) error_msg+exit "$0: Unrecognized option $1";; (*) break;; @@ -115,7 +117,7 @@ fi if $CREATE_ITEMS_HOLDINGS_SMALL || $CREATE_ITEMS_HOLDINGS_BIG then - if $DEPLOYMENT + if ${DEPLOYMENT} then error_msg+exit "Choose deployment option only" fi @@ -147,6 +149,13 @@ eval ${PREFIX} pipenv run invenio index queue init info_msg "Delete invenio_circulations index" eval ${PREFIX} pipenv run invenio index delete loans-loan-v1.0.0 --force --yes-i-know +if ${ES_MAPPING} +then + info_msg "ES mappings:" + mkdir -p tmp + pipenv run invenio utils dump_es_mappings -o tmp/mapping_before_setup.txt +fi + # Create admin role to restrict access info_msg "Create admin role to restrict access" eval ${PREFIX} pipenv run invenio roles create admin @@ -231,7 +240,7 @@ eval ${PREFIX} pipenv run invenio fixtures import_users ${DATA_PATH}/users.json # eval ${PREFIX} pipenv run invenio utils marc21tojson ${DATA_PATH}/documents_big.xml ${DATA_PATH}/documents_big.json ${DATA_PATH}/documents_big_errors.xml -v -r # eval ${PREFIX} pipenv run invenio utils marc21tojson ${DATA_PATH}/documents_small.xml ${DATA_PATH}/documents_small.json ${DATA_PATH}/documents_small_errors.xml -v -r -if $DEPLOYMENT +if ${DEPLOYMENT} then DOCUMENTS=${DATA_PATH}/documents_big.json ITEMS=${DATA_PATH}/items_big.json @@ -303,19 +312,19 @@ eval ${PREFIX} pipenv run invenio index run -c 4 --raise-on-error info_msg "Library budgets:" eval ${PREFIX} pipenv run invenio fixtures create --pid_type budg ${DATA_PATH}/budgets.json --append eval ${PREFIX} pipenv run invenio index reindex -t budg --yes-i-know -eval ${PREFIX} pipenv run invenio index run -c 4 --raise-on-error +eval ${PREFIX} pipenv run invenio index run -c 4 --raise-on-error # create acquisition accounts info_msg "Acquisition accounts:" eval ${PREFIX} pipenv run invenio fixtures create --pid_type acac ${DATA_PATH}/acq_accounts.json --append eval ${PREFIX} pipenv run invenio index reindex -t acac --yes-i-know -eval ${PREFIX} pipenv run invenio index run -c 4 --raise-on-error +eval ${PREFIX} pipenv run invenio index run -c 4 --raise-on-error # # OAI configuration info_msg "OAI configuration:" eval ${PREFIX} pipenv run invenio oaiharvester initconfig ${DATA_PATH}/oaisources.yml -if $DEPLOYMENT +if ${DEPLOYMENT} then # start oai harvesting asynchrone: beats must be running info_msg "Start OAI harvesting asynchrone" @@ -324,7 +333,15 @@ else info_msg "For ebooks harvesting run:" echo -e "\tpipenv run invenio oaiharvester harvest -n ebooks -a max=100 -q" fi -date +if ${ES_MAPPING} +then + info_msg "ES mappings:" + pipenv run invenio utils dump_es_mappings -o tmp/mapping_after_setup.txt + error_msg "ES mapping differences:" + diff --suppress-common-lines --side-by-side tmp/mapping_before_setup.txt tmp/mapping_after_setup.txt +fi + +date success_msg "Perfect ${PROGRAM}! See you soon…" exit 0 diff --git a/tests/api/test_external_services.py b/tests/api/test_external_services.py index 1cb7e8f02f..89e59e6e17 100644 --- a/tests/api/test_external_services.py +++ b/tests/api/test_external_services.py @@ -26,7 +26,6 @@ from utils import VerifyRecordPermissionPatch, get_json, to_relative_url from rero_ils.modules.documents.api import Document -from rero_ils.modules.documents.views import create_publication_statement @pytest.mark.external @@ -59,15 +58,6 @@ def test_documents_get(client, document): assert res.status_code == 200 data = get_json(res) document = document.replace_refs() - publisher_statements = [] - for provision_activity in document.get('provisionActivity', []): - publication_statement = create_publication_statement( - provision_activity - ).get('default') - if publication_statement: - publisher_statements.append(publication_statement) - if publisher_statements: - document['publisherStatement'] = publisher_statements assert data['hits']['hits'][0]['metadata'] == \ document.replace_refs().dumps() diff --git a/tests/ui/circ_policies/test_circ_policies_mapping.py b/tests/ui/circ_policies/test_circ_policies_mapping.py index a4ae7057ca..03265bbb6d 100644 --- a/tests/ui/circ_policies/test_circ_policies_mapping.py +++ b/tests/ui/circ_policies/test_circ_policies_mapping.py @@ -22,9 +22,22 @@ from rero_ils.modules.circ_policies.api import CircPoliciesSearch, CircPolicy -def test_circ_policies_search_mapping( - app, circulation_policies -): +def test_circ_policy_es_mapping(es_clear, db, org_martigny, + circ_policy_martigny_data_tmp): + """Test circulation policy elasticsearch mapping.""" + search = CircPoliciesSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + CircPolicy.create( + circ_policy_martigny_data_tmp, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_circ_policies_search_mapping(app, circulation_policies): """Test circulation policy search mapping.""" search = CircPoliciesSearch() @@ -40,18 +53,3 @@ def test_circ_policies_search_mapping( pids = [r.pid for r in search.query( 'match', name='temporary').source(['pid']).scan()] assert 'cipo3' in pids - - -def test_circ_policy_es_mapping(es, db, org_martigny, - circ_policy_martigny_data_tmp): - """Test circulation policy elasticsearch mapping.""" - search = CircPoliciesSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - CircPolicy.create( - circ_policy_martigny_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/documents/test_documents_api.py b/tests/ui/documents/test_documents_api.py index 0175202dc3..21b64a04b7 100644 --- a/tests/ui/documents/test_documents_api.py +++ b/tests/ui/documents/test_documents_api.py @@ -33,10 +33,10 @@ def test_document_create(db, document_data_tmp): ptty = Document.create(document_data_tmp, delete_pid=True) assert ptty == document_data_tmp assert ptty.get('pid') == '1' - assert ptty.dumps()['editionStatement'][0]['_text'] == { - 'chi-hani': '第3版 / 曾令良主编', - 'default': 'Di 3 ban / Zeng Lingliang zhu bian' - } + assert ptty.dumps()['editionStatement'][0]['_text'] == [ + {'language': 'default', 'value': 'Di 3 ban / Zeng Lingliang zhu bian'}, + {'language': 'chi-hani', 'value': '第3版 / 曾令良主编'} + ] ptty = Document.get_record_by_pid('1') assert ptty == document_data_tmp diff --git a/tests/ui/documents/test_documents_mapping.py b/tests/ui/documents/test_documents_mapping.py index cc7087297c..daf5bf0df9 100644 --- a/tests/ui/documents/test_documents_mapping.py +++ b/tests/ui/documents/test_documents_mapping.py @@ -23,9 +23,22 @@ from rero_ils.modules.documents.api import Document, DocumentsSearch -def test_document_search_mapping( - app, document_records -): +def test_document_es_mapping(es, db, org_martigny, + document_data_ref, item_lib_martigny, person): + """Test document elasticsearch mapping.""" + search = DocumentsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Document.create( + document_data_ref, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_document_search_mapping(app, document_records): """Test document search mapping.""" search = DocumentsSearch() @@ -55,18 +68,3 @@ def test_document_search_mapping( query = MultiMatch(query='Chamber of Secrets', fields=['title.eng']) c = search.query(query).count() assert c == 1 - - -def test_document_es_mapping(db, org_martigny, - document_data_tmp, item_lib_martigny): - """Test document elasticsearch mapping.""" - search = DocumentsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Document.create( - document_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/item_types/test_item_types_mapping.py b/tests/ui/item_types/test_item_types_mapping.py index 0af5708eb3..de2b95aadb 100644 --- a/tests/ui/item_types/test_item_types_mapping.py +++ b/tests/ui/item_types/test_item_types_mapping.py @@ -38,9 +38,7 @@ def test_item_type_es_mapping(es_clear, db, org_martigny, item_type_data_tmp): assert mapping == get_mapping(search.Meta.index) -def test_item_types_search_mapping( - app, item_types_records -): +def test_item_types_search_mapping(app, item_types_records): """Test item type search mapping.""" search = ItemTypesSearch() diff --git a/tests/ui/items/test_items_api.py b/tests/ui/items/test_items_api.py index 2c1063f7f6..34f79827fa 100644 --- a/tests/ui/items/test_items_api.py +++ b/tests/ui/items/test_items_api.py @@ -24,6 +24,22 @@ from rero_ils.modules.items.api import Item, ItemsSearch, item_id_fetcher +def test_item_es_mapping(document, loc_public_martigny, + item_type_standard_martigny, + item_lib_martigny_data_tmp): + """Test item elasticsearch mapping.""" + search = ItemsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Item.create( + item_lib_martigny_data_tmp, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + def test_item_organisation_pid(client, org_martigny, item_lib_martigny): """Test organisation pid has been added during the indexing.""" search = ItemsSearch() @@ -40,20 +56,21 @@ def test_item_item_location_retriever(item_lib_martigny, loc_public_martigny, def test_item_get_items_pid_by_document_pid(document, item_lib_martigny): """Test get items by document pid.""" - assert len(list(Item.get_items_pid_by_document_pid(document.pid))) == 1 + assert len(list(Item.get_items_pid_by_document_pid(document.pid))) == 2 -def test_item_create(db, es, item_lib_martigny_data_tmp, - item_lib_martigny): +def test_item_create(item_lib_martigny_data_tmp, item_lib_martigny): """Test itemanisation creation.""" item = Item.create(item_lib_martigny_data_tmp, delete_pid=True) del item['holding'] assert item == item_lib_martigny_data_tmp - assert item.get('pid') == '1' + # we have used item_lib_martigny_data_tmp two times -> pid == 2 + assert item.get('pid') == '2' assert item.can_delete item = Item.get_record_by_pid('1') del item['holding'] + item_lib_martigny_data_tmp['pid'] = '1' assert item == item_lib_martigny_data_tmp fetched_pid = item_id_fetcher(item.id, item) @@ -69,19 +86,3 @@ def test_item_can_delete(item_lib_martigny): """Test can delete""" assert item_lib_martigny.get_links_to_me() == {} assert item_lib_martigny.can_delete - - -def test_item_es_mapping(es, db, document, loc_public_martigny, - item_type_standard_martigny, - item_lib_martigny_data_tmp): - """Test item elasticsearch mapping.""" - search = ItemsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Item.create( - item_lib_martigny_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/libraries/test_libraries_mapping.py b/tests/ui/libraries/test_libraries_mapping.py index 91222894e4..3aa20bfdc2 100644 --- a/tests/ui/libraries/test_libraries_mapping.py +++ b/tests/ui/libraries/test_libraries_mapping.py @@ -22,13 +22,23 @@ from rero_ils.modules.libraries.api import LibrariesSearch, Library -def test_libraries_search_mapping( - app, libraries_records -): +def test_library_es_mapping(es_clear, db, lib_martigny_data, org_martigny): + """Test library elasticsearch mapping.""" + search = LibrariesSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Library.create( + lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) + assert mapping == get_mapping(search.Meta.index) + + +def test_libraries_search_mapping(app, libraries_records): """Test library search mapping.""" search = LibrariesSearch() - c = search.query('query_string', query='library').count() + c = search.query( + 'query_string', query='Fully Library Restricted Space' + ).count() assert c == 4 c = search.query('query_string', query='bibliothèque').count() @@ -46,13 +56,3 @@ def test_libraries_search_mapping( pids = [r.pid for r in search.query( 'match', name='Sion').source(['pid']).scan()] assert 'lib4' in pids - - -def test_library_es_mapping(es_clear, db, lib_martigny_data, org_martigny): - """Test library elasticsearch mapping.""" - search = LibrariesSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Library.create( - lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/loans/test_loans_api.py b/tests/ui/loans/test_loans_api.py index 101dc5200d..c092c2b173 100644 --- a/tests/ui/loans/test_loans_api.py +++ b/tests/ui/loans/test_loans_api.py @@ -28,6 +28,13 @@ from rero_ils.modules.loans.utils import get_default_loan_duration +def test_loan_es_mapping(es_clear, db): + """Test loans elasticsearch mapping.""" + search = current_circulation.loan_search + mapping = get_mapping(search.Meta.index) + assert mapping == get_mapping(search.Meta.index) + + def test_loans_create(loan_pending_martigny): """Test loan creation.""" assert loan_pending_martigny.get('state') == 'PENDING' @@ -44,10 +51,3 @@ def test_loans_elements(loan_pending_martigny, item_lib_fully): del new_loan['transaction_location_pid'] assert get_default_loan_duration(new_loan) == \ get_default_loan_duration(loan_pending_martigny) - - -def test_loan_es_mapping(es_clear, db): - """Test loans elasticsearch mapping.""" - search = current_circulation.loan_search - mapping = get_mapping(search.Meta.index) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/locations/conftest.py b/tests/ui/locations/conftest.py new file mode 100644 index 0000000000..174d82a433 --- /dev/null +++ b/tests/ui/locations/conftest.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Common pytest libraries.""" + +import pytest + + +@pytest.yield_fixture(scope='module') +def locations_records( + loc_public_martigny, + loc_restricted_martigny, + loc_public_saxon, + loc_restricted_saxon, + loc_public_fully, + loc_restricted_fully, + loc_public_sion, + loc_restricted_sion, + loc_online_martigny, + loc_online_saxon, + loc_online_fully, + loc_online_sion, + loc_online_aproz +): + """Locations for test mapping.""" + pass diff --git a/tests/ui/locations/test_location_mapping.py b/tests/ui/locations/test_location_mapping.py new file mode 100644 index 0000000000..3980179852 --- /dev/null +++ b/tests/ui/locations/test_location_mapping.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Libraries elasticsearch mapping tests.""" + +from utils import get_mapping + +from rero_ils.modules.locations.api import Location, LocationsSearch + + +def test_location_es_mapping(es_clear, db, loc_public_martigny_data, + lib_martigny, org_martigny): + """Test library elasticsearch mapping.""" + search = LocationsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Location.create( + loc_public_martigny_data, dbcommit=True, reindex=True, delete_pid=True) + new_mapping = get_mapping(search.Meta.index) + assert mapping == new_mapping + + +def test_location_search_mapping(app, locations_records): + """Test library search mapping.""" + search = LocationsSearch() + + c = search.query('match', code='MARTIGNY-PUBLIC').count() + assert c == 1 + + c = search.query('match', code='SAXON-PUBLIC').count() + assert c == 1 diff --git a/tests/ui/locations/test_locations_api.py b/tests/ui/locations/test_locations_api.py index 4cecb6ef1b..51b9f9c1e2 100644 --- a/tests/ui/locations/test_locations_api.py +++ b/tests/ui/locations/test_locations_api.py @@ -27,6 +27,21 @@ from rero_ils.modules.locations.api import location_id_fetcher as fetcher +def test_location_es_mapping(es_clear, db, lib_martigny, + loc_public_martigny_data): + """Test location elasticsearch mapping.""" + search = LocationsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Location.create( + loc_public_martigny_data, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + def test_location_create(db, es_clear, loc_public_martigny_data, lib_martigny, loc_online_martigny): """Test location creation.""" @@ -56,20 +71,6 @@ def test_location_organisation_pid(org_martigny, loc_public_martigny): assert location.organisation.pid == org_martigny.pid -def test_location_es_mapping(es, db, lib_martigny, loc_public_martigny_data): - """Test location elasticsearch mapping.""" - search = LocationsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Location.create( - loc_public_martigny_data, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) - - def test_location_can_delete(loc_public_martigny): """Test can delete.""" assert loc_public_martigny.get_links_to_me() == {} diff --git a/tests/ui/notifications/test_notifications_api.py b/tests/ui/notifications/test_notifications_api.py index 903a730efb..68147115da 100644 --- a/tests/ui/notifications/test_notifications_api.py +++ b/tests/ui/notifications/test_notifications_api.py @@ -29,19 +29,6 @@ notification_id_fetcher as fetcher -def test_notification_organisation_pid( - org_martigny, notification_availability_martigny): - """Test organisation pid has been added during the indexing.""" - search = NotificationsSearch() - pid = notification_availability_martigny.get('pid') - notification = next(search.filter('term', pid=pid).scan()) - assert notification.organisation.pid == org_martigny.pid - - # test notification can_delete - assert notification_availability_martigny.get_links_to_me() == {} - assert notification_availability_martigny.can_delete - - def test_notification_es_mapping( dummy_notification, loan_validated_martigny): """Test notification elasticsearch mapping.""" @@ -63,6 +50,19 @@ def test_notification_es_mapping( assert mapping == get_mapping(search.Meta.index) +def test_notification_organisation_pid( + app, org_martigny, notification_availability_martigny): + """Test organisation pid has been added during the indexing.""" + search = NotificationsSearch() + pid = notification_availability_martigny.get('pid') + notification = next(search.filter('term', pid=pid).scan()) + assert notification.organisation.pid == org_martigny.pid + + # test notification can_delete + assert notification_availability_martigny.get_links_to_me() == {} + assert notification_availability_martigny.can_delete + + def test_notification_create( es_clear, dummy_notification, loan_validated_martigny, mailbox): """Test notification creation.""" diff --git a/tests/ui/patron_types/conftest.py b/tests/ui/patron_types/conftest.py index 7b0e998ad0..66cfb268b6 100644 --- a/tests/ui/patron_types/conftest.py +++ b/tests/ui/patron_types/conftest.py @@ -22,7 +22,6 @@ @pytest.yield_fixture(scope='module') def patron_types_records( - patron_type_children_martigny, patron_type_adults_martigny, patron_type_youngsters_sion, patron_type_grown_sion diff --git a/tests/ui/patron_types/test_patron_types_mapping.py b/tests/ui/patron_types/test_patron_types_mapping.py index 41acc84260..d7aa7a02f5 100644 --- a/tests/ui/patron_types/test_patron_types_mapping.py +++ b/tests/ui/patron_types/test_patron_types_mapping.py @@ -22,36 +22,36 @@ from rero_ils.modules.patron_types.api import PatronType, PatronTypesSearch -def test_patron_types_search_mapping( - app, patron_types_records -): +def test_patron_type_es_mapping( + org_martigny, patron_type_children_martigny_data): + """Test patron types es mapping.""" + search = PatronTypesSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + PatronType.create( + patron_type_children_martigny_data, + dbcommit=True, + reindex=True, + delete_pid=False + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_patron_types_search_mapping(app, patron_types_records): """Test patron type search mapping.""" search = PatronTypesSearch() c = search.query('query_string', query='patrons').count() + # there is one more result from test_patron_type_es_mapping function assert c == 4 c = search.query('match', name='patrons').count() assert c == 0 c = search.query('match', name='children').count() + # there is one more result from test_patron_type_es_mapping function assert c == 1 pids = [r.pid for r in search.query( 'match', name='children').source(['pid']).scan()] assert 'ptty1' in pids - - -def test_patron_type_es_mapping(es_clear, db, org_martigny, - patron_type_children_martigny_data): - """Test patron types es mapping.""" - search = PatronTypesSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - PatronType.create( - patron_type_children_martigny_data, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/patrons/test_patrons_api.py b/tests/ui/patrons/test_patrons_api.py index f8785c3e14..9ba3a9e00f 100644 --- a/tests/ui/patrons/test_patrons_api.py +++ b/tests/ui/patrons/test_patrons_api.py @@ -27,6 +27,14 @@ patron_id_fetcher +def test_patron_es_mapping( + roles, es_clear, lib_martigny, librarian_martigny_data_tmp): + """Test patron elasticsearch mapping.""" + search = PatronsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping == get_mapping(search.Meta.index) + + def test_patron_create(app, roles, librarian_martigny_data_tmp, mailbox): """Test Patron creation.""" @@ -111,11 +119,3 @@ def test_user_librarian_can_delete(librarian_martigny): """Test can delete a librarian.""" assert librarian_martigny.get_links_to_me() == {} assert librarian_martigny.can_delete - - -def test_patron_es_mapping( - roles, es_clear, lib_martigny, librarian_martigny_data_tmp): - """Test patron elasticsearch mapping.""" - search = PatronsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/patrons/test_patrons_mapping.py b/tests/ui/patrons/test_patrons_mapping.py index 045a5a67f5..462a1beb3e 100644 --- a/tests/ui/patrons/test_patrons_mapping.py +++ b/tests/ui/patrons/test_patrons_mapping.py @@ -19,12 +19,19 @@ from utils import get_mapping -from rero_ils.modules.patrons.api import Patron, PatronsSearch +from rero_ils.modules.patrons.api import PatronsSearch -def test_patron_search_mapping( - app, patrons_records, librarian_saxon_no_email -): +def test_patron_es_mapping( + roles, es_clear, lib_martigny, librarian_martigny_data_tmp): + """Test patron elasticsearch mapping.""" + search = PatronsSearch() + mapping = get_mapping(search.Meta.index) + # TODO: create of an patron + assert mapping == get_mapping(search.Meta.index) + + +def test_patron_search_mapping(app, patrons_records, librarian_saxon_no_email): """Test patron search mapping.""" search = PatronsSearch() @@ -46,11 +53,3 @@ def test_patron_search_mapping( pids = [r.pid for r in search.query( 'match', first_name='Eléna').source(['pid']).scan()] assert librarian_saxon_no_email.pid in pids - - -def test_patron_es_mapping( - roles, es_clear, lib_martigny, librarian_martigny_data_tmp): - """Test patron elasticsearch mapping.""" - search = PatronsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/persons/test_persons_mapping.py b/tests/ui/persons/test_persons_mapping.py index a294d3c11e..7ae92f01a7 100644 --- a/tests/ui/persons/test_persons_mapping.py +++ b/tests/ui/persons/test_persons_mapping.py @@ -22,9 +22,21 @@ from rero_ils.modules.persons.api import Person, PersonsSearch -def test_persons_search_mapping( - app, person -): +def test_person_es_mapping(es_clear, db, person_data_tmp): + """Test mef elasticsearch mapping.""" + search = PersonsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Person.create( + person_data_tmp, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_persons_search_mapping(app, person): """Test Mef Persons search mapping.""" search = PersonsSearch() @@ -57,17 +69,3 @@ def test_persons_search_mapping( **{'gnd.variant_name_for_person': 'Arnoudt'}).\ count() assert c == 1 - - -def test_person_es_mapping(es_clear, db, person_data_tmp): - """Test mef elasticsearch mapping.""" - search = PersonsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Person.create( - person_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/unit/test_documents_dojson.py b/tests/unit/test_documents_dojson.py index b4beecdfd7..c637a5779f 100644 --- a/tests/unit/test_documents_dojson.py +++ b/tests/unit/test_documents_dojson.py @@ -725,9 +725,9 @@ def test_marc21_to_provisionActivity_unknown_place_2_agents(): ], 'startDate': '1968' }] - assert create_publication_statement(data.get('provisionActivity')[0]) == { - 'default': '[Lieu de publication non identifié] : Labor, Nathan, 1968' - } + assert create_publication_statement(data.get('provisionActivity')[0]) == [ + '[Lieu de publication non identifié] : Labor, Nathan, 1968' + ] def test_marc21_to_provisionActivity_3_places_dann_2_agents(): @@ -780,10 +780,9 @@ def test_marc21_to_provisionActivity_3_places_dann_2_agents(): ], 'startDate': '1975' }] - assert create_publication_statement(data.get('provisionActivity')[0]) == { - 'default': 'Hamm (Westf.) ; [dann] Herzberg ; [dann] Nordhausen : ' + - 'T. Bautz, 1975-' - } + assert create_publication_statement(data.get('provisionActivity')[0]) == [ + 'Hamm (Westf.) ; [dann] Herzberg ; [dann] Nordhausen : T. Bautz, 1975-' + ] def test_marc21_to_provisionActivity_2_places_1_agent(): @@ -831,9 +830,9 @@ def test_marc21_to_provisionActivity_2_places_1_agent(): ], 'startDate': '1966' }] - assert create_publication_statement(data.get('provisionActivity')[0]) == { - 'default': '[Louvain] ; [Paris] : [éditeur non identifié], [1966]' - } + assert create_publication_statement(data.get('provisionActivity')[0]) == [ + '[Louvain] ; [Paris] : [éditeur non identifié], [1966]' + ] def test_marc21_to_provisionActivity_1_place_1_agent_reprint_date(): @@ -924,8 +923,9 @@ def test_marc21_to_provisionActivity_1_place_1_agent_uncertain_date(): 'note': 'Date(s) incertaine(s) ou inconnue(s)', 'startDate': '1941' }] - assert create_publication_statement(data.get('provisionActivity')[0]) == { - 'default': 'Aurillac : Impr. moderne, [1941?]'} + assert create_publication_statement(data.get('provisionActivity')[0]) == [ + 'Aurillac : Impr. moderne, [1941?]' + ] def test_marc21_to_provisionActivity_1_place_1_agent_chi_hani(): @@ -993,10 +993,10 @@ def test_marc21_to_provisionActivity_1_place_1_agent_chi_hani(): ], 'startDate': '2017' }] - assert create_publication_statement(data.get('provisionActivity')[0]) == { - 'chi-hani': '北京 : 北京大学出版社, 2017', - 'default': 'Beijing : Beijing da xue chu ban she, 2017' - } + assert create_publication_statement(data.get('provisionActivity')[0]) == [ + 'Beijing : Beijing da xue chu ban she, 2017', + '北京 : 北京大学出版社, 2017' + ] marc21xml = """