From 99c667acc97e6bdcd0f06e6afb5c70b79cadd368 Mon Sep 17 00:00:00 2001 From: Peter Weber Date: Mon, 20 Jan 2020 15:36:51 +0100 Subject: [PATCH] ES: fix mapping * Adds missing mappings to json schemas. * Corrects mapping tests. Co-Authored-by: Peter Weber --- .gitignore | 4 + rero_ils/modules/cli.py | 20 ++ .../document-v0.0.1_src.json} | 179 ++++++++++++++++-- .../fees/mappings/v6/fees/fee-v0.0.1.json | 9 +- .../mappings/v6/holdings/holding-v0.0.1.json | 16 +- .../mappings/v6/libraries/library-v0.0.1.json | 3 + .../mappings/v6/loans/loan-ils-v0.0.1.json | 3 + .../v6/locations/location-v0.0.1.json | 9 +- .../v6/notifications/notification-v0.0.1.json | 7 + run-tests.sh | 3 +- scripts/bootstrap | 1 + scripts/setup | 26 ++- .../test_circ_policies_mapping.py | 34 ++-- tests/ui/documents/test_documents_mapping.py | 34 ++-- .../ui/item_types/test_item_types_mapping.py | 4 +- tests/ui/items/test_items_api.py | 32 ++-- tests/ui/libraries/test_libraries_mapping.py | 28 +-- tests/ui/loans/test_loans_api.py | 14 +- tests/ui/locations/conftest.py | 40 ++++ tests/ui/locations/test_location_mapping.py | 45 +++++ tests/ui/locations/test_locations_api.py | 29 +-- .../notifications/test_notifications_api.py | 26 +-- tests/ui/patron_types/conftest.py | 1 - .../patron_types/test_patron_types_mapping.py | 36 ++-- tests/ui/patrons/test_patrons_api.py | 16 +- tests/ui/patrons/test_patrons_mapping.py | 23 ++- tests/ui/persons/test_persons_mapping.py | 32 ++-- 27 files changed, 495 insertions(+), 179 deletions(-) rename rero_ils/modules/documents/mappings/{v6/documents/document.json => src/document-v0.0.1_src.json} (79%) create mode 100644 tests/ui/locations/conftest.py create mode 100644 tests/ui/locations/test_location_mapping.py diff --git a/.gitignore b/.gitignore index 0d48520cbe..e3bb1796b9 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,7 @@ cov.xml # these json files are generated during the setup rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json +rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json + +# ES mapping files +mapping*.txt diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py index 92f0519537..b222cd89b0 100644 --- a/rero_ils/modules/cli.py +++ b/rero_ils/modules/cli.py @@ -1324,3 +1324,23 @@ def run_tests(self, tests): dependency_tests.run_tests(tests) sys.exit(dependency_tests.missing + dependency_tests.not_found) + + +@utils.command('dump_es_mappings') +@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) +@click.option('-o', '--outfile', 'outfile', type=click.File('w'), default=None) +@with_appcontext +def dump_es_mappings(verbose, outfile): + """Dumps ES mappings.""" + click.secho('Dump ES mappings:', fg='green') + aliases = current_search.client.indices.get_alias('*') + mappings = current_search.client.indices.get_mapping() + for alias in sorted(aliases): + if alias[0] != '.': + mapping = mappings.get(alias, {}).get('mappings') + click.secho('{alias}'.format(alias=alias), fg='green') + if verbose or not outfile: + print(json.dumps(mapping, indent=2)) + if outfile: + json.dump(mapping, outfile, indent=2) + outfile.write('\n') diff --git a/rero_ils/modules/documents/mappings/v6/documents/document.json b/rero_ils/modules/documents/mappings/src/document-v0.0.1_src.json similarity index 79% rename from rero_ils/modules/documents/mappings/v6/documents/document.json rename to rero_ils/modules/documents/mappings/src/document-v0.0.1_src.json index 5be70bd240..5a4fc41f40 100644 --- a/rero_ils/modules/documents/mappings/v6/documents/document.json +++ b/rero_ils/modules/documents/mappings/src/document-v0.0.1_src.json @@ -24,7 +24,7 @@ } }, "mappings": { - "document": { + "document-v0.0.1": { "date_detection": false, "numeric_detection": false, "properties": { @@ -193,11 +193,7 @@ "type": "object", "properties": { "_text": { - "properties": { - "default": { - "type": "text" - } - } + "$ref": "#definitions/textLanguageScripts" }, "editionDesignation": { "type": "object", @@ -275,11 +271,7 @@ "format": "yyyy" }, "_text": { - "properties": { - "default": { - "type": "text" - } - } + "$ref": "#definitions/textLanguageScripts" } } }, @@ -502,6 +494,84 @@ "harvested": { "type": "boolean" }, + "ara-arab": { + "type": "text" + }, + "bel-cyrl": { + "type": "text" + }, + "chi-hani": { + "type": "text" + }, + "chi-zyyy": { + "type": "text" + }, + "chu-cyrl": { + "type": "text" + }, + "grc-grek": { + "type": "text" + }, + "gre-grek": { + "type": "text" + }, + "heb-hebr": { + "type": "text" + }, + "jpn-hani": { + "type": "text" + }, + "jpn-jpan": { + "type": "text" + }, + "kor-kore": { + "type": "text" + }, + "lad-hebr": { + "type": "text" + }, + "mac-cyrl": { + "type": "text" + }, + "per-arab": { + "type": "text" + }, + "rus-cyrl": { + "type": "text" + }, + "srp-cyrl": { + "type": "text" + }, + "ukr-cyrl": { + "type": "text" + }, + "yid-hebr": { + "type": "text" + }, + "und-arab": { + "type": "text" + }, + "und-cyrl": { + "type": "text" + }, + "und-grek": { + "type": "text" + }, + "und-hani": { + "type": "text" + }, + "und-hebr": { + "type": "text" + }, + "und-jpan": { + "type": "text" + }, + "und-kore": { + "type": "text" + }, + "und-zyyy": { + "type": "text" + }, "_created": { "type": "date" }, @@ -510,5 +580,92 @@ } } } + }, + "definitions": { + "textLanguageScripts": { + "properties": { + "default": { + "type": "text" + }, + "ara-arab": { + "type": "text" + }, + "bel-cyrl": { + "type": "text" + }, + "chi-hani": { + "type": "text" + }, + "chi-zyyy": { + "type": "text" + }, + "chu-cyrl": { + "type": "text" + }, + "grc-grek": { + "type": "text" + }, + "gre-grek": { + "type": "text" + }, + "heb-hebr": { + "type": "text" + }, + "jpn-hani": { + "type": "text" + }, + "jpn-jpan": { + "type": "text" + }, + "kor-kore": { + "type": "text" + }, + "lad-hebr": { + "type": "text" + }, + "mac-cyrl": { + "type": "text" + }, + "per-arab": { + "type": "text" + }, + "rus-cyrl": { + "type": "text" + }, + "srp-cyrl": { + "type": "text" + }, + "ukr-cyrl": { + "type": "text" + }, + "yid-hebr": { + "type": "text" + }, + "und-arab": { + "type": "text" + }, + "und-cyrl": { + "type": "text" + }, + "und-grek": { + "type": "text" + }, + "und-hani": { + "type": "text" + }, + "und-hebr": { + "type": "text" + }, + "und-jpan": { + "type": "text" + }, + "und-kore": { + "type": "text" + }, + "und-zyyy": { + "type": "text" + } + } + } } } diff --git a/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json b/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json index 5c397be915..0dc70462a0 100644 --- a/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json +++ b/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json @@ -44,6 +44,13 @@ "status": { "type": "keyword" }, + "organisation": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, "_created": { "type": "date" }, @@ -53,4 +60,4 @@ } } } -} \ No newline at end of file +} diff --git a/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json b/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json index 49af392d13..642e90c718 100644 --- a/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json +++ b/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json @@ -50,6 +50,20 @@ } } }, + "library": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, + "organisation": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, "_created": { "type": "date" }, @@ -59,4 +73,4 @@ } } } -} \ No newline at end of file +} diff --git a/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json b/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json index 8eeac9270f..2e4ee2ce4c 100644 --- a/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json +++ b/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json @@ -52,6 +52,9 @@ }, "period": { "type": "keyword" + }, + "data": { + "type": "long" } } }, diff --git a/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json b/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json index 4deb4a2e60..1afa464c45 100644 --- a/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json +++ b/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json @@ -40,6 +40,9 @@ "end_date": { "type": "date" }, + "extension_count": { + "type": "long" + }, "state": { "type": "keyword" }, diff --git a/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json b/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json index 92a8885ac9..5c4a49f452 100644 --- a/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json +++ b/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json @@ -37,6 +37,13 @@ } } }, + "organisation": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, "_created": { "type": "date" }, @@ -46,4 +53,4 @@ } } } -} \ No newline at end of file +} diff --git a/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json b/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json index a84cb1552c..a467b0f14d 100644 --- a/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json +++ b/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json @@ -34,6 +34,13 @@ } } }, + "organisation": { + "properties": { + "pid": { + "type": "keyword" + } + } + }, "_created": { "type": "date" }, diff --git a/run-tests.sh b/run-tests.sh index f934469ca7..5a051c3c25 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -59,6 +59,7 @@ msg "PROGRAM: ${PROGRAM}" # compile json files (resolve $ref) pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json +pipenv run invenio utils compile_json ./rero_ils/modules/documents/mappings/src/document-v0.0.1_src.json -o ./rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json if [ $# -eq 0 ] then @@ -105,4 +106,4 @@ if [ "$1" = "external" ] fi success_msg "Perfect ${PROGRAM}! See you soon…" -exit 0 \ No newline at end of file +exit 0 diff --git a/scripts/bootstrap b/scripts/bootstrap index 99f510c643..ba61c3ccd1 100755 --- a/scripts/bootstrap +++ b/scripts/bootstrap @@ -149,6 +149,7 @@ info_msg "Compile JSON files to resolve \$ref" echo pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json +pipenv run invenio utils compile_json ./rero_ils/modules/documents/mappings/src/document-v0.0.1_src.json -o ./rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json success_msg "${PROGRAM} finished!" exit 0 diff --git a/scripts/setup b/scripts/setup index d6c6049ae4..531f2864d5 100755 --- a/scripts/setup +++ b/scripts/setup @@ -75,12 +75,13 @@ STOP_EXECUTION=true CREATE_LAZY="" DONT_STOP="" PREFIX="" +ES_MAPPING=false # Displays program name msg "PROGRAM: ${PROGRAM}" # options may be followed by one colon to indicate they have a required argument -if ! options=$(getopt -o dsb -l -t deployment,create_items_holdings_small,create_items_holdings_big,lazy,pursue,time,data_path: -- "$@") +if ! options=$(getopt -o dsb -l -t deployment,create_items_holdings_small,create_items_holdings_big,lazy,pursue,data_path,time,es-mapping: -- "$@") then # something went wrong, getopt will put out an error message for us exit 1 @@ -97,6 +98,7 @@ do -p|--pursue) DONT_STOP="--dont-stop" ;; -D|--data_path) DATA_PATH=$2 ;; -t|--time) PREFIX="time" ;; # Get time for all commands + -m|--es-mapping) ES_MAPPING=true ;; (--) shift; break;; (-*) error_msg+exit "$0: Unrecognized option $1";; (*) break;; @@ -115,7 +117,7 @@ fi if $CREATE_ITEMS_HOLDINGS_SMALL || $CREATE_ITEMS_HOLDINGS_BIG then - if $DEPLOYMENT + if ${DEPLOYMENT} then error_msg+exit "Choose deployment option only" fi @@ -147,6 +149,12 @@ eval ${PREFIX} pipenv run invenio index queue init info_msg "Delete invenio_circulations index" eval ${PREFIX} pipenv run invenio index delete loans-loan-v1.0.0 --force --yes-i-know +if ${ES_MAPPING} +then + info_msg "ES mappings:" + pipenv run invenio utils dump_es_mappings -o mapping_before_setup.txt +fi + # Create admin role to restrict access info_msg "Create admin role to restrict access" eval ${PREFIX} pipenv run invenio roles create admin @@ -231,7 +239,7 @@ eval ${PREFIX} pipenv run invenio fixtures import_users ${DATA_PATH}/users.json # eval ${PREFIX} pipenv run invenio utils marc21tojson ${DATA_PATH}/documents_big.xml ${DATA_PATH}/documents_big.json ${DATA_PATH}/documents_big_errors.xml -v -r # eval ${PREFIX} pipenv run invenio utils marc21tojson ${DATA_PATH}/documents_small.xml ${DATA_PATH}/documents_small.json ${DATA_PATH}/documents_small_errors.xml -v -r -if $DEPLOYMENT +if ${DEPLOYMENT} then DOCUMENTS=${DATA_PATH}/documents_big.json ITEMS=${DATA_PATH}/items_big.json @@ -295,7 +303,7 @@ eval ${PREFIX} pipenv run invenio fixtures create_loans --fee ${DATA_PATH}/loans # # OAI configuration info_msg "OAI configuration:" eval ${PREFIX} pipenv run invenio oaiharvester initconfig ${DATA_PATH}/oaisources.yml -if $DEPLOYMENT +if ${DEPLOYMENT} then # start oai harvesting asynchrone: beats must be running info_msg "Start OAI harvesting asynchrone" @@ -304,7 +312,15 @@ else info_msg "For ebooks harvesting run:" echo -e "\tpipenv run invenio oaiharvester harvest -n ebooks -a max=100 -q" fi -date +if ${ES_MAPPING} +then + info_msg "ES mappings:" + pipenv run invenio utils dump_es_mappings -o mapping_after_setup.txt + error_msg "ES mapping differences:" + diff --suppress-common-lines --side-by-side mapping_before_setup.txt mapping_after_setup.txt +fi + +date success_msg "Perfect ${PROGRAM}! See you soon…" exit 0 diff --git a/tests/ui/circ_policies/test_circ_policies_mapping.py b/tests/ui/circ_policies/test_circ_policies_mapping.py index a4ae7057ca..03265bbb6d 100644 --- a/tests/ui/circ_policies/test_circ_policies_mapping.py +++ b/tests/ui/circ_policies/test_circ_policies_mapping.py @@ -22,9 +22,22 @@ from rero_ils.modules.circ_policies.api import CircPoliciesSearch, CircPolicy -def test_circ_policies_search_mapping( - app, circulation_policies -): +def test_circ_policy_es_mapping(es_clear, db, org_martigny, + circ_policy_martigny_data_tmp): + """Test circulation policy elasticsearch mapping.""" + search = CircPoliciesSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + CircPolicy.create( + circ_policy_martigny_data_tmp, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_circ_policies_search_mapping(app, circulation_policies): """Test circulation policy search mapping.""" search = CircPoliciesSearch() @@ -40,18 +53,3 @@ def test_circ_policies_search_mapping( pids = [r.pid for r in search.query( 'match', name='temporary').source(['pid']).scan()] assert 'cipo3' in pids - - -def test_circ_policy_es_mapping(es, db, org_martigny, - circ_policy_martigny_data_tmp): - """Test circulation policy elasticsearch mapping.""" - search = CircPoliciesSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - CircPolicy.create( - circ_policy_martigny_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/documents/test_documents_mapping.py b/tests/ui/documents/test_documents_mapping.py index cc7087297c..daf5bf0df9 100644 --- a/tests/ui/documents/test_documents_mapping.py +++ b/tests/ui/documents/test_documents_mapping.py @@ -23,9 +23,22 @@ from rero_ils.modules.documents.api import Document, DocumentsSearch -def test_document_search_mapping( - app, document_records -): +def test_document_es_mapping(es, db, org_martigny, + document_data_ref, item_lib_martigny, person): + """Test document elasticsearch mapping.""" + search = DocumentsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Document.create( + document_data_ref, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_document_search_mapping(app, document_records): """Test document search mapping.""" search = DocumentsSearch() @@ -55,18 +68,3 @@ def test_document_search_mapping( query = MultiMatch(query='Chamber of Secrets', fields=['title.eng']) c = search.query(query).count() assert c == 1 - - -def test_document_es_mapping(db, org_martigny, - document_data_tmp, item_lib_martigny): - """Test document elasticsearch mapping.""" - search = DocumentsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Document.create( - document_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/item_types/test_item_types_mapping.py b/tests/ui/item_types/test_item_types_mapping.py index 0af5708eb3..de2b95aadb 100644 --- a/tests/ui/item_types/test_item_types_mapping.py +++ b/tests/ui/item_types/test_item_types_mapping.py @@ -38,9 +38,7 @@ def test_item_type_es_mapping(es_clear, db, org_martigny, item_type_data_tmp): assert mapping == get_mapping(search.Meta.index) -def test_item_types_search_mapping( - app, item_types_records -): +def test_item_types_search_mapping(app, item_types_records): """Test item type search mapping.""" search = ItemTypesSearch() diff --git a/tests/ui/items/test_items_api.py b/tests/ui/items/test_items_api.py index 2c1063f7f6..2dcedd4501 100644 --- a/tests/ui/items/test_items_api.py +++ b/tests/ui/items/test_items_api.py @@ -24,6 +24,22 @@ from rero_ils.modules.items.api import Item, ItemsSearch, item_id_fetcher +def test_item_es_mapping(es_clear, db, document, loc_public_martigny, + item_type_standard_martigny, + item_lib_martigny_data_tmp): + """Test item elasticsearch mapping.""" + search = ItemsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Item.create( + item_lib_martigny_data_tmp, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + def test_item_organisation_pid(client, org_martigny, item_lib_martigny): """Test organisation pid has been added during the indexing.""" search = ItemsSearch() @@ -69,19 +85,3 @@ def test_item_can_delete(item_lib_martigny): """Test can delete""" assert item_lib_martigny.get_links_to_me() == {} assert item_lib_martigny.can_delete - - -def test_item_es_mapping(es, db, document, loc_public_martigny, - item_type_standard_martigny, - item_lib_martigny_data_tmp): - """Test item elasticsearch mapping.""" - search = ItemsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Item.create( - item_lib_martigny_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/libraries/test_libraries_mapping.py b/tests/ui/libraries/test_libraries_mapping.py index 91222894e4..3aa20bfdc2 100644 --- a/tests/ui/libraries/test_libraries_mapping.py +++ b/tests/ui/libraries/test_libraries_mapping.py @@ -22,13 +22,23 @@ from rero_ils.modules.libraries.api import LibrariesSearch, Library -def test_libraries_search_mapping( - app, libraries_records -): +def test_library_es_mapping(es_clear, db, lib_martigny_data, org_martigny): + """Test library elasticsearch mapping.""" + search = LibrariesSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Library.create( + lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) + assert mapping == get_mapping(search.Meta.index) + + +def test_libraries_search_mapping(app, libraries_records): """Test library search mapping.""" search = LibrariesSearch() - c = search.query('query_string', query='library').count() + c = search.query( + 'query_string', query='Fully Library Restricted Space' + ).count() assert c == 4 c = search.query('query_string', query='bibliothèque').count() @@ -46,13 +56,3 @@ def test_libraries_search_mapping( pids = [r.pid for r in search.query( 'match', name='Sion').source(['pid']).scan()] assert 'lib4' in pids - - -def test_library_es_mapping(es_clear, db, lib_martigny_data, org_martigny): - """Test library elasticsearch mapping.""" - search = LibrariesSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Library.create( - lib_martigny_data, dbcommit=True, reindex=True, delete_pid=True) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/loans/test_loans_api.py b/tests/ui/loans/test_loans_api.py index 101dc5200d..c092c2b173 100644 --- a/tests/ui/loans/test_loans_api.py +++ b/tests/ui/loans/test_loans_api.py @@ -28,6 +28,13 @@ from rero_ils.modules.loans.utils import get_default_loan_duration +def test_loan_es_mapping(es_clear, db): + """Test loans elasticsearch mapping.""" + search = current_circulation.loan_search + mapping = get_mapping(search.Meta.index) + assert mapping == get_mapping(search.Meta.index) + + def test_loans_create(loan_pending_martigny): """Test loan creation.""" assert loan_pending_martigny.get('state') == 'PENDING' @@ -44,10 +51,3 @@ def test_loans_elements(loan_pending_martigny, item_lib_fully): del new_loan['transaction_location_pid'] assert get_default_loan_duration(new_loan) == \ get_default_loan_duration(loan_pending_martigny) - - -def test_loan_es_mapping(es_clear, db): - """Test loans elasticsearch mapping.""" - search = current_circulation.loan_search - mapping = get_mapping(search.Meta.index) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/locations/conftest.py b/tests/ui/locations/conftest.py new file mode 100644 index 0000000000..174d82a433 --- /dev/null +++ b/tests/ui/locations/conftest.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Common pytest libraries.""" + +import pytest + + +@pytest.yield_fixture(scope='module') +def locations_records( + loc_public_martigny, + loc_restricted_martigny, + loc_public_saxon, + loc_restricted_saxon, + loc_public_fully, + loc_restricted_fully, + loc_public_sion, + loc_restricted_sion, + loc_online_martigny, + loc_online_saxon, + loc_online_fully, + loc_online_sion, + loc_online_aproz +): + """Locations for test mapping.""" + pass diff --git a/tests/ui/locations/test_location_mapping.py b/tests/ui/locations/test_location_mapping.py new file mode 100644 index 0000000000..3980179852 --- /dev/null +++ b/tests/ui/locations/test_location_mapping.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Libraries elasticsearch mapping tests.""" + +from utils import get_mapping + +from rero_ils.modules.locations.api import Location, LocationsSearch + + +def test_location_es_mapping(es_clear, db, loc_public_martigny_data, + lib_martigny, org_martigny): + """Test library elasticsearch mapping.""" + search = LocationsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Location.create( + loc_public_martigny_data, dbcommit=True, reindex=True, delete_pid=True) + new_mapping = get_mapping(search.Meta.index) + assert mapping == new_mapping + + +def test_location_search_mapping(app, locations_records): + """Test library search mapping.""" + search = LocationsSearch() + + c = search.query('match', code='MARTIGNY-PUBLIC').count() + assert c == 1 + + c = search.query('match', code='SAXON-PUBLIC').count() + assert c == 1 diff --git a/tests/ui/locations/test_locations_api.py b/tests/ui/locations/test_locations_api.py index 4cecb6ef1b..51b9f9c1e2 100644 --- a/tests/ui/locations/test_locations_api.py +++ b/tests/ui/locations/test_locations_api.py @@ -27,6 +27,21 @@ from rero_ils.modules.locations.api import location_id_fetcher as fetcher +def test_location_es_mapping(es_clear, db, lib_martigny, + loc_public_martigny_data): + """Test location elasticsearch mapping.""" + search = LocationsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Location.create( + loc_public_martigny_data, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + def test_location_create(db, es_clear, loc_public_martigny_data, lib_martigny, loc_online_martigny): """Test location creation.""" @@ -56,20 +71,6 @@ def test_location_organisation_pid(org_martigny, loc_public_martigny): assert location.organisation.pid == org_martigny.pid -def test_location_es_mapping(es, db, lib_martigny, loc_public_martigny_data): - """Test location elasticsearch mapping.""" - search = LocationsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Location.create( - loc_public_martigny_data, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) - - def test_location_can_delete(loc_public_martigny): """Test can delete.""" assert loc_public_martigny.get_links_to_me() == {} diff --git a/tests/ui/notifications/test_notifications_api.py b/tests/ui/notifications/test_notifications_api.py index 903a730efb..68147115da 100644 --- a/tests/ui/notifications/test_notifications_api.py +++ b/tests/ui/notifications/test_notifications_api.py @@ -29,19 +29,6 @@ notification_id_fetcher as fetcher -def test_notification_organisation_pid( - org_martigny, notification_availability_martigny): - """Test organisation pid has been added during the indexing.""" - search = NotificationsSearch() - pid = notification_availability_martigny.get('pid') - notification = next(search.filter('term', pid=pid).scan()) - assert notification.organisation.pid == org_martigny.pid - - # test notification can_delete - assert notification_availability_martigny.get_links_to_me() == {} - assert notification_availability_martigny.can_delete - - def test_notification_es_mapping( dummy_notification, loan_validated_martigny): """Test notification elasticsearch mapping.""" @@ -63,6 +50,19 @@ def test_notification_es_mapping( assert mapping == get_mapping(search.Meta.index) +def test_notification_organisation_pid( + app, org_martigny, notification_availability_martigny): + """Test organisation pid has been added during the indexing.""" + search = NotificationsSearch() + pid = notification_availability_martigny.get('pid') + notification = next(search.filter('term', pid=pid).scan()) + assert notification.organisation.pid == org_martigny.pid + + # test notification can_delete + assert notification_availability_martigny.get_links_to_me() == {} + assert notification_availability_martigny.can_delete + + def test_notification_create( es_clear, dummy_notification, loan_validated_martigny, mailbox): """Test notification creation.""" diff --git a/tests/ui/patron_types/conftest.py b/tests/ui/patron_types/conftest.py index 7b0e998ad0..66cfb268b6 100644 --- a/tests/ui/patron_types/conftest.py +++ b/tests/ui/patron_types/conftest.py @@ -22,7 +22,6 @@ @pytest.yield_fixture(scope='module') def patron_types_records( - patron_type_children_martigny, patron_type_adults_martigny, patron_type_youngsters_sion, patron_type_grown_sion diff --git a/tests/ui/patron_types/test_patron_types_mapping.py b/tests/ui/patron_types/test_patron_types_mapping.py index 41acc84260..d7aa7a02f5 100644 --- a/tests/ui/patron_types/test_patron_types_mapping.py +++ b/tests/ui/patron_types/test_patron_types_mapping.py @@ -22,36 +22,36 @@ from rero_ils.modules.patron_types.api import PatronType, PatronTypesSearch -def test_patron_types_search_mapping( - app, patron_types_records -): +def test_patron_type_es_mapping( + org_martigny, patron_type_children_martigny_data): + """Test patron types es mapping.""" + search = PatronTypesSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + PatronType.create( + patron_type_children_martigny_data, + dbcommit=True, + reindex=True, + delete_pid=False + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_patron_types_search_mapping(app, patron_types_records): """Test patron type search mapping.""" search = PatronTypesSearch() c = search.query('query_string', query='patrons').count() + # there is one more result from test_patron_type_es_mapping function assert c == 4 c = search.query('match', name='patrons').count() assert c == 0 c = search.query('match', name='children').count() + # there is one more result from test_patron_type_es_mapping function assert c == 1 pids = [r.pid for r in search.query( 'match', name='children').source(['pid']).scan()] assert 'ptty1' in pids - - -def test_patron_type_es_mapping(es_clear, db, org_martigny, - patron_type_children_martigny_data): - """Test patron types es mapping.""" - search = PatronTypesSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - PatronType.create( - patron_type_children_martigny_data, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/patrons/test_patrons_api.py b/tests/ui/patrons/test_patrons_api.py index f8785c3e14..9ba3a9e00f 100644 --- a/tests/ui/patrons/test_patrons_api.py +++ b/tests/ui/patrons/test_patrons_api.py @@ -27,6 +27,14 @@ patron_id_fetcher +def test_patron_es_mapping( + roles, es_clear, lib_martigny, librarian_martigny_data_tmp): + """Test patron elasticsearch mapping.""" + search = PatronsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping == get_mapping(search.Meta.index) + + def test_patron_create(app, roles, librarian_martigny_data_tmp, mailbox): """Test Patron creation.""" @@ -111,11 +119,3 @@ def test_user_librarian_can_delete(librarian_martigny): """Test can delete a librarian.""" assert librarian_martigny.get_links_to_me() == {} assert librarian_martigny.can_delete - - -def test_patron_es_mapping( - roles, es_clear, lib_martigny, librarian_martigny_data_tmp): - """Test patron elasticsearch mapping.""" - search = PatronsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/patrons/test_patrons_mapping.py b/tests/ui/patrons/test_patrons_mapping.py index 045a5a67f5..462a1beb3e 100644 --- a/tests/ui/patrons/test_patrons_mapping.py +++ b/tests/ui/patrons/test_patrons_mapping.py @@ -19,12 +19,19 @@ from utils import get_mapping -from rero_ils.modules.patrons.api import Patron, PatronsSearch +from rero_ils.modules.patrons.api import PatronsSearch -def test_patron_search_mapping( - app, patrons_records, librarian_saxon_no_email -): +def test_patron_es_mapping( + roles, es_clear, lib_martigny, librarian_martigny_data_tmp): + """Test patron elasticsearch mapping.""" + search = PatronsSearch() + mapping = get_mapping(search.Meta.index) + # TODO: create of an patron + assert mapping == get_mapping(search.Meta.index) + + +def test_patron_search_mapping(app, patrons_records, librarian_saxon_no_email): """Test patron search mapping.""" search = PatronsSearch() @@ -46,11 +53,3 @@ def test_patron_search_mapping( pids = [r.pid for r in search.query( 'match', first_name='Eléna').source(['pid']).scan()] assert librarian_saxon_no_email.pid in pids - - -def test_patron_es_mapping( - roles, es_clear, lib_martigny, librarian_martigny_data_tmp): - """Test patron elasticsearch mapping.""" - search = PatronsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping == get_mapping(search.Meta.index) diff --git a/tests/ui/persons/test_persons_mapping.py b/tests/ui/persons/test_persons_mapping.py index a294d3c11e..7ae92f01a7 100644 --- a/tests/ui/persons/test_persons_mapping.py +++ b/tests/ui/persons/test_persons_mapping.py @@ -22,9 +22,21 @@ from rero_ils.modules.persons.api import Person, PersonsSearch -def test_persons_search_mapping( - app, person -): +def test_person_es_mapping(es_clear, db, person_data_tmp): + """Test mef elasticsearch mapping.""" + search = PersonsSearch() + mapping = get_mapping(search.Meta.index) + assert mapping + Person.create( + person_data_tmp, + dbcommit=True, + reindex=True, + delete_pid=True + ) + assert mapping == get_mapping(search.Meta.index) + + +def test_persons_search_mapping(app, person): """Test Mef Persons search mapping.""" search = PersonsSearch() @@ -57,17 +69,3 @@ def test_persons_search_mapping( **{'gnd.variant_name_for_person': 'Arnoudt'}).\ count() assert c == 1 - - -def test_person_es_mapping(es_clear, db, person_data_tmp): - """Test mef elasticsearch mapping.""" - search = PersonsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - Person.create( - person_data_tmp, - dbcommit=True, - reindex=True, - delete_pid=True - ) - assert mapping == get_mapping(search.Meta.index)