ES: fix mapping

* Adds missing mappings to jsonschemas.
* Corrects mapping tests.
* Corrects _text creation and mappings.
* Deletes publisherStatement.

Co-Authored-by: Peter Weber <[email protected]>

.gitignore

@@ -80,3 +80,7 @@ cov.xml
 # these json files are generated during the setup
 rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json
 rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json
+rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json
+
+# ES mapping files
+tmp/

MANIFEST.in

@@ -35,6 +35,9 @@ recursive-include rero_ils *.po *.pot *.mo
 recursive-include docker *.cfg *.conf *.crt *.ini *.key *.pem *.sh
 exclude rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json
 exclude rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json
+# ES mappings
+exclude mapping_after_setup.txt
+exclude mapping_before_setup.txt
 
 # added by check_manifest.py
 include *.base

rero_ils/modules/cli.py

@@ -1324,3 +1324,23 @@ def run_tests(self, tests):
     dependency_tests.run_tests(tests)
 
     sys.exit(dependency_tests.missing + dependency_tests.not_found)
+
+
+@utils.command('dump_es_mappings')
+@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False)
+@click.option('-o', '--outfile', 'outfile', type=click.File('w'), default=None)
+@with_appcontext
+def dump_es_mappings(verbose, outfile):
+    """Dumps ES mappings."""
+    click.secho('Dump ES mappings:', fg='green')
+    aliases = current_search.client.indices.get_alias('*')
+    mappings = current_search.client.indices.get_mapping()
+    for alias in sorted(aliases):
+        if alias[0] != '.':
+            mapping = mappings.get(alias, {}).get('mappings')
+            click.secho('{alias}'.format(alias=alias), fg='green')
+            if verbose or not outfile:
+                print(json.dumps(mapping, indent=2))
+            if outfile:
+                json.dump(mapping, outfile, indent=2)
+                outfile.write('\n')

rero_ils/modules/documents/listener.py

@@ -17,7 +17,6 @@
 
 """Signals connector for Document."""
 
-from .views import create_publication_statement
 from ..documents.api import DocumentsSearch
 from ..holdings.api import Holding, HoldingsSearch
 from ..items.api import ItemsSearch
@@ -78,17 +77,6 @@ def enrich_document_data(sender, json=None, record=None, index=None,
         if holdings:
             json['holdings'] = holdings
 
-        # provisionActivity
-        publisher_statements = []
-        for provision_activity in record.get('provisionActivity', []):
-            publication_statement = create_publication_statement(
-                provision_activity
-            ).get('default')
-            if publication_statement:
-                publisher_statements.append(publication_statement)
-        if publisher_statements:
-            json['publisherStatement'] = publisher_statements
-
         # MEF person ES index update
         authors = []
         for author in json.get('authors', []):

rero_ils/modules/documents/mappings/v6/documents/document.json

@@ -194,8 +194,11 @@
           "properties": {
             "_text": {
               "properties": {
-                "default": {
+                "value": {
                   "type": "text"
+                },
+                "language": {
+                  "type": "keyword"
                 }
               }
             },
@@ -245,9 +248,6 @@
             "statement": {
               "type": "object",
               "properties": {
-                "type": {
-                  "type": "text"
-                },
                 "label": {
                   "type": "object",
                   "properties": {
@@ -259,6 +259,9 @@
                       "type": "keyword"
                     }
                   }
+                },
+                "type": {
+                  "type": "keyword"
                 }
               }
             },
@@ -274,19 +277,21 @@
               "type": "date",
               "format": "yyyy"
             },
+            "type": {
+              "type": "text"
+            },
             "_text": {
               "properties": {
-                "default": {
+                "value": {
                   "type": "text"
+                },
+                "language": {
+                  "type": "keyword"
                 }
               }
             }
           }
         },
-        "publisherStatement": {
-          "type": "keyword",
-          "index": false
-        },
         "extent": {
           "type": "text",
           "analyzer": "global_lowercase_asciifolding"
@@ -435,6 +440,9 @@
             "pid": {
               "type": "keyword"
             },
+            "available": {
+              "type": "boolean"
+            },
             "call_number": {
               "type": "keyword"
             },

rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html

@@ -48,12 +48,12 @@ <h1 class="col-sm-10 col-md-11">{{ record.title }}</h1>
 
         <!-- EDITION STATEMENT-->
         {% if record.editionStatement %}
-        {{ dl_list(_('Edition'), record.editionStatement | edition_format) }}
+        {{ dl_list(_('Edition'), record.editionStatement|edition_format) }}
         {% endif %}
 
         <!-- PUBLICATION STATEMENT -->
         {% for provision_activity in record.provisionActivity %}
-        {{ dl_dict(_(provision_activity.type), provision_activity|create_publication_statement ) }}
+        {{ dl_list(_(provision_activity.type), provision_activity|create_publication_statement ) }}
         {% endfor %}
 
         <!-- COPYRIGHT DATE -->

rero_ils/modules/documents/utils.py

@@ -83,10 +83,14 @@ def publication_statement_text(provision_activity):
         statement_type = statement['type']
 
     # date field: remove ';' and append
+    statement_text = []
     for key, value in statement_with_language.items():
         value = remove_trailing_punctuation(value)
-        statement_with_language[key] = value
-    return statement_with_language
+        if key == 'default':
+            statement_text.insert(0, {'value': value, 'language': key})
+        else:
+            statement_text.append({'value': value, 'language': key})
+    return statement_text
 
 
 def series_format_text(serie):
@@ -101,7 +105,6 @@ def series_format_text(serie):
 
 def edition_format_text(edition):
     """Format edition for _text."""
-    edition_with_language = {'default': ''}
     designations = edition.get('editionDesignation', [])
     responsibilities = edition.get('responsibility', [])
     designation_output = {}
@@ -115,13 +118,17 @@ def edition_format_text(edition):
         value = responsibility.get('value', '')
         responsibility_output[language] = value
 
+    edition_text = []
     for key, value in designation_output.items():
-        output_value = remove_trailing_punctuation(
+        value = remove_trailing_punctuation(
             '{designation} / {responsibility}'.format(
-                designation=value,
+                designation=designation_output.get(key),
                 responsibility=responsibility_output.get(key, ''),
             )
         )
-        edition_with_language[key] = output_value
+        if key == 'default':
+            edition_text.insert(0, {'value': value, 'language': key})
+        else:
+            edition_text.append({'value': value, 'language': key})
 
-    return edition_with_language
+    return edition_text

rero_ils/modules/documents/views.py

@@ -295,10 +295,8 @@ def edition_format(editions):
     for edition in editions:
         languages = edition_format_text(edition)
         if languages:
-            output.append(languages['default'])
-            del languages['default']
-            for key, value in languages.items():
-                output.append(value)
+            for edition_text in languages:
+                output.append(edition_text.get('value'))
     return output
 
 
@@ -431,4 +429,8 @@ def document_availability(document_pid):
 @blueprint.app_template_filter()
 def create_publication_statement(provision_activity):
     """Create publication statement from place, agent and date values."""
-    return publication_statement_text(provision_activity)
+    output = []
+    publication_texts = publication_statement_text(provision_activity)
+    for publication_text in publication_texts:
+        output.append(publication_text.get('value'))
+    return output

rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json

@@ -44,6 +44,13 @@
         "status": {
           "type": "keyword"
         },
+        "organisation": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "_created": {
           "type": "date"
         },
@@ -53,4 +60,4 @@
       }
     }
   }
-}
+}

rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json

@@ -50,6 +50,20 @@
             }
           }
         },
+        "library": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
+        "organisation": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "_created": {
           "type": "date"
         },
@@ -59,4 +73,4 @@
       }
     }
   }
-}
+}

rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json

@@ -52,6 +52,9 @@
                 },
                 "period": {
                   "type": "keyword"
+                },
+                "data": {
+                  "type": "long"
                 }
               }
             },

rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json

@@ -40,6 +40,9 @@
         "end_date": {
           "type": "date"
         },
+        "extension_count": {
+          "type": "long"
+        },
         "state": {
           "type": "keyword"
         },

rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json

@@ -37,6 +37,13 @@
             }
           }
         },
+        "organisation": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "_created": {
           "type": "date"
         },
@@ -46,4 +53,4 @@
       }
     }
   }
-}
+}

rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json

@@ -34,6 +34,13 @@
             }
           }
         },
+        "organisation": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "_created": {
           "type": "date"
         },

rero_ils/modules/vendors/mappings/v6/vendors/vendor-v0.0.1.json

@@ -23,6 +23,9 @@
         "vendor_name": {
           "type": "keyword"
         },
+        "communication_language": {
+          "type": "keyword"
+        },
         "currency": {
           "type": "keyword"
         },
@@ -36,6 +39,34 @@
             }
           }
         },
+        "default_contact": {
+          "properties": {
+            "*.json": {
+              "type": "kewword"
+            },
+            "country": {
+              "type": "kewword"
+            },
+            "email": {
+              "type": "keyword"
+            },
+            "phone": {
+              "type": "keyword"
+            },
+            "postal_code": {
+              "type": "keyword"
+            },
+            "street": {
+              "type": "text"
+            }
+          }
+        },
+        "note": {
+          "type": "text"
+        },
+        "website": {
+          "type": "text"
+        },
         "_created": {
           "type": "date"
         },

run-tests.sh

@@ -105,4 +105,4 @@ if [ "$1" = "external" ]
 fi
 
 success_msg "Perfect ${PROGRAM}! See you soon…"
-exit 0
+exit 0