ES: fix mapping

* Adds missing mappings to json schemas. * Corrects mapping tests. Co-Authored-by: Peter Weber <[email protected]>
rero · Jan 21, 2020 · 99c667a · 99c667a
1 parent f61ad51
commit 99c667a
Show file tree

Hide file tree

Showing 27 changed files with 495 additions and 179 deletions.
diff --git a/.gitignore b/.gitignore
@@ -80,3 +80,7 @@ cov.xml
 # these json files are generated during the setup
 rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json
 rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json
+rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json
+
+# ES mapping files
+mapping*.txt
diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py
@@ -1324,3 +1324,23 @@ def run_tests(self, tests):
     dependency_tests.run_tests(tests)
 
     sys.exit(dependency_tests.missing + dependency_tests.not_found)
+
+
+@utils.command('dump_es_mappings')
+@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False)
+@click.option('-o', '--outfile', 'outfile', type=click.File('w'), default=None)
+@with_appcontext
+def dump_es_mappings(verbose, outfile):
+    """Dumps ES mappings."""
+    click.secho('Dump ES mappings:', fg='green')
+    aliases = current_search.client.indices.get_alias('*')
+    mappings = current_search.client.indices.get_mapping()
+    for alias in sorted(aliases):
+        if alias[0] != '.':
+            mapping = mappings.get(alias, {}).get('mappings')
+            click.secho('{alias}'.format(alias=alias), fg='green')
+            if verbose or not outfile:
+                print(json.dumps(mapping, indent=2))
+            if outfile:
+                json.dump(mapping, outfile, indent=2)
+                outfile.write('\n')
diff --git a/...ments/mappings/v6/documents/document.json → ...nts/mappings/src/document-v0.0.1_src.json b/...ments/mappings/v6/documents/document.json → ...nts/mappings/src/document-v0.0.1_src.json
@@ -24,7 +24,7 @@
     }
   },
   "mappings": {
-    "document": {
+    "document-v0.0.1": {
       "date_detection": false,
       "numeric_detection": false,
       "properties": {
@@ -193,11 +193,7 @@
           "type": "object",
           "properties": {
             "_text": {
-              "properties": {
-                "default": {
-                  "type": "text"
-                }
-              }
+              "$ref": "#definitions/textLanguageScripts"
             },
             "editionDesignation": {
               "type": "object",
@@ -275,11 +271,7 @@
               "format": "yyyy"
             },
             "_text": {
-              "properties": {
-                "default": {
-                  "type": "text"
-                }
-              }
+              "$ref": "#definitions/textLanguageScripts"
             }
           }
         },
@@ -502,6 +494,84 @@
         "harvested": {
           "type": "boolean"
         },
+        "ara-arab": {
+          "type": "text"
+        },
+        "bel-cyrl": {
+          "type": "text"
+        },
+        "chi-hani": {
+          "type": "text"
+        },
+        "chi-zyyy": {
+          "type": "text"
+        },
+        "chu-cyrl": {
+          "type": "text"
+        },
+        "grc-grek": {
+          "type": "text"
+        },
+        "gre-grek": {
+          "type": "text"
+        },
+        "heb-hebr": {
+          "type": "text"
+        },
+        "jpn-hani": {
+          "type": "text"
+        },
+        "jpn-jpan": {
+          "type": "text"
+        },
+        "kor-kore": {
+          "type": "text"
+        },
+        "lad-hebr": {
+          "type": "text"
+        },
+        "mac-cyrl": {
+          "type": "text"
+        },
+        "per-arab": {
+          "type": "text"
+        },
+        "rus-cyrl": {
+          "type": "text"
+        },
+        "srp-cyrl": {
+          "type": "text"
+        },
+        "ukr-cyrl": {
+          "type": "text"
+        },
+        "yid-hebr": {
+          "type": "text"
+        },
+        "und-arab": {
+          "type": "text"
+        },
+        "und-cyrl": {
+          "type": "text"
+        },
+        "und-grek": {
+          "type": "text"
+        },
+        "und-hani": {
+          "type": "text"
+        },
+        "und-hebr": {
+          "type": "text"
+        },
+        "und-jpan": {
+          "type": "text"
+        },
+        "und-kore": {
+          "type": "text"
+        },
+        "und-zyyy": {
+          "type": "text"
+        },
         "_created": {
           "type": "date"
         },
@@ -510,5 +580,92 @@
         }
       }
     }
+  },
+  "definitions": {
+    "textLanguageScripts": {
+      "properties": {
+        "default": {
+          "type": "text"
+        },
+        "ara-arab": {
+          "type": "text"
+        },
+        "bel-cyrl": {
+          "type": "text"
+        },
+        "chi-hani": {
+          "type": "text"
+        },
+        "chi-zyyy": {
+          "type": "text"
+        },
+        "chu-cyrl": {
+          "type": "text"
+        },
+        "grc-grek": {
+          "type": "text"
+        },
+        "gre-grek": {
+          "type": "text"
+        },
+        "heb-hebr": {
+          "type": "text"
+        },
+        "jpn-hani": {
+          "type": "text"
+        },
+        "jpn-jpan": {
+          "type": "text"
+        },
+        "kor-kore": {
+          "type": "text"
+        },
+        "lad-hebr": {
+          "type": "text"
+        },
+        "mac-cyrl": {
+          "type": "text"
+        },
+        "per-arab": {
+          "type": "text"
+        },
+        "rus-cyrl": {
+          "type": "text"
+        },
+        "srp-cyrl": {
+          "type": "text"
+        },
+        "ukr-cyrl": {
+          "type": "text"
+        },
+        "yid-hebr": {
+          "type": "text"
+        },
+        "und-arab": {
+          "type": "text"
+        },
+        "und-cyrl": {
+          "type": "text"
+        },
+        "und-grek": {
+          "type": "text"
+        },
+        "und-hani": {
+          "type": "text"
+        },
+        "und-hebr": {
+          "type": "text"
+        },
+        "und-jpan": {
+          "type": "text"
+        },
+        "und-kore": {
+          "type": "text"
+        },
+        "und-zyyy": {
+          "type": "text"
+        }
+      }
+    }
   }
 }
diff --git a/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json b/rero_ils/modules/fees/mappings/v6/fees/fee-v0.0.1.json
@@ -44,6 +44,13 @@
         "status": {
           "type": "keyword"
         },
+        "organisation": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "_created": {
           "type": "date"
         },
@@ -53,4 +60,4 @@
       }
     }
   }
-}
+}
diff --git a/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json b/rero_ils/modules/holdings/mappings/v6/holdings/holding-v0.0.1.json
@@ -50,6 +50,20 @@
             }
           }
         },
+        "library": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
+        "organisation": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "_created": {
           "type": "date"
         },
@@ -59,4 +73,4 @@
       }
     }
   }
-}
+}
diff --git a/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json b/rero_ils/modules/libraries/mappings/v6/libraries/library-v0.0.1.json
@@ -52,6 +52,9 @@
                 },
                 "period": {
                   "type": "keyword"
+                },
+                "data": {
+                  "type": "long"
                 }
               }
             },

diff --git a/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json b/rero_ils/modules/loans/mappings/v6/loans/loan-ils-v0.0.1.json
@@ -40,6 +40,9 @@
         "end_date": {
           "type": "date"
         },
+        "extension_count": {
+          "type": "long"
+        },
         "state": {
           "type": "keyword"
         },

diff --git a/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json b/rero_ils/modules/locations/mappings/v6/locations/location-v0.0.1.json
@@ -37,6 +37,13 @@
             }
           }
         },
+        "organisation": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "_created": {
           "type": "date"
         },
@@ -46,4 +53,4 @@
       }
     }
   }
-}
+}
diff --git a/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json b/rero_ils/modules/notifications/mappings/v6/notifications/notification-v0.0.1.json
@@ -34,6 +34,13 @@
             }
           }
         },
+        "organisation": {
+          "properties": {
+            "pid": {
+              "type": "keyword"
+            }
+          }
+        },
         "_created": {
           "type": "date"
         },

diff --git a/run-tests.sh b/run-tests.sh
@@ -59,6 +59,7 @@ msg "PROGRAM: ${PROGRAM}"
 # compile json files (resolve $ref)
 pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json
 pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json
+pipenv run invenio utils compile_json ./rero_ils/modules/documents/mappings/src/document-v0.0.1_src.json -o ./rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json
 
 if [ $# -eq 0 ]
     then
@@ -105,4 +106,4 @@ if [ "$1" = "external" ]
 fi
 
 success_msg "Perfect ${PROGRAM}! See you soon…"
-exit 0
+exit 0
diff --git a/scripts/bootstrap b/scripts/bootstrap
@@ -149,6 +149,7 @@ info_msg "Compile JSON files to resolve \$ref"
 echo
 pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json
 pipenv run invenio utils compile_json ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1_src.json -o ./rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json
+pipenv run invenio utils compile_json ./rero_ils/modules/documents/mappings/src/document-v0.0.1_src.json -o ./rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json
 
 success_msg "${PROGRAM} finished!"
 exit 0