all: fix and improve several things

* Adds a deletion_date field to JSON schema and mapping in order to track when a MEF contribution record is deleted. * Changes `max_result_window` to 20000 as in config.py. * Changes `number_of_shards` to 8 for better ES performance. * Changes `number_of_replicas` to 1 for better ES performance and redundancy. * Implements lazy loading for patrons fixtures. * Adds sorting to `get_items_pid_by_holding_pid` so we always get the same order. * Adds sorting to `get_holdings_by_document_item_type` so we always get the same order. * Creates vendor fixtures before holdings fixtures in setup because holdings have $ref's to vendors. * Implements parallel indexing during setup. * Corrects `data/pid_dependencies.json` used for pid dependency checking. Co-Authored-by: Peter Weber <[email protected]>
rero · Dec 11, 2020 · 5a331a7 · 5a331a7
1 parent a22543e
commit 5a331a7
Show file tree

Hide file tree

Showing 38 changed files with 446 additions and 156 deletions.
diff --git a/.github/workflows/continuous-integration-test.yml b/.github/workflows/continuous-integration-test.yml
@@ -34,14 +34,14 @@ jobs:
       with:
         version: 1.0.10
 
-    - name: Use Cache
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cache/pip
-          ~/.cache/pypoetry/virtualenvs
-          ~/.npm
-        key: ${{ runner.os }}-pip-venv-${{ hashFiles('**/poetry.lock') }}
+    # - name: Use Cache
+    #   uses: actions/cache@v2
+    #   with:
+    #     path: |
+    #       ~/.cache/pip
+    #       ~/.cache/pypoetry/virtualenvs
+    #       ~/.npm
+    #     key: ${{ runner.os }}-pip-venv-${{ hashFiles('**/poetry.lock') }}
 
     - name: Bootstrap
       if: ${{ matrix.dependencies == 'dev' }}

diff --git a/data/pid_dependencies.json b/data/pid_dependencies.json
@@ -22,8 +22,8 @@
     ]
   },
   {
-    "name": "document",
-    "filename": "documents_big.json"
+    "name": "pickup_location",
+    "filename": "locations.json"
   },
   {
     "name": "item_type",
@@ -40,6 +40,15 @@
     "dependencies": [
       {
         "name": "organisation"
+      },
+      {
+        "name": "library_exceptions",
+        "optional": "True",
+        "sublist": [
+          {
+            "name": "library"
+          }
+        ]
       }
     ]
   },
@@ -64,11 +73,43 @@
       },
       {
         "name": "library",
-        "ref": "libraries",
         "optional": "True"
       }
     ]
   },
+  {
+    "name": "vendor",
+    "filename": "vendors.json",
+    "dependencies": [
+      {
+        "name": "organisation"
+      }
+    ]
+  },
+  {
+    "name": "patron",
+    "filename": "users.json",
+    "dependencies": [
+      {
+        "name": "library",
+        "optional": "True"
+      },
+      {
+        "name": "patron",
+        "optional": "True",
+        "sublist": [
+          {
+            "name": "patron_type",
+            "optional": "True"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "name": "document",
+    "filename": "documents_big.json"
+  },
   {
     "name": "holding",
     "filename": "holdings_big.json",
@@ -92,14 +133,122 @@
       {
         "name": "location"
       },
+      {
+        "name": "document"
+      },
       {
         "name": "item_type"
       },
       {
-        "name": "document"
+        "name": "holding",
+        "optional": "True"
+      },
+      {
+        "name": "organisation",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "patterns",
+    "filename": "patterns.json"
+  },
+  {
+    "name": "budget",
+    "filename": "budgets.json",
+    "dependencies": [
+      {
+        "name": "organisation",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "collection",
+    "filename": "collections.json",
+    "dependencies": [
+      {
+        "name": "librarie",
+        "ref": "libraries",
+        "optional": "True"
+      },
+      {
+        "name": "item",
+        "ref": "items",
+        "optional": "True"
+      },
+      {
+        "name": "organisation"
+      }
+    ]
+  },
+  {
+    "name": "acq_account",
+    "filename": "acq_accounts.json",
+    "dependencies": [
+      {
+        "name": "budget"
+      },
+      {
+        "name": "library"
+      },
+      {
+        "name": "organisation",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "template",
+    "filename": "templates.json",
+    "dependencies": [
+      {
+        "name": "organisation"
+      },
+      {
+        "name": "creator",
+        "ref": "patron"
+      }
+    ]
+  },
+  {
+    "name": "loan",
+    "filename": "loans.json",
+    "dependencies": [
+      {
+        "name": "item",
+        "optional": "True"
+      },
+      {
+        "name": "patron",
+        "optional": "True"
       },
       {
-        "name": "holding"
+        "name": "document",
+        "optional": "True"
+      },
+      {
+        "name": "organisation",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "local_field",
+    "filename": "local_fields.json",
+    "dependencies": [
+      {
+        "name": "organisation",
+        "optional": "True"
+      },
+      {
+        "name": "parent",
+        "optional": "True",
+        "refs": {
+          "document": "documents",
+          "item": "items",
+          "holding": "holdings"
+        }
       }
     ]
   }