Merge pull request #144 from jpmckinney/unique-perf

fix: unique_ids has exponential running time if no ID fields
OpenDataServices · Jun 28, 2024 · 3623ecf · 3623ecf
2 parents f96d8c9 + 9ad32b8
commit 3623ecf
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 27 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -7,35 +7,29 @@ jobs:
       matrix:
         cove: [ 'oc4ids' , 'ocds' ]
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v1
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: 3.8
+        python-version: '3.10'
+        cache: pip
+        cache-dependency-path: '**/requirements*.txt'
     - name: InstallCommon
       run: |
         sudo apt-get install gettext translate-toolkit
 
-    - name: oc4ids
-      if: matrix.cove == 'oc4ids'
+    - name: Clone
       run: |
-        git clone https://github.com/open-contracting/cove-oc4ids.git
-        git clone https://github.com/open-contracting/lib-cove-oc4ids.git
-
-    - name: ocds
-      if: matrix.cove == 'ocds'
-      run: |
-        git clone https://github.com/open-contracting/cove-ocds.git
-        git clone https://github.com/open-contracting/lib-cove-ocds.git
+        git clone https://github.com/open-contracting/cove-${{ matrix.cove }}.git
+        git clone https://github.com/open-contracting/lib-cove-${{ matrix.cove }}.git
 
     - name: Install
       run: |
-        pip install -r cove-${{ matrix.cove }}/requirements.txt
-        pip install -r cove-${{ matrix.cove }}/requirements_dev.txt
         pip install -r requirements_dev.txt
+        pip install -r cove-${{ matrix.cove }}/requirements_dev.txt
         # Make sure we're using local libs rather than one brought in
         # via requirements.
-        pip install -e .
         pip install -e ./lib-cove-${{ matrix.cove }}/
+        pip install -e .
         pip list
 
     - name: Compile Messages
@@ -51,13 +45,6 @@ jobs:
         cd cove-${{ matrix.cove }}
         DJANGO_SETTINGS_MODULE=cove_project.settings py.test
 
-    - name: lib-cove-ocds requirements
-      if: matrix.cove == 'ocds'
-      run: |
-        # Upgrade Django for lib-cove-ocds tests, as they rely on some details
-        # of entity quoting that have changed
-        pip install --upgrade Django
-
     - name: Test cove lib instance
       run: |
         cd lib-cove-${{ matrix.cove }}

diff --git a/libcove/lib/common.py b/libcove/lib/common.py
@@ -89,28 +89,29 @@ def unique_ids(validator, ui, instance, schema, id_names=["id"]):
     if ui and validator.is_type(instance, "array"):
         non_unique_ids = set()
         all_ids = set()
+        uniq_has_run = False
+
         for item in instance:
             try:
                 item_ids = tuple(item.get(id_name) for id_name in id_names)
             except AttributeError:
                 # if item is not a dict
                 item_ids = None
             if item_ids and all(
-                item_id is not None
-                and not isinstance(item_id, list)
-                and not isinstance(item_id, dict)
+                item_id is not None and not isinstance(item_id, (dict, list))
                 for item_id in item_ids
             ):
                 if item_ids in all_ids:
                     non_unique_ids.add(item_ids)
                 all_ids.add(item_ids)
-            else:
+            elif not uniq_has_run:
                 if not uniq(instance):
                     msg = "Array has non-unique elements"
                     err = ValidationError(msg, instance=instance)
                     err.error_id = "uniqueItems_no_ids"
                     yield err
                     return
+                uniq_has_run = True
 
         for non_unique_id in sorted(non_unique_ids):
             if len(id_names) == 1: