Merge branch 'develop' into zm/update-dm-format

.github/workflows/cache.yml

@@ -13,7 +13,7 @@ jobs:
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       REPO: ${{ github.repository }}
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/full.yml

@@ -22,7 +22,7 @@ jobs:
 
   build:
     needs: search_cache
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
@@ -94,7 +94,7 @@ jobs:
 
   rest_api_testing:
     needs: build
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
@@ -160,7 +160,8 @@ jobs:
       - name: Install SDK
         run: |
           pip3 install -r ./tests/python/requirements.txt \
-            -e './cvat-sdk[pytorch]' -e ./cvat-cli
+            -e './cvat-sdk[pytorch]' -e ./cvat-cli \
+            --extra-index-url https://download.pytorch.org/whl/cpu
 
       - name: Running REST API and SDK tests
         id: run_tests
@@ -187,7 +188,7 @@ jobs:
 
   unit_testing:
     needs: build
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
@@ -207,11 +208,10 @@ jobs:
 
       - name: Running OPA tests
         run: |
-          python cvat/apps/iam/rules/tests/generate_tests.py \
-            --output-dir cvat/apps/iam/rules/
+          python cvat/apps/iam/rules/tests/generate_tests.py
 
-          docker compose run --rm -v "$PWD/cvat/apps/iam/rules/:/mnt/rules" \
-            cvat_opa test /mnt/rules
+          docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \
+            cvat_opa test cvat/apps/*/rules
 
       - name: Running unit tests
         env:
@@ -247,7 +247,7 @@ jobs:
 
   e2e_testing:
     needs: build
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:

.github/workflows/helm.yml

@@ -21,7 +21,7 @@ jobs:
       github.event.pull_request.draft == false &&
       !startsWith(github.event.pull_request.title, '[WIP]') &&
       !startsWith(github.event.pull_request.title, '[Dependent]')
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
 

.github/workflows/main.yml

@@ -28,7 +28,7 @@ jobs:
 
   build:
     needs: search_cache
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
@@ -127,7 +127,7 @@ jobs:
 
   rest_api_testing:
     needs: build
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
@@ -170,7 +170,8 @@ jobs:
       - name: Install SDK
         run: |
           pip3 install -r ./tests/python/requirements.txt \
-            -e './cvat-sdk[pytorch]' -e ./cvat-cli
+            -e './cvat-sdk[pytorch]' -e ./cvat-cli \
+            --extra-index-url https://download.pytorch.org/whl/cpu
 
       - name: Run REST API and SDK tests
         id: run_tests
@@ -208,7 +209,7 @@ jobs:
 
   unit_testing:
     needs: build
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
@@ -226,11 +227,10 @@ jobs:
 
       - name: Running OPA tests
         run: |
-          python cvat/apps/iam/rules/tests/generate_tests.py \
-            --output-dir cvat/apps/iam/rules/
+          python cvat/apps/iam/rules/tests/generate_tests.py
 
-          docker compose run --rm -v "$PWD/cvat/apps/iam/rules/:/mnt/rules" \
-            cvat_opa test /mnt/rules
+          docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \
+            cvat_opa test cvat/apps/*/rules
 
       - name: Running unit tests
         env:
@@ -274,7 +274,7 @@ jobs:
 
   e2e_testing:
     needs: build
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
@@ -385,10 +385,17 @@ jobs:
           name: cypress_screenshots_${{ matrix.specs }}
           path: ${{ github.workspace }}/tests/cypress/screenshots
 
+      - name: Uploading cypress videos as an artifact
+        if: failure()
+        uses: actions/[email protected]
+        with:
+          name: cypress_videos_${{ matrix.specs }}
+          path: ${{ github.workspace }}/tests/cypress/videos
+
   publish_dev_images:
     if: github.ref == 'refs/heads/develop'
     needs: [rest_api_testing, unit_testing, e2e_testing]
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
@@ -427,7 +434,7 @@ jobs:
           docker push "${UI_IMAGE_REPO}:dev"
 
   codecov:
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     needs: [unit_testing, e2e_testing, rest_api_testing]
     steps:
       - uses: actions/checkout@v4

.github/workflows/regallint.yml

@@ -6,7 +6,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Setup Regal
-        uses: StyraInc/setup-regal@v0.2.0
+        uses: StyraInc/setup-regal@v1
         with:
-          version: v0.11.0
-      - run: regal lint --format=github cvat/apps/iam/rules
+          version: v0.21.3
+      - run: regal lint --format=github cvat/apps/*/rules

.github/workflows/schedule.yml

@@ -11,7 +11,7 @@ env:
 
 jobs:
   check_updates:
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     env:
       REPO: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -44,7 +44,7 @@ jobs:
 
   build:
     needs: search_cache
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
@@ -107,7 +107,7 @@ jobs:
 
   unit_testing:
     needs: build
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
@@ -153,11 +153,10 @@ jobs:
 
       - name: OPA tests
         run: |
-          python cvat/apps/iam/rules/tests/generate_tests.py \
-            --output-dir cvat/apps/iam/rules/
+          python cvat/apps/iam/rules/tests/generate_tests.py
 
-          docker compose run --rm -v "$PWD/cvat/apps/iam/rules/:/mnt/rules" \
-            cvat_opa test /mnt/rules
+          docker compose run --rm -v "$PWD:/mnt/src:ro" -w /mnt/src \
+            cvat_opa test cvat/apps/*/rules
 
       - name: REST API and SDK tests
         run: |
@@ -190,7 +189,7 @@ jobs:
 
   e2e_testing:
     needs: build
-    runs-on: ubuntu-latest-big
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:

.gitignore

@@ -64,3 +64,5 @@ cvat-core/reports
 # produced by prepare in the root package.json script
 .husky
 
+# produced by cvat/apps/iam/rules/tests/generate_tests.py
+/cvat/apps/*/rules/*_test.gen.rego

.vscode/launch.json

@@ -167,7 +167,6 @@
             "env": {
                 "CVAT_SERVERLESS": "1",
                 "ALLOWED_HOSTS": "*",
-                "IAM_OPA_BUNDLE": "1",
                 "DJANGO_LOG_SERVER_HOST": "localhost",
                 "DJANGO_LOG_SERVER_PORT": "8282"
             },

.vscode/settings.json

@@ -39,5 +39,10 @@
     "python.testing.cwd": "${workspaceFolder}/tests",
     "cSpell.words": [
         "crowdsourcing"
-    ]
+    ],
+    "isort.args":["--profile", "black"],
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.formatOnSave": true
+    }
 }

CHANGELOG.md

@@ -16,6 +16,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!-- scriv-insert-here -->
 
+<a id='changelog-2.12.1'></a>
+## \[2.12.1\] - 2024-04-26
+
+### Fixed
+
+- Formats with the custom `track_id` attribute should import `outside` track shapes properly (e.g. `COCO`, `COCO Keypoints`, `Datumaro`, `PASCAL VOC`)
+  (<https://github.com/opencv/cvat/pull/7669>)
+
+- Inefficient resources fetching in admin panel leading to 504 Gateway Timeout
+  (<https://github.com/cvat-ai/cvat/pull/7767>)
+
+- Optimized memory usage when retrieving annotations by disabling internal Django QuerySet caching
+  (<https://github.com/cvat-ai/cvat/pull/7748>)
+
+- Annotations are not shown on the `0` frame sometimes
+  (<https://github.com/cvat-ai/cvat/pull/7796>)
+
+- Extra requests in PolicyEnforcer when at least one policy is rejected, others are not checked
+  (<https://github.com/cvat-ai/cvat/pull/7803>)
+
+- Project's `updated_date` was not updated after changing annotations in jobs
+  (<https://github.com/cvat-ai/cvat/pull/7808>)
+
 <a id='changelog-2.12.0'></a>
 ## \[2.12.0\] - 2024-04-15
 

Dockerfile

@@ -17,6 +17,9 @@ RUN apt-get update && \
         pkg-config \
         python3-dev \
         python3-pip \
+        libxml2-dev \
+        libxmlsec1-dev \
+        libxmlsec1-openssl \
     && rm -rf /var/lib/apt/lists/*
 
 ARG PIP_VERSION
@@ -119,6 +122,9 @@ RUN apt-get update && \
         libldap-2.5-0 \
         libpython3.10 \
         libsasl2-2 \
+        libxml2 \
+        libxmlsec1 \
+        libxmlsec1-openssl \
         nginx \
         p7zip-full \
         poppler-utils \

changelog.d/20240331_170807_vidit.agarwal.eee20_quality_report.md

@@ -0,0 +1,4 @@
+### Added
+
+- Quality Report calculation will now also include annotation of type Tag.
+  (<https://github.com/opencv/cvat/pull/7582>)

changelog.d/20240416_022857_vidit.agarwal.eee20_ui_tag_annotations.md

@@ -0,0 +1,4 @@
+### Added
+
+- Added feature to show tags of GT and manual job in separate row. Tags of GT job have '(GT)' in their name.
+  (<https://github.com/cvat-ai/cvat/pull/7774>)

changelog.d/20240426_101506_boris_requests_clickhouse_optimization.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Analytics report calculation fails with timeout because of redundant number of requests to ClickHouse
+  (<https://github.com/cvat-ai/cvat/pull/7804>)

changelog.d/20240426_143800_boris_manual_analytics_report.md

@@ -0,0 +1,4 @@
+### Changed
+
+- Analytics reports calculation may be initiated manually instead of automatic scheduling
+  (<https://github.com/cvat-ai/cvat/pull/7805>)

changelog.d/20240429_110227_boris_fixed_change_frame_duration.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Incorrect duration of `change:frame` event
+  (<https://github.com/cvat-ai/cvat/pull/7817>)

changelog.d/20240429_124319_jackylamhk_patch_2.md

@@ -0,0 +1,4 @@
+### Security
+
+- Disable the nginx server signature by default to make it slightly harder for attackers to find known vulnerabilities.
+  (<https://github.com/cvat-ai/cvat/pull/7814>)

changelog.d/20240429_125117_zahadhamov_update_nuclio_version.md

@@ -0,0 +1,10 @@
+### Changed
+
+- Update the Nuclio version and related packages/libraries
+  (<https://github.com/cvat-ai/cvat/pull/7787>)
+
+### Removed
+
+- The `mask_rcnn` function has been removed because it was using python3.6.
+  In new version of Nuclio python3.6 is no longer supported. Nuclio officially recommends using python3.9.
+  Running `mask_rcnn` on python3.9 causes errors within the function and package conflicts. (<https://github.com/cvat-ai/cvat/pull/7787>)

changelog.d/20240429_145914_boris_fixed_incorrect_cs_request.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Infinite loading cloud storage update page when a lot of cloud storages are available for a user
+  (<https://github.com/cvat-ai/cvat/pull/7823>)

changelog.d/20240430_115445_boris_fixed_infinite_loading_nonexisting_cs.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Opening update CS page sends infinite requests when CS id does not exist
+  (<https://github.com/cvat-ai/cvat/pull/7828>)

changelog.d/20240430_172649_boris_enable_tus_retries.md

@@ -0,0 +1,4 @@
+### Fixed
+
+Uploading files with TUS immediately failed when one of the requests failed
+  (<https://github.com/cvat-ai/cvat/pull/7830>)

changelog.d/20240501_095709_boris_requests_clickhouse_optimization.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Longer analytics report calculation because of inefficient requests to analytics db
+  (<https://github.com/cvat-ai/cvat/pull/7833>)

changelog.d/20240501_111444_boris_reworked_empty_first_frame_fix.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Cannot read properties of undefined (reading 'addClass')
+  (<https://github.com/cvat-ai/cvat/pull/7834>)

changelog.d/20240503_101921_boris_fixed_length_of_undefined.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Fixed exception 'Could not read property length of undefined' when copy/paste a skeleton point
+  (<https://github.com/cvat-ai/cvat/pull/7843>)

changelog.d/20240503_105638_boris_do_not_allow_removing_last_keyframe.md

@@ -0,0 +1,4 @@
+### Changed
+
+- Remove keyframe button is disabled when there is only one keyframe element
+  (<https://github.com/cvat-ai/cvat/pull/7844>)

changelog.d/20240503_110728_maria_fix_task_creation_from_video_with_no_valid_keyframes.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Task creation from a video file without keyframes allowing for random iteration
+  (<https://github.com/cvat-ai/cvat/pull/7838>)

changelog.d/20240507_103107_boris_fixed_exception.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Cannot read property 'annotations' of null when uploading annotations into a job
+  (<https://github.com/cvat-ai/cvat/pull/7857>)

changelog.d/20240507_153530_boris_fixed_vertical_polylines.md

@@ -0,0 +1,4 @@
+### Fixed
+
+- Vertical polyline of two points is difficult to select
+  (<https://github.com/cvat-ai/cvat/pull/7860>)