feat: Add Library Collections REST endpoints [FC-0062] (#35321)

* feat: Add Library Collections REST endpoints

* test: Add tests for Collections REST APIs

* chore: Add missing __init__ files

* feat: Add events emitting for Collections

* feat: Add REST endpoints to update Components in a Collections (temp) (#674)

* feat: add/remove components to/from a collection

* docs: Add warning about unstable REST APIs

* chore: updates openedx-events==9.14.0

* chore: updates openedx-learning==0.11.4

* fix: assert collection doc have unique id

---------

Co-authored-by: Jillian <jill@opencraft.com>
Co-authored-by: Chris Chávez <xnpiochv@gmail.com>
Co-authored-by: Rômulo Penido <romulo.penido@gmail.com>

Author: 4 people

docs/hooks/events.rst

@@ -233,17 +233,29 @@ Content Authoring Events
      - 2023-07-20
 
    * - `LIBRARY_BLOCK_CREATED <https://github.com/openedx/openedx-events/blob/c0eb4ba1a3d7d066d58e5c87920b8ccb0645f769/openedx_events/content_authoring/signals.py#L167>`_
-     - org.openedx.content_authoring.content_library.created.v1
+     - org.openedx.content_authoring.library_block.created.v1
      - 2023-07-20
 
    * - `LIBRARY_BLOCK_UPDATED <https://github.com/openedx/openedx-events/blob/c0eb4ba1a3d7d066d58e5c87920b8ccb0645f769/openedx_events/content_authoring/signals.py#L178>`_
-     - org.openedx.content_authoring.content_library.updated.v1
+     - org.openedx.content_authoring.library_block.updated.v1
      - 2023-07-20
 
    * - `LIBRARY_BLOCK_DELETED <https://github.com/openedx/openedx-events/blob/c0eb4ba1a3d7d066d58e5c87920b8ccb0645f769/openedx_events/content_authoring/signals.py#L189>`_
-     - org.openedx.content_authoring.content_library.deleted.v1
+     - org.openedx.content_authoring.library_block.deleted.v1
      - 2023-07-20
 
    * - `CONTENT_OBJECT_TAGS_CHANGED <https://github.com/openedx/openedx-events/blob/c0eb4ba1a3d7d066d58e5c87920b8ccb0645f769/openedx_events/content_authoring/signals.py#L207>`_
      - org.openedx.content_authoring.content.object.tags.changed.v1
      - 2024-03-31
+
+   * - `LIBRARY_COLLECTION_CREATED <https://github.com/openedx/openedx-events/blob/main/openedx_events/content_authoring/signals.py#L219>`_
+     - org.openedx.content_authoring.content_library.collection.created.v1
+     - 2024-08-23
+
+   * - `LIBRARY_COLLECTION_UPDATED <https://github.com/openedx/openedx-events/blob/main/openedx_events/content_authoring/signals.py#L230>`_
+     - org.openedx.content_authoring.content_library.collection.updated.v1
+     - 2024-08-23
+
+   * - `LIBRARY_COLLECTION_DELETED <https://github.com/openedx/openedx-events/blob/main/openedx_events/content_authoring/signals.py#L241>`_
+     - org.openedx.content_authoring.content_library.collection.deleted.v1
+     - 2024-08-23

openedx/core/djangoapps/content/search/api.py

@@ -296,16 +296,12 @@ def rebuild_index(status_cb: Callable[[str], None] | None = None) -> None:
     status_cb("Counting courses...")
     num_courses = CourseOverview.objects.count()
 
-    # Get the list of collections
-    status_cb("Counting collections...")
-    num_collections = authoring_api.get_collections().count()
-
     # Some counters so we can track our progress as indexing progresses:
-    num_contexts = num_courses + num_libraries + num_collections
+    num_contexts = num_courses + num_libraries
     num_contexts_done = 0  # How many courses/libraries we've indexed
     num_blocks_done = 0  # How many individual components/XBlocks we've indexed
 
-    status_cb(f"Found {num_courses} courses, {num_libraries} libraries and {num_collections} collections.")
+    status_cb(f"Found {num_courses} courses, {num_libraries} libraries.")
     with _using_temp_index(status_cb) as temp_index_name:
         ############## Configure the index ##############
 
@@ -390,10 +386,43 @@ def index_library(lib_key: str) -> list:
                     status_cb(f"Error indexing library {lib_key}: {err}")
             return docs
 
+        ############## Collections ##############
+        def index_collection_batch(batch, num_done) -> int:
+            docs = []
+            for collection in batch:
+                try:
+                    doc = searchable_doc_for_collection(collection)
+                    # Uncomment below line once collections are tagged.
+                    # doc.update(searchable_doc_tags(collection.id))
+                    docs.append(doc)
+                except Exception as err:  # pylint: disable=broad-except
+                    status_cb(f"Error indexing collection {collection}: {err}")
+                num_done += 1
+
+            if docs:
+                try:
+                    # Add docs in batch of 100 at once (usually faster than adding one at a time):
+                    _wait_for_meili_task(client.index(temp_index_name).add_documents(docs))
+                except (TypeError, KeyError, MeilisearchError) as err:
+                    status_cb(f"Error indexing collection batch {p}: {err}")
+            return num_done
+
         for lib_key in lib_keys:
-            status_cb(f"{num_contexts_done + 1}/{num_contexts}. Now indexing library {lib_key}")
+            status_cb(f"{num_contexts_done + 1}/{num_contexts}. Now indexing blocks in library {lib_key}")
             lib_docs = index_library(lib_key)
             num_blocks_done += len(lib_docs)
+
+            # To reduce memory usage on large instances, split up the Collections into pages of 100 collections:
+            library = lib_api.get_library(lib_key)
+            collections = authoring_api.get_collections(library.learning_package.id, enabled=True)
+            num_collections = collections.count()
+            num_collections_done = 0
+            status_cb(f"{num_collections_done + 1}/{num_collections}. Now indexing collections in library {lib_key}")
+            paginator = Paginator(collections, 100)
+            for p in paginator.page_range:
+                num_collections_done = index_collection_batch(paginator.page(p).object_list, num_collections_done)
+            status_cb(f"{num_collections_done}/{num_collections} collections indexed for library {lib_key}")
+
             num_contexts_done += 1
 
         ############## Courses ##############
@@ -430,39 +459,6 @@ def add_with_children(block):
                 num_contexts_done += 1
                 num_blocks_done += len(course_docs)
 
-        ############## Collections ##############
-        status_cb("Indexing collections...")
-
-        def index_collection_batch(batch, num_contexts_done) -> int:
-            docs = []
-            for collection in batch:
-                status_cb(
-                    f"{num_contexts_done + 1}/{num_contexts}. "
-                    f"Now indexing collection {collection.title} ({collection.id})"
-                )
-                try:
-                    doc = searchable_doc_for_collection(collection)
-                    # Uncomment below line once collections are tagged.
-                    # doc.update(searchable_doc_tags(collection.id))
-                    docs.append(doc)
-                except Exception as err:  # pylint: disable=broad-except
-                    status_cb(f"Error indexing collection {collection}: {err}")
-                finally:
-                    num_contexts_done += 1
-
-            if docs:
-                try:
-                    # Add docs in batch of 100 at once (usually faster than adding one at a time):
-                    _wait_for_meili_task(client.index(temp_index_name).add_documents(docs))
-                except (TypeError, KeyError, MeilisearchError) as err:
-                    status_cb(f"Error indexing collection batch {p}: {err}")
-            return num_contexts_done
-
-        # To reduce memory usage on large instances, split up the Collections into pages of 100 collections:
-        paginator = Paginator(authoring_api.get_collections(enabled=True), 100)
-        for p in paginator.page_range:
-            num_contexts_done = index_collection_batch(paginator.page(p).object_list, num_contexts_done)
-
     status_cb(f"Done! {num_blocks_done} blocks indexed across {num_contexts_done} courses, collections and libraries.")
 
 

openedx/core/djangoapps/content/search/tests/test_api.py

@@ -177,8 +177,16 @@ def setUp(self):
 
         # Create a collection:
         self.learning_package = authoring_api.get_learning_package_by_key(self.library.key)
+        with freeze_time(created_date):
+            self.collection = authoring_api.create_collection(
+                learning_package_id=self.learning_package.id,
+                key="MYCOL",
+                title="my_collection",
+                created_by=None,
+                description="my collection description"
+            )
         self.collection_dict = {
-            'id': 1,
+            'id': self.collection.id,
             'type': 'collection',
             'display_name': 'my_collection',
             'description': 'my collection description',
@@ -189,13 +197,6 @@ def setUp(self):
             "access_id": lib_access.id,
             'breadcrumbs': [{'display_name': 'Library'}]
         }
-        with freeze_time(created_date):
-            self.collection = authoring_api.create_collection(
-                learning_package_id=self.learning_package.id,
-                title="my_collection",
-                created_by=None,
-                description="my collection description"
-            )
 
     @override_settings(MEILISEARCH_ENABLED=False)
     def test_reindex_meilisearch_disabled(self, mock_meilisearch):

openedx/core/djangoapps/content/search/tests/test_documents.py

@@ -215,6 +215,7 @@ def test_collection_with_no_library(self):
             )
             collection = authoring_api.create_collection(
                 learning_package_id=learning_package.id,
+                key="MYCOL",
                 title="my_collection",
                 created_by=None,
                 description="my collection description"
@@ -223,11 +224,11 @@ def test_collection_with_no_library(self):
         assert doc == {
             "id": collection.id,
             "type": "collection",
-            "display_name": collection.title,
-            "description": collection.description,
+            "display_name": "my_collection",
+            "description": "my collection description",
             "context_key": learning_package.key,
             "access_id": self.toy_course_access_id,
-            "breadcrumbs": [{"display_name": learning_package.title}],
+            "breadcrumbs": [{"display_name": "some learning_package"}],
             "created": created_date.timestamp(),
             "modified": created_date.timestamp(),
         }