feat: Add REST endpoints to update Components in a Collections (temp) (…

…#674)

* chore: uses openedx-learning==0.11.3
* feat: add/remove components to/from a collection
  Sends a CONTENT_OBJECT_TAGS_CHANGED for each component added/removed.
* docs: Add warning about unstable REST APIs
* refactor: use oel_collections.get_collections as oel_collections.get_learning_package_collections has been removed.
* test: fixes flaky collection search test
* refactor: simplify the REST API params and validation

openedx/core/djangoapps/content/search/api.py

@@ -296,16 +296,12 @@ def rebuild_index(status_cb: Callable[[str], None] | None = None) -> None:
     status_cb("Counting courses...")
     num_courses = CourseOverview.objects.count()
 
-    # Get the list of collections
-    status_cb("Counting collections...")
-    num_collections = authoring_api.get_collections().count()
-
     # Some counters so we can track our progress as indexing progresses:
-    num_contexts = num_courses + num_libraries + num_collections
+    num_contexts = num_courses + num_libraries
     num_contexts_done = 0  # How many courses/libraries we've indexed
     num_blocks_done = 0  # How many individual components/XBlocks we've indexed
 
-    status_cb(f"Found {num_courses} courses, {num_libraries} libraries and {num_collections} collections.")
+    status_cb(f"Found {num_courses} courses, {num_libraries} libraries.")
     with _using_temp_index(status_cb) as temp_index_name:
         ############## Configure the index ##############
 
@@ -390,10 +386,43 @@ def index_library(lib_key: str) -> list:
                     status_cb(f"Error indexing library {lib_key}: {err}")
             return docs
 
+        ############## Collections ##############
+        def index_collection_batch(batch, num_done) -> int:
+            docs = []
+            for collection in batch:
+                try:
+                    doc = searchable_doc_for_collection(collection)
+                    # Uncomment below line once collections are tagged.
+                    # doc.update(searchable_doc_tags(collection.id))
+                    docs.append(doc)
+                except Exception as err:  # pylint: disable=broad-except
+                    status_cb(f"Error indexing collection {collection}: {err}")
+                num_done += 1
+
+            if docs:
+                try:
+                    # Add docs in batch of 100 at once (usually faster than adding one at a time):
+                    _wait_for_meili_task(client.index(temp_index_name).add_documents(docs))
+                except (TypeError, KeyError, MeilisearchError) as err:
+                    status_cb(f"Error indexing collection batch {p}: {err}")
+            return num_done
+
         for lib_key in lib_keys:
-            status_cb(f"{num_contexts_done + 1}/{num_contexts}. Now indexing library {lib_key}")
+            status_cb(f"{num_contexts_done + 1}/{num_contexts}. Now indexing blocks in library {lib_key}")
             lib_docs = index_library(lib_key)
             num_blocks_done += len(lib_docs)
+
+            # To reduce memory usage on large instances, split up the Collections into pages of 100 collections:
+            library = lib_api.get_library(lib_key)
+            collections = authoring_api.get_collections(library.learning_package.id, enabled=True)
+            num_collections = collections.count()
+            num_collections_done = 0
+            status_cb(f"{num_collections_done + 1}/{num_collections}. Now indexing collections in library {lib_key}")
+            paginator = Paginator(collections, 100)
+            for p in paginator.page_range:
+                num_collections_done = index_collection_batch(paginator.page(p).object_list, num_collections_done)
+            status_cb(f"{num_collections_done}/{num_collections} collections indexed for library {lib_key}")
+
             num_contexts_done += 1
 
         ############## Courses ##############
@@ -430,39 +459,6 @@ def add_with_children(block):
                 num_contexts_done += 1
                 num_blocks_done += len(course_docs)
 
-        ############## Collections ##############
-        status_cb("Indexing collections...")
-
-        def index_collection_batch(batch, num_contexts_done) -> int:
-            docs = []
-            for collection in batch:
-                status_cb(
-                    f"{num_contexts_done + 1}/{num_contexts}. "
-                    f"Now indexing collection {collection.title} ({collection.id})"
-                )
-                try:
-                    doc = searchable_doc_for_collection(collection)
-                    # Uncomment below line once collections are tagged.
-                    # doc.update(searchable_doc_tags(collection.id))
-                    docs.append(doc)
-                except Exception as err:  # pylint: disable=broad-except
-                    status_cb(f"Error indexing collection {collection}: {err}")
-                finally:
-                    num_contexts_done += 1
-
-            if docs:
-                try:
-                    # Add docs in batch of 100 at once (usually faster than adding one at a time):
-                    _wait_for_meili_task(client.index(temp_index_name).add_documents(docs))
-                except (TypeError, KeyError, MeilisearchError) as err:
-                    status_cb(f"Error indexing collection batch {p}: {err}")
-            return num_contexts_done
-
-        # To reduce memory usage on large instances, split up the Collections into pages of 100 collections:
-        paginator = Paginator(authoring_api.get_collections(enabled=True), 100)
-        for p in paginator.page_range:
-            num_contexts_done = index_collection_batch(paginator.page(p).object_list, num_contexts_done)
-
     status_cb(f"Done! {num_blocks_done} blocks indexed across {num_contexts_done} courses, collections and libraries.")
 
 

openedx/core/djangoapps/content/search/tests/test_api.py

@@ -177,8 +177,15 @@ def setUp(self):
 
         # Create a collection:
         self.learning_package = authoring_api.get_learning_package_by_key(self.library.key)
+        with freeze_time(created_date):
+            self.collection = authoring_api.create_collection(
+                learning_package_id=self.learning_package.id,
+                title="my_collection",
+                created_by=None,
+                description="my collection description"
+            )
         self.collection_dict = {
-            'id': 1,
+            'id': self.collection.id,
             'type': 'collection',
             'display_name': 'my_collection',
             'description': 'my collection description',
@@ -189,13 +196,6 @@ def setUp(self):
             "access_id": lib_access.id,
             'breadcrumbs': [{'display_name': 'Library'}]
         }
-        with freeze_time(created_date):
-            self.collection = authoring_api.create_collection(
-                learning_package_id=self.learning_package.id,
-                title="my_collection",
-                created_by=None,
-                description="my collection description"
-            )
 
     @override_settings(MEILISEARCH_ENABLED=False)
     def test_reindex_meilisearch_disabled(self, mock_meilisearch):

openedx/core/djangoapps/content_libraries/api.py

@@ -69,15 +69,20 @@
 from django.utils.translation import gettext as _
 from edx_rest_api_client.client import OAuthAPIClient
 from lxml import etree
-from opaque_keys.edx.keys import UsageKey, UsageKeyV2
+from opaque_keys.edx.keys import BlockTypeKey, UsageKey, UsageKeyV2
 from opaque_keys.edx.locator import (
     LibraryLocatorV2,
     LibraryUsageLocatorV2,
     LibraryLocator as LibraryLocatorV1
 )
 from opaque_keys import InvalidKeyError
-from openedx_events.content_authoring.data import ContentLibraryData, LibraryBlockData
+from openedx_events.content_authoring.data import (
+    ContentLibraryData,
+    ContentObjectData,
+    LibraryBlockData,
+)
 from openedx_events.content_authoring.signals import (
+    CONTENT_OBJECT_TAGS_CHANGED,
     CONTENT_LIBRARY_CREATED,
     CONTENT_LIBRARY_DELETED,
     CONTENT_LIBRARY_UPDATED,
@@ -86,7 +91,7 @@
     LIBRARY_BLOCK_UPDATED,
 )
 from openedx_learning.api import authoring as authoring_api
-from openedx_learning.api.authoring_models import Component, MediaType, LearningPackage
+from openedx_learning.api.authoring_models import Collection, Component, MediaType, LearningPackage, PublishableEntity
 from organizations.models import Organization
 from xblock.core import XBlock
 from xblock.exceptions import XBlockNotFoundError
@@ -111,6 +116,8 @@
 
 ContentLibraryNotFound = ContentLibrary.DoesNotExist
 
+ContentLibraryCollectionNotFound = Collection.DoesNotExist
+
 
 class ContentLibraryBlockNotFound(XBlockNotFoundError):
     """ XBlock not found in the content library """
@@ -1062,6 +1069,68 @@ def revert_changes(library_key):
     )
 
 
+def update_collection_components(
+    collection: Collection,
+    usage_keys: list[UsageKeyV2],
+    created_by: int | None = None,
+    remove=False,
+) -> Collection:
+    """
+    Associates the Collection with Components for the given UsageKeys.
+
+    By default the Components are added to the Collection.
+    If remove=True, the Components are removed from the Collection.
+
+    Raises:
+    * ContentLibraryCollectionNotFound if no Collection with the given pk is found in the given library.
+    * ContentLibraryBlockNotFound if any of the given usage_keys don't match Components in the given library.
+
+    Returns the updated Collection.
+    """
+    # Fetch the Component.key values for the provided UsageKeys.
+    component_keys = []
+    for usage_key in usage_keys:
+        # Parse the block_family from the key to use as namespace.
+        block_type = BlockTypeKey.from_string(str(usage_key))
+
+        try:
+            component = authoring_api.get_component_by_key(
+                collection.learning_package_id,
+                namespace=block_type.block_family,
+                type_name=usage_key.block_type,
+                local_key=usage_key.block_id,
+            )
+        except Component.DoesNotExist as exc:
+            raise ContentLibraryBlockNotFound(usage_key) from exc
+
+        component_keys.append(component.key)
+
+    # Note: Component.key matches its PublishableEntity.key
+    entities_qset = PublishableEntity.objects.filter(
+        key__in=component_keys,
+    )
+
+    if remove:
+        collection = authoring_api.remove_from_collection(
+            collection.pk,
+            entities_qset,
+        )
+    else:
+        collection = authoring_api.add_to_collection(
+            collection.pk,
+            entities_qset,
+            created_by=created_by,
+        )
+
+    # Emit a CONTENT_OBJECT_TAGS_CHANGED event for each of the objects added/removed
+    for usage_key in usage_keys:
+        CONTENT_OBJECT_TAGS_CHANGED.send_event(
+            content_object=ContentObjectData(object_id=usage_key),
+        )
+
+    return collection
+
+
 # V1/V2 Compatibility Helpers
 # (Should be removed as part of
 #  https://github.com/openedx/edx-platform/issues/32457)

openedx/core/djangoapps/content_libraries/serializers.py

@@ -4,7 +4,10 @@
 # pylint: disable=abstract-method
 from django.core.validators import validate_unicode_slug
 from rest_framework import serializers
+from rest_framework.exceptions import ValidationError
 
+from opaque_keys.edx.keys import UsageKeyV2
+from opaque_keys import InvalidKeyError
 
 from openedx_learning.api.authoring_models import Collection
 from openedx.core.djangoapps.content_libraries.constants import (
@@ -266,3 +269,33 @@ class ContentLibraryCollectionCreateOrUpdateSerializer(serializers.Serializer):
 
     title = serializers.CharField()
     description = serializers.CharField()
+
+
+class UsageKeyV2Serializer(serializers.Serializer):
+    """
+    Serializes a UsageKeyV2.
+    """
+    def to_representation(self, value: UsageKeyV2) -> str:
+        """
+        Returns the UsageKeyV2 value as a string.
+        """
+        return str(value)
+
+    def to_internal_value(self, value: str) -> UsageKeyV2:
+        """
+        Returns a UsageKeyV2 from the string value.
+
+        Raises ValidationError if invalid UsageKeyV2.
+        """
+        try:
+            return UsageKeyV2.from_string(value)
+        except InvalidKeyError as err:
+            raise ValidationError from err
+
+
+class ContentLibraryCollectionComponentsUpdateSerializer(serializers.Serializer):
+    """
+    Serializer for adding/removing Components to/from a Collection.
+    """
+
+    usage_keys = serializers.ListField(child=UsageKeyV2Serializer(), allow_empty=False)