Eugeny/problem type search

cms/djangoapps/contentstore/courseware_index.py

@@ -8,13 +8,9 @@
 from django.utils.translation import ugettext as _
 from eventtracking import tracker
 from xmodule.modulestore import ModuleStoreEnum
+from xmodule.library_tools import normalize_key_for_search
 from search.search_engine_base import SearchEngine
 
-
-# Use default index and document names for now
-INDEX_NAME = "courseware_index"
-DOCUMENT_TYPE = "courseware_content"
-
 # REINDEX_AGE is the default amount of time that we look back for changes
 # that might have happened. If we are provided with a time at which the
 # indexing is triggered, then we know it is safe to only index items
@@ -25,13 +21,6 @@
 log = logging.getLogger('edx.modulestore')
 
 
-def indexing_is_enabled():
-    """
-    Checks to see if the indexing feature is enabled
-    """
-    return settings.FEATURES.get('ENABLE_COURSEWARE_INDEX', False)
-
-
 class SearchIndexingError(Exception):
     """ Indicates some error(s) occured during indexing """
 
@@ -40,18 +29,69 @@ def __init__(self, message, error_list):
         self.error_list = error_list
 
 
-class CoursewareSearchIndexer(object):
+class SearchIndexerBase(object):
     """
-    Class to perform indexing for courseware search from different modulestores
+    Base class to perform indexing for courseware or library search from different modulestores
     """
 
+    INDEX_NAME = None
+    DOCUMENT_TYPE = None
+    ENABLE_INDEXING_KEY = None
+
+    INDEX_EVENT = {
+        'name': None,
+        'category': None
+    }
+
     @classmethod
-    def index_course(cls, modulestore, course_key, triggered_at=None, reindex_age=REINDEX_AGE):
+    def indexing_is_enabled(cls):
+        """
+        Checks to see if the indexing feature is enabled
+        """
+        return settings.FEATURES.get(cls.ENABLE_INDEXING_KEY, False)
+
+    @classmethod
+    def normalize_structure_key(cls, structure_key):
+        """ Normalizes structure key for use in indexing """
+        raise NotImplementedError("Should be overridden in child classes")
+
+    @classmethod
+    def _fetch_top_level(cls, modulestore, structure_key):
+        """ Fetch the item from the modulestore location """
+        raise NotImplementedError("Should be overridden in child classes")
+
+    @classmethod
+    def _get_location_info(cls, normalized_structure_key):
+        """ Builds location info dictionary """
+        raise NotImplementedError("Should be overridden in child classes")
+
+    @classmethod
+    def _id_modifier(cls, usage_id):
+        """ Modifies usage_id to submit to index """
+        return usage_id
+
+    @classmethod
+    def remove_deleted_items(cls, searcher, structure_key, exclude_items):
+        """
+        remove any item that is present in the search index that is not present in updated list of indexed items
+        as we find items we can shorten the set of items to keep
+        """
+        response = searcher.search(
+            doc_type=cls.DOCUMENT_TYPE,
+            field_dictionary=cls._get_location_info(structure_key),
+            exclude_ids=exclude_items
+        )
+        result_ids = [result["data"]["id"] for result in response["results"]]
+        for result_id in result_ids:
+            searcher.remove(cls.DOCUMENT_TYPE, result_id)
+
+    @classmethod
+    def index(cls, modulestore, structure_key, triggered_at=None, reindex_age=REINDEX_AGE):
         """
         Process course for indexing
 
         Arguments:
-        course_key (CourseKey) - course identifier
+        structure_key (CourseKey|LibraryKey) - course or library identifier
 
         triggered_at (datetime) - provides time at which indexing was triggered;
             useful for index updates - only things changed recently from that date
@@ -64,13 +104,12 @@ def index_course(cls, modulestore, course_key, triggered_at=None, reindex_age=RE
         Number of items that have been added to the index
         """
         error_list = []
-        searcher = SearchEngine.get_search_engine(INDEX_NAME)
+        searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
         if not searcher:
             return
 
-        location_info = {
-            "course": unicode(course_key),
-        }
+        structure_key = cls.normalize_structure_key(structure_key)
+        location_info = cls._get_location_info(structure_key)
 
         # Wrap counter in dictionary - otherwise we seem to lose scope inside the embedded function `index_item`
         indexed_count = {
@@ -101,7 +140,7 @@ def index_item(item, skip_index=False):
             if not item_index_dictionary and not item.has_children:
                 return
 
-            item_id = unicode(item.scope_ids.usage_id)
+            item_id = unicode(cls._id_modifier(item.scope_ids.usage_id))
             indexed_items.add(item_id)
             if item.has_children:
                 # determine if it's okay to skip adding the children herein based upon how recently any may have changed
@@ -122,38 +161,24 @@ def index_item(item, skip_index=False):
                 if item.start:
                     item_index['start_date'] = item.start
 
-                searcher.index(DOCUMENT_TYPE, item_index)
+                searcher.index(cls.DOCUMENT_TYPE, item_index)
                 indexed_count["count"] += 1
             except Exception as err:  # pylint: disable=broad-except
                 # broad exception so that index operation does not fail on one item of many
                 log.warning('Could not index item: %s - %r', item.location, err)
                 error_list.append(_('Could not index item: {}').format(item.location))
 
-        def remove_deleted_items():
-            """
-            remove any item that is present in the search index that is not present in updated list of indexed items
-            as we find items we can shorten the set of items to keep
-            """
-            response = searcher.search(
-                doc_type=DOCUMENT_TYPE,
-                field_dictionary={"course": unicode(course_key)},
-                exclude_ids=indexed_items
-            )
-            result_ids = [result["data"]["id"] for result in response["results"]]
-            for result_id in result_ids:
-                searcher.remove(DOCUMENT_TYPE, result_id)
-
         try:
             with modulestore.branch_setting(ModuleStoreEnum.RevisionOption.published_only):
-                course = modulestore.get_course(course_key, depth=None)
-                for item in course.get_children():
+                structure = cls._fetch_top_level(modulestore, structure_key)
+                for item in structure.get_children():
                     index_item(item)
-                remove_deleted_items()
+                cls.remove_deleted_items(searcher, structure_key, indexed_items)
         except Exception as err:  # pylint: disable=broad-except
             # broad exception so that index operation does not prevent the rest of the application from working
             log.exception(
                 "Indexing error encountered, courseware index may be out of date %s - %r",
-                course_key,
+                structure_key,
                 err
             )
             error_list.append(_('General indexing error occurred'))
@@ -164,31 +189,111 @@ def remove_deleted_items():
         return indexed_count["count"]
 
     @classmethod
-    def do_course_reindex(cls, modulestore, course_key):
+    def _do_reindex(cls, modulestore, structure_key):
         """
-        (Re)index all content within the given course, tracking the fact that a full reindex has taking place
+        (Re)index all content within the given structure (course or library),
+        tracking the fact that a full reindex has taken place
         """
-        indexed_count = cls.index_course(modulestore, course_key)
+        indexed_count = cls.index(modulestore, structure_key)
         if indexed_count:
-            cls._track_index_request('edx.course.index.reindexed', indexed_count)
+            cls._track_index_request(cls.INDEX_EVENT['name'], cls.INDEX_EVENT['category'], indexed_count)
         return indexed_count
 
     @classmethod
-    def _track_index_request(cls, event_name, indexed_count):
+    def _track_index_request(cls, event_name, category, indexed_count):
         """Track content index requests.
 
         Arguments:
             event_name (str):  Name of the event to be logged.
+            category (str): category of indexed items
+            indexed_count (int): number of indexed items
         Returns:
             None
 
         """
         data = {
             "indexed_count": indexed_count,
-            'category': 'courseware_index',
+            'category': category,
         }
 
         tracker.emit(
             event_name,
             data
         )
+
+
+class CoursewareSearchIndexer(SearchIndexerBase):
+    """
+    Class to perform indexing for courseware search from different modulestores
+    """
+    INDEX_NAME = "courseware_index"
+    DOCUMENT_TYPE = "courseware_content"
+    ENABLE_INDEXING_KEY = 'ENABLE_COURSEWARE_INDEX'
+
+    INDEX_EVENT = {
+        'name': 'edx.course.index.reindexed',
+        'category': 'courseware_index'
+    }
+
+    @classmethod
+    def normalize_structure_key(cls, structure_key):
+        """ Normalizes structure key for use in indexing """
+        return structure_key
+
+    @classmethod
+    def _fetch_top_level(cls, modulestore, structure_key):
+        """ Fetch the item from the modulestore location """
+        return modulestore.get_course(structure_key, depth=None)
+
+    @classmethod
+    def _get_location_info(cls, normalized_structure_key):
+        """ Builds location info dictionary """
+        return {"course": unicode(normalized_structure_key)}
+
+    @classmethod
+    def do_course_reindex(cls, modulestore, course_key):
+        """
+        (Re)index all content within the given course, tracking the fact that a full reindex has taken place
+        """
+        return cls._do_reindex(modulestore, course_key)
+
+
+class LibrarySearchIndexer(SearchIndexerBase):
+    """
+    Base class to perform indexing for library search from different modulestores
+    """
+    INDEX_NAME = "library_index"
+    DOCUMENT_TYPE = "library_content"
+    ENABLE_INDEXING_KEY = 'ENABLE_LIBRARY_INDEX'
+
+    INDEX_EVENT = {
+        'name': 'edx.library.index.reindexed',
+        'category': 'library_index'
+    }
+
+    @classmethod
+    def normalize_structure_key(cls, structure_key):
+        """ Normalizes structure key for use in indexing """
+        return normalize_key_for_search(structure_key)
+
+    @classmethod
+    def _fetch_top_level(cls, modulestore, structure_key):
+        """ Fetch the item from the modulestore location """
+        return modulestore.get_library(structure_key, depth=None)
+
+    @classmethod
+    def _get_location_info(cls, normalized_structure_key):
+        """ Builds location info dictionary """
+        return {"library": unicode(normalized_structure_key)}
+
+    @classmethod
+    def _id_modifier(cls, usage_id):
+        """ Modifies usage_id to submit to index """
+        return usage_id.replace(library_key=(usage_id.library_key.replace(version_guid=None, branch=None)))
+
+    @classmethod
+    def do_library_reindex(cls, modulestore, library_key):
+        """
+        (Re)index all content within the given library, tracking the fact that a full reindex has taken place
+        """
+        return cls._do_reindex(modulestore, library_key)

cms/djangoapps/contentstore/signals.py

@@ -1,11 +1,11 @@
-""" receiver of course_published events in order to trigger indexing task """
+""" receivers of course_published and library_updated events in order to trigger indexing task """
 from datetime import datetime
 from pytz import UTC
 
 from django.dispatch import receiver
 
 from xmodule.modulestore.django import SignalHandler
-from contentstore.courseware_index import indexing_is_enabled
+from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer
 
 
 @receiver(SignalHandler.course_published)
@@ -15,5 +15,16 @@ def listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable=
     """
     # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
     from .tasks import update_search_index
-    if indexing_is_enabled():
+    if CoursewareSearchIndexer.indexing_is_enabled():
         update_search_index.delay(unicode(course_key), datetime.now(UTC).isoformat())
+
+
+@receiver(SignalHandler.library_updated)
+def listen_for_library_update(sender, library_key, **kwargs):  # pylint: disable=unused-argument
+    """
+    Receives signal and kicks off celery task to update search index
+    """
+    # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
+    from .tasks import update_library_index
+    if LibrarySearchIndexer.indexing_is_enabled():
+        update_library_index.delay(unicode(library_key), datetime.now(UTC).isoformat())

cms/djangoapps/contentstore/tasks.py

@@ -10,7 +10,7 @@
 
 from django.contrib.auth.models import User
 
-from contentstore.courseware_index import CoursewareSearchIndexer, SearchIndexingError
+from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer, SearchIndexingError
 from contentstore.utils import initialize_permissions
 from course_action_state.models import CourseRerunState
 from opaque_keys.edx.keys import CourseKey
@@ -82,19 +82,36 @@ def deserialize_fields(json_fields):
     return fields
 
 
+def _parse_time(time_isoformat):
+    """ Parses time from iso format """
+    return datetime.strptime(
+        # remove the +00:00 from the end of the formats generated within the system
+        time_isoformat.split('+')[0],
+        "%Y-%m-%dT%H:%M:%S.%f"
+    ).replace(tzinfo=UTC)
+
+
 @task()
 def update_search_index(course_id, triggered_time_isoformat):
     """ Updates course search index. """
     try:
         course_key = CourseKey.from_string(course_id)
-        triggered_time = datetime.strptime(
-            # remove the +00:00 from the end of the formats generated within the system
-            triggered_time_isoformat.split('+')[0],
-            "%Y-%m-%dT%H:%M:%S.%f"
-        ).replace(tzinfo=UTC)
-        CoursewareSearchIndexer.index_course(modulestore(), course_key, triggered_at=triggered_time)
+        CoursewareSearchIndexer.index(modulestore(), course_key, triggered_at=(_parse_time(triggered_time_isoformat)))
 
     except SearchIndexingError as exc:
         LOGGER.error('Search indexing error for complete course %s - %s', course_id, unicode(exc))
     else:
         LOGGER.debug('Search indexing successful for complete course %s', course_id)
+
+
+@task()
+def update_library_index(library_id, triggered_time_isoformat):
+    """ Updates course search index. """
+    try:
+        library_key = CourseKey.from_string(library_id)
+        LibrarySearchIndexer.index(modulestore(), library_key, triggered_at=(_parse_time(triggered_time_isoformat)))
+
+    except SearchIndexingError as exc:
+        LOGGER.error('Search indexing error for library %s - %s', library_id, unicode(exc))
+    else:
+        LOGGER.debug('Search indexing successful for library %s', library_id)