From 12ab9faf4ec9cf844f1ca19d2bde539c39127b9c Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Sun, 21 Jan 2018 19:11:29 -0500 Subject: [PATCH 1/5] Appsembler: Added search api app to CMS The search api provides a programmatic interface to course indexing. The driver is to support course reindexing from Taxoma. This commit is just the app. It does not contain the hooks to use the code: * cms/urls.py * cms/envs/common.py --- cms/djangoapps/search_api/__init__.py | 2 + cms/djangoapps/search_api/api.py | 24 ++++++ cms/djangoapps/search_api/permissions.py | 10 +++ cms/djangoapps/search_api/urls.py | 9 +++ cms/djangoapps/search_api/views.py | 97 ++++++++++++++++++++++++ 5 files changed, 142 insertions(+) create mode 100644 cms/djangoapps/search_api/__init__.py create mode 100644 cms/djangoapps/search_api/api.py create mode 100644 cms/djangoapps/search_api/permissions.py create mode 100644 cms/djangoapps/search_api/urls.py create mode 100644 cms/djangoapps/search_api/views.py diff --git a/cms/djangoapps/search_api/__init__.py b/cms/djangoapps/search_api/__init__.py new file mode 100644 index 000000000000..6c6d7e1b090f --- /dev/null +++ b/cms/djangoapps/search_api/__init__.py @@ -0,0 +1,2 @@ + +API_VERSION = 'v0' diff --git a/cms/djangoapps/search_api/api.py b/cms/djangoapps/search_api/api.py new file mode 100644 index 000000000000..d741e1e5a92d --- /dev/null +++ b/cms/djangoapps/search_api/api.py @@ -0,0 +1,24 @@ + +from contentstore.courseware_index import CoursewareSearchIndexer +from opaque_keys.edx.keys import CourseKey +from xmodule.modulestore.django import modulestore + +def reindex_course(course_id): + """ + Arguments: + course_id - The course id for a course. This is the 'course_id' property + for the course as returend from: + /api/courses/v1/courses/ + + Raises: + InvalidKeyError - if the opaque course + SearchIndexingError - If the reindexing fails + + References + course.py#reindex_course_and_check_access + + """ + course_key = CourseKey.from_string(course_id) + with modulestore().bulk_operations(course_key): + return CoursewareSearchIndexer.do_course_reindex(modulestore(), + course_key) diff --git a/cms/djangoapps/search_api/permissions.py b/cms/djangoapps/search_api/permissions.py new file mode 100644 index 000000000000..9139839bde48 --- /dev/null +++ b/cms/djangoapps/search_api/permissions.py @@ -0,0 +1,10 @@ +from rest_framework.permissions import BasePermission + + +class IsStaffUser(BasePermission): + """ + Allow access to only staff users + """ + def has_permission(self, request, view): + return request.user and request.user.is_active and ( + request.user.is_staff or request.user.is_superuser) \ No newline at end of file diff --git a/cms/djangoapps/search_api/urls.py b/cms/djangoapps/search_api/urls.py new file mode 100644 index 000000000000..a2d7b920adfb --- /dev/null +++ b/cms/djangoapps/search_api/urls.py @@ -0,0 +1,9 @@ +from django.conf.urls import url + +from . import API_VERSION, views + +urlpatterns = [ + url(r'^$', views.SearchIndex.as_view(), name='search_api_index'), + url(r'^{}/reindex-course'.format(API_VERSION), + views.CourseIndexer.as_view(), name='reindex-course'), +] diff --git a/cms/djangoapps/search_api/views.py b/cms/djangoapps/search_api/views.py new file mode 100644 index 000000000000..9f383304da0d --- /dev/null +++ b/cms/djangoapps/search_api/views.py @@ -0,0 +1,97 @@ + +import json + +from django.conf import settings +from django.http import HttpResponse + +from rest_framework.authentication import ( + BasicAuthentication, + SessionAuthentication, + TokenAuthentication, +) +from rest_framework.decorators import api_view, authentication_classes, permission_classes +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_http_methods +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.views import APIView + +from opaque_keys import InvalidKeyError + +from . import api +from .permissions import IsStaffUser + +# Restrict access to the LMS server +ALLOWED_ORIGIN = settings.LMS_BASE + +description = """ +Appembler Open edX search api. +Opens up access to Open edX'sa search infrastructure via HTTP (REST) API interfaces. +""" + +class SearchIndex(APIView): + authentication_classes = ( + BasicAuthentication, + SessionAuthentication, + TokenAuthentication + ) + + permission_classes = ( IsAuthenticated, IsStaffUser, ) + def get(self, request, format=None): + return Response({ + 'message': 'CMS Search API', + }) + + +class CourseIndexer(APIView): + authentication_classes = ( + BasicAuthentication, + SessionAuthentication, + TokenAuthentication + ) + + permission_classes = ( IsAuthenticated, IsStaffUser, ) + + def get(self, request, format=None): + return Response({ + 'message': 'Course Indexer', + }) + + def post(self, request, format=None): + + request_data = json.loads(request.body) + course_id = request_data.get('course_id') + try: + results = api.reindex_course(course_id) + response_data = { + 'course_id': course_id, + 'status': 'OK', + 'message': 'course reindex initiated', + 'results': results, + } + response = Response(response_data) + response['Access-Control-Allow-Origin'] = ALLOWED_ORIGIN + response['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' + response['Access-Control-Allow-Headers'] = '*' + return response + except Exception as e: + if isinstance(e, InvalidKeyError): + message = 'InvalidKeyError: Cannot find key for course string ' + \ + '"{}"'.format(course_id) + status = 400 + else: + message = 'Exception "{}" msg: {}'.format(e.__class__, e.message) + status = 500 + return Response(json.dumps({ + 'course_id': course_id, + 'status': 'ERROR', + 'message': message, + }), status=status) + + def options(self, request, format=None): + response = Response() + response['Access-Control-Allow-Origin'] = ALLOWED_ORIGIN + response['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS' + # Options do not allow wildcard for access-control-allow-headers + response['Access-Control-Allow-Headers'] = 'Content-Type' + return response From e73b6a5380695c6efac5e3b8a3cd0b5e038b557d Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Sun, 21 Jan 2018 19:21:04 -0500 Subject: [PATCH 2/5] Appsembler: Added hooks for the CMS search api This commit adds required hooks to `cms/urls.py` and `cms/envs/common.py` in order to use the search api --- cms/envs/common.py | 5 +++++ cms/urls.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/cms/envs/common.py b/cms/envs/common.py index abe1edbd3f61..776b18de6f1b 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -959,6 +959,11 @@ # Unusual migrations 'database_fixups', + + # Appsembler API Extensions + 'rest_framework', + 'rest_framework.authtoken', + 'search_api', ) diff --git a/cms/urls.py b/cms/urls.py index 7b241d501ba4..62d166b6287b 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -210,3 +210,8 @@ url(r'^404$', handler404), url(r'^500$', handler500), ) + +# Appsembler API extensions +urlpatterns += ( + url(r'^api/search/', include('serach_api.urls')), +) From e3e0d8810a0eee30ca448cd55a589005e4e786cc Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 23 Jan 2018 10:44:48 -0500 Subject: [PATCH 3/5] Appsembler: added search api to CMS urls.py --- cms/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/urls.py b/cms/urls.py index 62d166b6287b..6b219db7ce58 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -213,5 +213,5 @@ # Appsembler API extensions urlpatterns += ( - url(r'^api/search/', include('serach_api.urls')), + url(r'^api/search/', include('search_api.urls')), ) From a9f3aaa9bb3488c0b72c75b50924e0065575e247 Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 23 Jan 2018 12:04:18 -0500 Subject: [PATCH 4/5] Added support for Taxoman in CMS courseware_index.py Courseware Index performs the initial indexing. This is what provides the custom facets that the built-in LMS courseware discovery uses --- .../contentstore/courseware_index.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cms/djangoapps/contentstore/courseware_index.py b/cms/djangoapps/contentstore/courseware_index.py index b497d3455639..d1d538080bfa 100644 --- a/cms/djangoapps/contentstore/courseware_index.py +++ b/cms/djangoapps/contentstore/courseware_index.py @@ -29,6 +29,18 @@ log = logging.getLogger('edx.modulestore') +# Interim code to get courseware_index to work with Taxoman +if hasattr(settings,'FEATURES') and settings.FEATURES.get('ENABLE_TAXOMAN', False): + try: + from taxoman_api.models import Facet, FacetValue, CourseFacetValue + using_taxoman = True + except ImportError: + log.error('Taxoman enabled, but unable to import taxoman_api package (ImportError') + using_taxoman = False +else: + using_taxoman = False + + def strip_html_content_to_text(html_content): """ Gets only the textual part for html content - useful for building text to be searched """ # Removing HTML-encoded non-breaking space characters @@ -527,11 +539,31 @@ def from_course_mode(self, **kwargs): return [mode.slug for mode in CourseMode.modes_for_course(course.id)] + def from_taxoman(self, **kwargs): + '''Fetches the assigned value to the facet in taxoman + ''' + if using_taxoman: + course = kwargs.get('course', None) + if not course: + raise ValueError("Context dictionary does not contain expected argument 'course'") + course_facet_value = CourseFacetValue.objects.filter( + course_id=course.id, + facet_value__facet__slug=self.property_name).values_list( + 'facet_value__value', flat=True) + return list(course_facet_value) + else: + # Interim hack: return an empty list, which should have a net zero + # effect if not enabling taxoman + return [] + # Source location options - either from the course or the about info FROM_ABOUT_INFO = from_about_dictionary FROM_COURSE_PROPERTY = from_course_property FROM_COURSE_MODE = from_course_mode + # Appsembler addition - Interim implementation + FROM_TAXOMAN = from_taxoman + class CourseAboutSearchIndexer(object): """ @@ -573,6 +605,14 @@ class CourseAboutSearchIndexer(object): AboutInfo("catalog_visibility", AboutInfo.PROPERTY, AboutInfo.FROM_COURSE_PROPERTY), ] + # Appsembler addition + if using_taxoman: + for facet in Facet.objects.all(): + print("facet = {}".format(facet)) + ABOUT_INFORMATION_TO_INCLUDE.append( + AboutInfo(facet.slug, AboutInfo.PROPERTY, AboutInfo.FROM_TAXOMAN) + ) + @classmethod def index_about_information(cls, modulestore, course): """ From 6a9954aecb05b1e853e25232e43d9d04dfcf6939 Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 23 Jan 2018 16:23:38 -0500 Subject: [PATCH 5/5] Appsembler: Enable conditional use of Taxoman to LMS Enables conditional use of Taxoman in devstack and prod/staging environments --- lms/djangoapps/courseware/views/views.py | 18 ++++++++++++++++++ lms/envs/aws_appsembler.py | 16 ++++++++++++++++ lms/envs/common.py | 3 +++ lms/envs/devstack_appsembler.py | 21 +++++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/lms/djangoapps/courseware/views/views.py b/lms/djangoapps/courseware/views/views.py index 2590ef37e250..5c3e3e438214 100644 --- a/lms/djangoapps/courseware/views/views.py +++ b/lms/djangoapps/courseware/views/views.py @@ -104,6 +104,19 @@ log = logging.getLogger("edx.courseware") +# Interim code to get courseware views to work with Taxoman +if settings.TAXOMAN_ENABLED: + try: + from taxoman_api.models import Facet + using_taxoman = True + except ImportError: + log.error('Taxoman enabled, but unable to import taxoman_api package (ImportError') + using_taxoman = False +else: + using_taxoman = False + + + # Only display the requirements on learner dashboard for # credit and verified modes. @@ -144,6 +157,11 @@ def courses(request): courses_list = [] programs_list = [] course_discovery_meanings = getattr(settings, 'COURSE_DISCOVERY_MEANINGS', {}) + if using_taxoman: + for facet in Facet.objects.all(): + if not course_discovery_meanings.get(facet.slug): + course_discovery_meanings[facet.slug] = { 'name': facet.name } + if not settings.FEATURES.get('ENABLE_COURSE_DISCOVERY'): courses_list = get_courses(request.user) diff --git a/lms/envs/aws_appsembler.py b/lms/envs/aws_appsembler.py index 8a82cf2ef401..65bb08dd23e1 100644 --- a/lms/envs/aws_appsembler.py +++ b/lms/envs/aws_appsembler.py @@ -3,6 +3,16 @@ from .aws import * from .appsembler import * +if FEATURES.get('ENABLE_TAXOMAN', False): + try: + import taxoman.settings + TAXOMAN_ENABLED = True + except ImportError: + TAXOMAN_ENABLED = False +else: + TAXOMAN_ENABLED = False + + ENV_APPSEMBLER_FEATURES = ENV_TOKENS.get('APPSEMBLER_FEATURES', {}) for feature, value in ENV_APPSEMBLER_FEATURES.items(): APPSEMBLER_FEATURES[feature] = value @@ -132,3 +142,9 @@ except ImportError: pass + +if TAXOMAN_ENABLED: + WEBPACK_LOADER['TAXOMAN_APP'] = { + 'BUNDLE_DIR_NAME': taxoman.settings.bundle_dir_name, + 'STATS_FILE': taxoman.settings.stats_file, + } diff --git a/lms/envs/common.py b/lms/envs/common.py index 58a173bab83b..18bd2f89ab6a 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2024,6 +2024,7 @@ # User API 'rest_framework', + 'rest_framework.authtoken', 'openedx.core.djangoapps.user_api', # Shopping cart @@ -3042,3 +3043,5 @@ ############## Settings for the Enterprise App ###################### ENTERPRISE_ENROLLMENT_API_URL = LMS_ROOT_URL + "/api/enrollment/v1/" + +WEBPACK_LOADER = {} diff --git a/lms/envs/devstack_appsembler.py b/lms/envs/devstack_appsembler.py index 7ba9aa2d3fec..3ca18ebd1d5e 100644 --- a/lms/envs/devstack_appsembler.py +++ b/lms/envs/devstack_appsembler.py @@ -1,9 +1,24 @@ # devstack_appsembler.py import os + from .devstack import * from .appsembler import * + +if FEATURES.get('ENABLE_TAXOMAN', False): + try: + # Just a check, we don't need it for the settings + import taxoman_api + # We need this for webpack loader + import taxoman.settings + TAXOMAN_ENABLED = True + except ImportError: + TAXOMAN_ENABLED = False +else: + TAXOMAN_ENABLED = False + + ENV_APPSEMBLER_FEATURES = ENV_TOKENS.get('APPSEMBLER_FEATURES', {}) for feature, value in ENV_APPSEMBLER_FEATURES.items(): APPSEMBLER_FEATURES[feature] = value @@ -123,3 +138,9 @@ # override devstack.py automatic enabling of courseware discovery FEATURES['ENABLE_COURSE_DISCOVERY'] = ENV_TOKENS['FEATURES'].get('ENABLE_COURSE_DISCOVERY', FEATURES['ENABLE_COURSE_DISCOVERY']) + +if TAXOMAN_ENABLED: + WEBPACK_LOADER['TAXOMAN_APP'] = { + 'BUNDLE_DIR_NAME': taxoman.settings.bundle_dir_name, + 'STATS_FILE': taxoman.settings.stats_file, + }