Enable mongodb to be used as back-end for git-based authoring workflow

cms/djangoapps/contentstore/management/commands/import.py

@@ -2,7 +2,7 @@
 Script for importing courseware from XML format
 """
 
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError, make_option
 from xmodule.modulestore.xml_importer import import_from_xml
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
@@ -14,18 +14,26 @@ class Command(BaseCommand):
     """
     help = 'Import the specified data directory into the default ModuleStore'
 
+    option_list = BaseCommand.option_list + (
+        make_option('--nostatic',
+                    action='store_true',
+                    help='Skip import of static content'),
+                   )
+
     def handle(self, *args, **options):
         "Execute the command"
         if len(args) == 0:
-            raise CommandError("import requires at least one argument: <data directory> [<course dir>...]")
+            raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]")
 
         data_dir = args[0]
+        do_import_static = not (options.get('nostatic', False))
         if len(args) > 1:
             course_dirs = args[1:]
         else:
             course_dirs = None
         print("Importing.  Data_dir={data}, course_dirs={courses}".format(
             data=data_dir,
-            courses=course_dirs))
+            courses=course_dirs,
+            dis=do_import_static))
         import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False,
-                        static_content_store=contentstore(), verbose=True)
+                        static_content_store=contentstore(), verbose=True, do_import_static=do_import_static)

common/djangoapps/static_replace/__init__.py

@@ -90,7 +90,7 @@ def replace_course_url(match):
     return re.sub(_url_replace_regex('/course/'), replace_course_url, text)
 
 
-def replace_static_urls(text, data_directory, course_id=None):
+def replace_static_urls(text, data_directory, course_id=None, static_asset_path=''):
     """
     Replace /static/$stuff urls either with their correct url as generated by collectstatic,
     (/static/$md5_hashed_stuff) or by the course-specific content static url
@@ -100,6 +100,7 @@ def replace_static_urls(text, data_directory, course_id=None):
     text: The source text to do the substitution in
     data_directory: The directory in which course data is stored
     course_id: The course identifier used to distinguish static content for this course in studio
+    static_asset_path: Path for static assets, which overrides data_directory and course_namespace, if nonempty
     """
 
     def replace_static_url(match):
@@ -127,7 +128,7 @@ def replace_static_url(match):
                 url = StaticContent.convert_legacy_static_url_with_course_id(rest, course_id)
         # Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
         else:
-            course_path = "/".join((data_directory, rest))
+            course_path = "/".join((static_asset_path or data_directory, rest))
 
             try:
                 if staticfiles_storage.exists(rest):
@@ -143,7 +144,7 @@ def replace_static_url(match):
         return "".join([quote, url, quote])
 
     return re.sub(
-        _url_replace_regex('/static/(?!{data_dir})'.format(data_dir=data_directory)),
+        _url_replace_regex('/static/(?!{data_dir})'.format(data_dir=static_asset_path or data_directory)),
         replace_static_url,
         text
     )

common/djangoapps/xmodule_modifiers.py

@@ -76,7 +76,7 @@ def _get_html():
     return _get_html
 
 
-def replace_static_urls(get_html, data_dir, course_id=None):
+def replace_static_urls(get_html, data_dir, course_id=None, static_asset_path=''):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -85,7 +85,7 @@ def replace_static_urls(get_html, data_dir, course_id=None):
 
     @wraps(get_html)
     def _get_html():
-        return static_replace.replace_static_urls(get_html(), data_dir, course_id)
+        return static_replace.replace_static_urls(get_html(), data_dir, course_id, static_asset_path=static_asset_path)
     return _get_html
 
 

common/lib/xmodule/xmodule/modulestore/inheritance.py

@@ -9,7 +9,8 @@
     # intended to be set per-course, but can be overridden in for specific
     # elements.  Can be a float.
     'days_early_for_beta',
-    'giturl'  # for git edit link
+    'giturl',  # for git edit link
+    'static_asset_path',       # for static assets placed outside xcontent contentstore
 )
 
 

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -51,7 +51,10 @@ def import_static_content(modules, course_loc, course_data_path, static_content_
                     content.thumbnail_location = thumbnail_location
 
                 #then commit the content
-                static_content_store.save(content)
+                try:
+                    static_content_store.save(content)
+                except Exception as err:
+                    log.exception('Error importing {0}, error={1}'.format(fullname_with_subpath, err))
 
                 #store the remapping information which will be needed to subsitute in the module data
                 remap_dict[fullname_with_subpath] = content_loc.name
@@ -64,7 +67,8 @@ def import_static_content(modules, course_loc, course_data_path, static_content_
 def import_from_xml(store, data_dir, course_dirs=None,
                     default_class='xmodule.raw_module.RawDescriptor',
                     load_error_modules=True, static_content_store=None, target_location_namespace=None,
-                    verbose=False, draft_store=None):
+                    verbose=False, draft_store=None,
+                    do_import_static=True):
     """
     Import the specified xml data_dir into the "store" modulestore,
     using org and course as the location org and course.
@@ -77,6 +81,10 @@ def import_from_xml(store, data_dir, course_dirs=None,
     expects a 'url_name' as an identifier to where things are on disk e.g. ../policies/<url_name>/policy.json as well as metadata keys in
     the policy.json. so we need to keep the original url_name during import
 
+    do_import_static: if False, then static files are not imported into the static content store.  This can be employed for courses which
+                      have substantial unchanging static content, which is to inefficient to import every time the course is loaded.
+                      Static content for some courses may also be served directly by nginx, instead of going through django.
+
     """
 
     xml_module_store = XMLModuleStore(
@@ -116,8 +124,17 @@ def import_from_xml(store, data_dir, course_dirs=None,
                     course_data_path = path(data_dir) / module.data_dir
                     course_location = module.location
 
+                    log.debug('======> IMPORTING course to location {0}'.format(course_location))
+
                     module = remap_namespace(module, target_location_namespace)
 
+                    if not do_import_static:
+                        module.lms.static_asset_path = module.data_dir			# for old-style xblock where this was actually linked to kvs
+                        module._model_data['static_asset_path'] = module.data_dir
+                        log.debug('course static_asset_path={0}'.format(module.lms.static_asset_path))
+
+                    log.debug('course data_dir={0}'.format(module.data_dir))
+
                     # cdodge: more hacks (what else). Seems like we have a problem when importing a course (like 6.002) which
                     # does not have any tabs defined in the policy file. The import goes fine and then displays fine in LMS,
                     # but if someone tries to add a new tab in the CMS, then the LMS barfs because it expects that -
@@ -129,18 +146,36 @@ def import_from_xml(store, data_dir, course_dirs=None,
                                        {"type": "wiki", "name": "Wiki"}]  # note, add 'progress' when we can support it on Edge
 
                     import_module(module, store, course_data_path, static_content_store, course_location,
-                                  target_location_namespace or course_location)
+                                  target_location_namespace or course_location, do_import_static=do_import_static)
 
                     course_items.append(module)
 
             # then import all the static content
-            if static_content_store is not None:
+            if static_content_store is not None and do_import_static:
                 _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location
 
                 # first pass to find everything in /static/
                 import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store,
                                       _namespace_rename, subpath='static', verbose=verbose)
 
+            elif verbose and not do_import_static:
+                log.debug('Skipping import of static content, since do_import_static={0}'.format(do_import_static))
+
+            # no matter what do_import_static is, import "static_import" directory
+
+            # This is needed because the "about" pages (eg "overview") are loaded via load_extra_content, and
+            # do not inherit the lms metadata from the course module, and thus do not get "static_content_store"
+            # properly defined.   Static content referenced in those extra pages thus need to come through the
+            # c4x:// contentstore, unfortunately.  Tell users to copy that content into the "static_import" subdir.
+
+            simport = 'static_import'
+            if os.path.exists(course_data_path / simport):
+                _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location
+
+                import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store,
+                                      _namespace_rename, subpath=simport, verbose=verbose)
+
+
             # finally loop through all the modules
             for module in xml_module_store.modules[course_id].itervalues():
                 if module.category == 'course':
@@ -156,7 +191,8 @@ def import_from_xml(store, data_dir, course_dirs=None,
                     log.debug('importing module location {0}'.format(module.location))
 
                 import_module(module, store, course_data_path, static_content_store, course_location,
-                              target_location_namespace if target_location_namespace else course_location)
+                              target_location_namespace if target_location_namespace else course_location,
+                              do_import_static=do_import_static)
 
             # now import any 'draft' items
             if draft_store is not None:
@@ -176,7 +212,8 @@ def import_from_xml(store, data_dir, course_dirs=None,
 
 
 def import_module(module, store, course_data_path, static_content_store,
-                  source_course_location, dest_course_location, allow_not_found=False):
+                  source_course_location, dest_course_location, allow_not_found=False,
+                  do_import_static=True):
 
     logging.debug('processing import of module {0}...'.format(module.location.url()))
 
@@ -196,7 +233,7 @@ def import_module(module, store, course_data_path, static_content_store,
     else:
         module_data = content
 
-    if isinstance(module_data, basestring):
+    if isinstance(module_data, basestring) and do_import_static:
         # we want to convert all 'non-portable' links in the module_data (if it is a string) to
         # portable strings (e.g. /static/)
         module_data = rewrite_nonportable_content_links(

lms/djangoapps/courseware/courses.py

@@ -82,7 +82,7 @@ def course_image_url(course):
     """Try to look up the image url for the course.  If it's not found,
     log an error and return the dead link"""
     if modulestore().get_modulestore_type(course.location.course_id) == XML_MODULESTORE_TYPE:
-        return '/static/' + course.data_dir + "/images/course_image.jpg"
+        return '/static/' + (course.lms.static_asset_path or getattr(course, 'data_dir', '')) + "/images/course_image.jpg"
     else:
         loc = course.location._replace(tag='c4x', category='asset', name='images_course_image.jpg')
         _path = StaticContent.get_url_path_from_location(loc)

lms/djangoapps/courseware/module_render.py

@@ -349,6 +349,7 @@ def publish(event):
             static_replace.replace_static_urls,
             data_directory=getattr(descriptor, 'data_dir', None),
             course_id=course_id,
+            static_asset_path=descriptor.lms.static_asset_path,
         ),
         replace_course_urls=partial(
             static_replace.replace_course_urls,
@@ -407,7 +408,8 @@ def publish(event):
     module.get_html = replace_static_urls(
         _get_html,
         getattr(descriptor, 'data_dir', None),
-        course_id=course_id
+        course_id=course_id,
+        static_asset_path=descriptor.lms.static_asset_path
     )
 
     # Allow URLs of the form '/course/' refer to the root of multicourse directory

lms/xmodule_namespace.py

@@ -56,3 +56,4 @@ class LmsNamespace(Namespace):
         default=None,
         scope=Scope.settings
     )
+    static_asset_path = String(help="Path to use for static assets - overrides Studio c4x://", scope=Scope.settings, default='')