Enable mongodb to be used as back-end for git-based authoring workflow

cms/djangoapps/contentstore/management/commands/import.py

@@ -2,7 +2,7 @@
 Script for importing courseware from XML format
 """
 
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError, make_option
 from xmodule.modulestore.xml_importer import import_from_xml
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
@@ -14,18 +14,26 @@ class Command(BaseCommand):
     """
     help = 'Import the specified data directory into the default ModuleStore'
 
+    option_list = BaseCommand.option_list + (
+        make_option('--nostatic',
+                    action='store_true',
+                    help='Skip import of static content'),
+    )
+
     def handle(self, *args, **options):
         "Execute the command"
         if len(args) == 0:
-            raise CommandError("import requires at least one argument: <data directory> [<course dir>...]")
+            raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]")
 
         data_dir = args[0]
+        do_import_static = not (options.get('nostatic', False))
         if len(args) > 1:
             course_dirs = args[1:]
         else:
             course_dirs = None
         print("Importing.  Data_dir={data}, course_dirs={courses}".format(
             data=data_dir,
-            courses=course_dirs))
+            courses=course_dirs,
+            dis=do_import_static))
         import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False,
-                        static_content_store=contentstore(), verbose=True)
+                        static_content_store=contentstore(), verbose=True, do_import_static=do_import_static)

cms/djangoapps/contentstore/tests/test_contentstore.py

@@ -476,7 +476,7 @@ def test_asset_import(self):
         content_store = contentstore()
 
         module_store = modulestore('direct')
-        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)
+        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, verbose=True)
 
         course_location = CourseDescriptor.id_to_location('edX/toy/2012_Fall')
         course = module_store.get_item(course_location)

cms/djangoapps/contentstore/tests/test_import_nostatic.py

@@ -0,0 +1,123 @@
+#pylint: disable=E1101
+'''
+Tests for importing with no static
+'''
+
+from django.test.client import Client
+from django.test.utils import override_settings
+from django.conf import settings
+from path import path
+import copy
+
+from django.contrib.auth.models import User
+
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.contentstore.content import StaticContent
+
+from xmodule.course_module import CourseDescriptor
+
+from xmodule.exceptions import NotFoundError
+from uuid import uuid4
+
+
+TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
+TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % uuid4().hex
+
+
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
+class ContentStoreImportNoStaticTest(ModuleStoreTestCase):
+    """
+    Tests that rely on the toy and test_import_course courses.
+    NOTE: refactor using CourseFactory so they do not.
+    """
+    def setUp(self):
+
+        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
+        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
+        uname = 'testuser'
+        email = 'test+courses@edx.org'
+        password = 'foo'
+
+        # Create the use so we can log them in.
+        self.user = User.objects.create_user(uname, email, password)
+
+        # Note that we do not actually need to do anything
+        # for registration if we directly mark them active.
+        self.user.is_active = True
+        # Staff has access to view all courses
+        self.user.is_staff = True
+
+        # Save the data that we've just changed to the db.
+        self.user.save()
+
+        self.client = Client()
+        self.client.login(username=uname, password=password)
+
+    def load_test_import_course(self):
+        '''
+        Load the standard course used to test imports (for do_import_static=False behavior).
+        '''
+        content_store = contentstore()
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True)
+        course_location = CourseDescriptor.id_to_location('edX/test_import_course/2012_Fall')
+        course = module_store.get_item(course_location)
+        self.assertIsNotNone(course)
+
+        return module_store, content_store, course, course_location
+
+    def test_static_import(self):
+        '''
+        Stuff in static_import should always be imported into contentstore
+        '''
+        _, content_store, course, course_location = self.load_test_import_course()
+
+        # make sure we have ONE asset in our contentstore ("should_be_imported.html")
+        all_assets = content_store.get_all_content_for_course(course_location)
+        print "len(all_assets)=%d" % len(all_assets)
+        self.assertEqual(len(all_assets), 1)
+
+        content = None
+        try:
+            location = StaticContent.get_location_from_path('/c4x/edX/test_import_course/asset/should_be_imported.html')
+            content = content_store.find(location)
+        except NotFoundError:
+            pass
+
+        self.assertIsNotNone(content)
+
+        # make sure course.lms.static_asset_path is correct
+        print "static_asset_path = {0}".format(course.lms.static_asset_path)
+        self.assertEqual(course.lms.static_asset_path, 'test_import_course')
+
+    def test_asset_import_nostatic(self):
+        '''
+        This test validates that an image asset is NOT imported when do_import_static=False
+        '''
+        content_store = contentstore()
+
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
+
+        course_location = CourseDescriptor.id_to_location('edX/toy/2012_Fall')
+        module_store.get_item(course_location)
+
+        # make sure we have NO assets in our contentstore
+        all_assets = content_store.get_all_content_for_course(course_location)
+        print "len(all_assets)=%d" % len(all_assets)
+        self.assertEqual(len(all_assets), 0)
+
+    def test_no_static_link_rewrites_on_import(self):
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
+
+        handouts = module_store.get_item(Location(['i4x', 'edX', 'toy', 'course_info', 'handouts', None]))
+        self.assertIn('/static/', handouts.data)
+
+        handouts = module_store.get_item(Location(['i4x', 'edX', 'toy', 'html', 'toyhtml', None]))
+        self.assertIn('/static/', handouts.data)

common/djangoapps/static_replace/__init__.py

@@ -90,7 +90,7 @@ def replace_course_url(match):
     return re.sub(_url_replace_regex('/course/'), replace_course_url, text)
 
 
-def replace_static_urls(text, data_directory, course_id=None):
+def replace_static_urls(text, data_directory, course_id=None, static_asset_path=''):
     """
     Replace /static/$stuff urls either with their correct url as generated by collectstatic,
     (/static/$md5_hashed_stuff) or by the course-specific content static url
@@ -100,6 +100,7 @@ def replace_static_urls(text, data_directory, course_id=None):
     text: The source text to do the substitution in
     data_directory: The directory in which course data is stored
     course_id: The course identifier used to distinguish static content for this course in studio
+    static_asset_path: Path for static assets, which overrides data_directory and course_namespace, if nonempty
     """
 
     def replace_static_url(match):
@@ -116,7 +117,7 @@ def replace_static_url(match):
         if settings.DEBUG and finders.find(rest, True):
             return original
         # if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
-        elif course_id and modulestore().get_modulestore_type(course_id) != XML_MODULESTORE_TYPE:
+        elif (not static_asset_path) and course_id and modulestore().get_modulestore_type(course_id) != XML_MODULESTORE_TYPE:
             # first look in the static file pipeline and see if we are trying to reference
             # a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule)
 
@@ -135,7 +136,7 @@ def replace_static_url(match):
                 url = StaticContent.convert_legacy_static_url_with_course_id(rest, course_id)
         # Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
         else:
-            course_path = "/".join((data_directory, rest))
+            course_path = "/".join((static_asset_path or data_directory, rest))
 
             try:
                 if staticfiles_storage.exists(rest):
@@ -152,7 +153,7 @@ def replace_static_url(match):
 
 
     return re.sub(
-        _url_replace_regex('/static/(?!{data_dir})'.format(data_dir=data_directory)),
+        _url_replace_regex('/static/(?!{data_dir})'.format(data_dir=static_asset_path or data_directory)),
         replace_static_url,
         text
     )

common/djangoapps/xmodule_modifiers.py

@@ -76,7 +76,7 @@ def _get_html():
     return _get_html
 
 
-def replace_static_urls(get_html, data_dir, course_id=None):
+def replace_static_urls(get_html, data_dir, course_id=None, static_asset_path=''):
     """
     Updates the supplied module with a new get_html function that wraps
     the old get_html function and substitutes urls of the form /static/...
@@ -85,7 +85,7 @@ def replace_static_urls(get_html, data_dir, course_id=None):
 
     @wraps(get_html)
     def _get_html():
-        return static_replace.replace_static_urls(get_html(), data_dir, course_id)
+        return static_replace.replace_static_urls(get_html(), data_dir, course_id, static_asset_path=static_asset_path)
     return _get_html
 
 

common/lib/xmodule/xmodule/modulestore/inheritance.py

@@ -9,7 +9,8 @@
     # intended to be set per-course, but can be overridden in for specific
     # elements.  Can be a float.
     'days_early_for_beta',
-    'giturl'  # for git edit link
+    'giturl',  # for git edit link
+    'static_asset_path',       # for static assets placed outside xcontent contentstore
 )
 
 

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -10,7 +10,9 @@
 from xmodule.tests import DATA_DIR
 from xmodule.modulestore import Location
 from xmodule.modulestore.mongo import MongoModuleStore, MongoKeyValueStore
-from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.draft import DraftModuleStore
+from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
+from xmodule.contentstore.mongo import MongoContentStore
 
 from xmodule.modulestore.tests.test_modulestore import check_path_to_location
 
@@ -35,7 +37,7 @@ def setupClass(cls):
         # is ok only as long as none of the tests modify the db.
         # If (when!) that changes, need to either reload the db, or load
         # once and copy over to a tmp db for each test.
-        cls.store = cls.initdb()
+        cls.store, cls.content_store, cls.draft_store = cls.initdb()
 
     @classmethod
     def teardownClass(cls):
@@ -46,10 +48,28 @@ def teardownClass(cls):
     def initdb():
         # connect to the db
         store = MongoModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
+        # since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
+        # as well
+        content_store = MongoContentStore(HOST, DB)
+        #
+        # Also test draft store imports
+        #
+        draft_store = DraftModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
         # Explicitly list the courses to load (don't want the big one)
-        courses = ['toy', 'simple']
-        import_from_xml(store, DATA_DIR, courses)
-        return store
+        courses = ['toy', 'simple', 'simple_with_draft']
+        import_from_xml(store, DATA_DIR, courses, draft_store=draft_store, static_content_store=content_store)
+
+        # also test a course with no importing of static content
+        import_from_xml(
+            store,
+            DATA_DIR,
+            ['test_import_course'],
+            static_content_store=content_store,
+            do_import_static=False,
+            verbose=True
+        )
+
+        return store, content_store, draft_store
 
     @staticmethod
     def destroy_db(connection):
@@ -77,10 +97,12 @@ def test_mongo_modulestore_type(self):
     def test_get_courses(self):
         '''Make sure the course objects loaded properly'''
         courses = self.store.get_courses()
-        assert_equals(len(courses), 2)
+        assert_equals(len(courses), 4)
         courses.sort(key=lambda c: c.id)
         assert_equals(courses[0].id, 'edX/simple/2012_Fall')
-        assert_equals(courses[1].id, 'edX/toy/2012_Fall')
+        assert_equals(courses[1].id, 'edX/simple_with_draft/2012_Fall')
+        assert_equals(courses[2].id, 'edX/test_import_course/2012_Fall')
+        assert_equals(courses[3].id, 'edX/toy/2012_Fall')
 
     def test_loads(self):
         assert_not_equals(
@@ -112,6 +134,13 @@ def test_path_to_location(self):
         '''Make sure that path_to_location works'''
         check_path_to_location(self.store)
 
+    def test_xlinter(self):
+        '''
+        Run through the xlinter, we know the 'toy' course has violations, but the
+        number will continue to grow over time, so just check > 0
+        '''
+        assert_not_equals(perform_xlint(DATA_DIR, ['toy']), 0)
+
     def test_get_courses_has_no_templates(self):
         courses = self.store.get_courses()
         for course in courses:
@@ -129,7 +158,7 @@ def get_tab_name(index):
 
             Assumes the information is desired for courses[1] ('toy' course).
             """
-            return courses[1].tabs[index]['name']
+            return courses[2].tabs[index]['name']
 
         # There was a bug where model.save was not getting called after the static tab name
         # was set set for tabs that have a URL slug. 'Syllabus' and 'Resources' fall into that