Merge appsembler/tahoe/develop into appsembler/tahoe/master

cms/envs/common.py

@@ -882,7 +882,11 @@
 ################################# CELERY ######################################
 
 # Auto discover tasks fails to detect contentstore tasks
-CELERY_IMPORTS = ('cms.djangoapps.contentstore.tasks')
+CELERY_IMPORTS = (
+    'cms.djangoapps.contentstore.tasks',
+    'openedx.core.djangoapps.bookmarks.tasks',
+    'openedx.core.djangoapps.ccxcon.tasks',
+)
 
 # Message configuration
 

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -51,7 +51,7 @@ def close_connections(self):
         """
         Closes any open connections to the underlying databases
         """
-        self.fs_files.database.connection.close()
+        self.fs_files.database.client.close()
 
     def _drop_database(self, database=True, collections=True, connections=True):
         """
@@ -65,10 +65,10 @@ def _drop_database(self, database=True, collections=True, connections=True):
 
         If connections is True, then close the connection to the database as well.
         """
-        connection = self.fs_files.database.connection
+        connection = self.fs_files.database.client
 
         if database:
-            connection.drop_database(self.fs_files.database)
+            connection.drop_database(self.fs_files.database.name)
         elif collections:
             self.fs_files.drop()
             self.chunks.drop()
@@ -297,15 +297,18 @@ def _get_all_content_for_course(self,
                 }
             })
 
-        items = self.fs_files.aggregate(pipeline_stages)
-        if items['result']:
-            result = items['result'][0]
-            count = result['count']
-            assets = list(result['results'])
-        else:
-            # no results
-            count = 0
-            assets = []
+        cursor = self.fs_files.aggregate(pipeline_stages)
+        # Set values if result of query is empty
+        count = 0
+        assets = []
+        try:
+            result = cursor.next()
+            if result:
+                count = result['count']
+                assets = list(result['results'])
+        except StopIteration:
+            # Skip if no assets were returned
+            pass
 
         # We're constructing the asset key immediately after retrieval from the database so that
         # callers are insulated from knowing how our identifiers are stored.

common/lib/xmodule/xmodule/course_metadata_utils.py

@@ -159,7 +159,7 @@ def sorting_dates(start, advertised_start, announcement):
         start = dateutil.parser.parse(advertised_start)
         if start.tzinfo is None:
             start = start.replace(tzinfo=utc)
-    except (ValueError, AttributeError):
+    except (TypeError, ValueError, AttributeError):
         start = start
 
     now = datetime.now(utc)

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -599,14 +599,7 @@ def close_connections(self):
         """
         Closes any open connections to the underlying database
         """
-        self.collection.database.connection.close()
-
-    def mongo_wire_version(self):
-        """
-        Returns the wire version for mongo. Only used to unit tests which instrument the connection.
-        """
-        self.database.connection._ensure_connected()
-        return self.database.connection.max_wire_version
+        self.collection.database.client.close()
 
     def _drop_database(self, database=True, collections=True, connections=True):
         """
@@ -623,14 +616,14 @@ def _drop_database(self, database=True, collections=True, connections=True):
         # drop the assets
         super(MongoModuleStore, self)._drop_database(database, collections, connections)
 
-        connection = self.collection.database.connection
+        connection = self.collection.database.client
 
         if database:
             connection.drop_database(self.collection.database.proxied_object)
         elif collections:
             self.collection.drop()
         else:
-            self.collection.remove({})
+            self.collection.delete_many({})
 
         if connections:
             connection.close()
@@ -1908,8 +1901,8 @@ def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
             dest_course_key (CourseKey): identifier of course to copy to
         """
         source_assets = self._find_course_assets(source_course_key)
-        dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': unicode(dest_course_key)}
-        self.asset_collection.remove({'course_id': unicode(dest_course_key)})
+        dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': six.text_type(dest_course_key)}
+        self.asset_collection.delete_many({'course_id': six.text_type(dest_course_key)})
         # Update the document.
         self.asset_collection.insert(dest_assets)
 
@@ -1981,7 +1974,7 @@ def delete_all_asset_metadata(self, course_key, user_id):
         # A single document exists per course to store the course asset metadata.
         try:
             course_assets = self._find_course_assets(course_key)
-            self.asset_collection.remove(course_assets.doc_id)
+            self.asset_collection.delete_many({'_id': course_assets.doc_id})
         except ItemNotFoundError:
             # When deleting asset metadata, if a course's asset metadata is not present, no big deal.
             pass
@@ -1990,9 +1983,11 @@ def heartbeat(self):
         """
         Check that the db is reachable.
         """
-        if self.database.connection.alive():
+        try:
+            # The ismaster command is cheap and does not require auth.
+            self.database.client.admin.command('ismaster')
             return {ModuleStoreEnum.Type.mongo: True}
-        else:
+        except pymongo.errors.ConnectionFailure:
             raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo')
 
     def ensure_indexes(self):

common/lib/xmodule/xmodule/modulestore/mongo/draft.py

@@ -166,7 +166,7 @@ def delete_course(self, course_key, user_id):
 
         # delete all of the db records for the course
         course_query = self._course_key_to_son(course_key)
-        self.collection.remove(course_query, multi=True)
+        self.collection.delete_many(course_query)
         self.delete_all_asset_metadata(course_key, user_id)
 
         self._emit_course_deleted_signal(course_key)
@@ -639,7 +639,7 @@ def _internal(tier):
         if len(to_be_deleted) > 0:
             bulk_record = self._get_bulk_ops_record(root_usages[0].course_key)
             bulk_record.dirty = True
-            self.collection.remove({'_id': {'$in': to_be_deleted}}, safe=self.collection.safe)
+            self.collection.delete_many({'_id': {'$in': to_be_deleted}})
 
     @memoize_in_request_cache('request_cache')
     def has_changes(self, xblock):
@@ -743,7 +743,7 @@ def _internal_depth_first(item_location, is_root):
         bulk_record = self._get_bulk_ops_record(course_key)
         if len(to_be_deleted) > 0:
             bulk_record.dirty = True
-            self.collection.remove({'_id': {'$in': to_be_deleted}})
+            self.collection.delete_many({'_id': {'$in': to_be_deleted}})
 
         self._flag_publish_event(course_key)
 

common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py

@@ -308,9 +308,11 @@ def heartbeat(self):
         """
         Check that the db is reachable.
         """
-        if self.database.connection.alive():
+        try:
+            # The ismaster command is cheap and does not require auth.
+            self.database.client.admin.command('ismaster')
             return True
-        else:
+        except pymongo.errors.ConnectionFailure:
             raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo')
 
     def get_structure(self, key, course_context=None):
@@ -601,13 +603,7 @@ def close_connections(self):
         """
         Closes any open connections to the underlying databases
         """
-        self.database.connection.close()
-
-    def mongo_wire_version(self):
-        """
-        Returns the wire version for mongo. Only used to unit tests which instrument the connection.
-        """
-        return self.database.connection.max_wire_version
+        self.database.client.close()
 
     def _drop_database(self, database=True, collections=True, connections=True):
         """
@@ -621,7 +617,7 @@ def _drop_database(self, database=True, collections=True, connections=True):
 
         If connections is True, then close the connection to the database as well.
         """
-        connection = self.database.connection
+        connection = self.database.client
 
         if database:
             connection.drop_database(self.database.name)

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -736,12 +736,6 @@ def close_connections(self):
         """
         self.db_connection.close_connections()
 
-    def mongo_wire_version(self):
-        """
-        Returns the wire version for mongo. Only used to unit tests which instrument the connection.
-        """
-        return self.db_connection.mongo_wire_version
-
     def _drop_database(self, database=True, collections=True, connections=True):
         """
         A destructive operation to drop the underlying database and close all connections.

common/lib/xmodule/xmodule/modulestore/tests/factories.py

@@ -606,8 +606,6 @@ def mongo_uses_error_check(store):
     """
     Does mongo use the error check as a separate message?
     """
-    if hasattr(store, 'mongo_wire_version'):
-        return store.mongo_wire_version() <= 1
     if hasattr(store, 'modulestores'):
         return any([mongo_uses_error_check(substore) for substore in store.modulestores])
     return False
@@ -626,16 +624,16 @@ def check_mongo_calls_range(max_finds=float("inf"), min_finds=0, max_sends=None,
     :param min_sends: If non-none, make sure number of send calls are >=min_sends
     """
     with check_sum_of_calls(
-        pymongo.message,
-        ['query', 'get_more'],
+        pymongo.collection.Collection,
+        ['find'],
         max_finds,
         min_finds,
     ):
         if max_sends is not None or min_sends is not None:
             with check_sum_of_calls(
-                pymongo.message,
+                pymongo.collection.Collection,
                 # mongo < 2.6 uses insert, update, delete and _do_batched_insert. >= 2.6 _do_batched_write
-                ['insert', 'update', 'delete', '_do_batched_write_command', '_do_batched_insert', ],
+                ['insert', 'update', 'bulk_write', '_delete'],
                 max_sends if max_sends is not None else float("inf"),
                 min_sends if min_sends is not None else 0,
             ):

common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py

@@ -151,11 +151,11 @@ def _import_course(self, content_store, modulestore):
         (MIXED_OLD_MONGO_MODULESTORE_BUILDER, 0, True, False, 359),
         # The line below shows the way this traversal *should* be done
         # (if you'll eventually access all the fields and load all the definitions anyway).
-        (MIXED_SPLIT_MODULESTORE_BUILDER, None, False, True, 4),
+        (MIXED_SPLIT_MODULESTORE_BUILDER, None, False, True, 3),
         (MIXED_SPLIT_MODULESTORE_BUILDER, None, True, True, 38),
         (MIXED_SPLIT_MODULESTORE_BUILDER, 0, False, True, 38),
         (MIXED_SPLIT_MODULESTORE_BUILDER, 0, True, True, 38),
-        (MIXED_SPLIT_MODULESTORE_BUILDER, None, False, False, 4),
+        (MIXED_SPLIT_MODULESTORE_BUILDER, None, False, False, 3),
         (MIXED_SPLIT_MODULESTORE_BUILDER, None, True, False, 3),
         (MIXED_SPLIT_MODULESTORE_BUILDER, 0, False, False, 3),
         (MIXED_SPLIT_MODULESTORE_BUILDER, 0, True, False, 3)
@@ -177,7 +177,7 @@ def test_number_mongo_calls(self, store_builder, depth, lazy, access_all_block_f
 
     @ddt.data(
         (MIXED_OLD_MONGO_MODULESTORE_BUILDER, 176),
-        (MIXED_SPLIT_MODULESTORE_BUILDER, 5),
+        (MIXED_SPLIT_MODULESTORE_BUILDER, 4),
     )
     @ddt.unpack
     def test_lazy_when_course_previously_cached(self, store_builder, num_mongo_calls):

common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_mongo_connection.py

@@ -1,6 +1,9 @@
 """ Test the behavior of split_mongo/MongoConnection """
 import unittest
 from mock import patch
+from pymongo.errors import ConnectionFailure
+
+from xmodule.exceptions import HeartbeatFailure
 from xmodule.modulestore.split_mongo.mongo_connection import MongoConnection
 from xmodule.exceptions import HeartbeatFailure
 
@@ -14,7 +17,7 @@ class TestHeartbeatFailureException(unittest.TestCase):
     def test_heartbeat_raises_exception_when_connection_alive_is_false(self, *calls):
         # pylint: disable=W0613
         with patch('mongodb_proxy.MongoProxy') as mock_proxy:
-            mock_proxy.return_value.alive.return_value = False
+            mock_proxy.return_value.admin.command.side_effect = ConnectionFailure('Test')
             useless_conn = MongoConnection('useless', 'useless', 'useless')
 
             with self.assertRaises(HeartbeatFailure):

common/lib/xmodule/xmodule/mongo_utils.py

@@ -4,11 +4,20 @@
 import logging
 
 import pymongo
-from pymongo import ReadPreference
 from mongodb_proxy import MongoProxy
+from pymongo.read_preferences import (
+    ReadPreference,
+    read_pref_mode_from_name,
+    _MONGOS_MODES,
+    _MODES
+)
+
 
 logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
+# This will yeld a map of all available Mongo modes and their name
+MONGO_READ_PREFERENCE_MAP = dict(zip(_MONGOS_MODES, _MODES))
+
 
 # pylint: disable=bad-continuation
 def connect_to_mongodb(
@@ -34,10 +43,19 @@ def connect_to_mongodb(
         # No 'replicaSet' in kwargs - so no secondary reads.
         mongo_client_class = pymongo.MongoClient
 
-    # If read_preference is given as a name of a valid ReadPreference.<NAME> constant
-    # such as "SECONDARY_PREFERRED", convert it. Otherwise pass it through unchanged.
+    # If the MongoDB server uses a separate authentication database that should be specified here
+    auth_source = kwargs.pop('auth_source', '') or None
+
+    # If read_preference is given as a name of a valid ReadPreference.<NAME>
+    # constant such as "SECONDARY_PREFERRED" or a mongo mode such as
+    # "secondaryPreferred", convert it. Otherwise pass it through unchanged.
     if 'read_preference' in kwargs:
-        read_preference = getattr(ReadPreference, kwargs['read_preference'], None)
+        read_preference = MONGO_READ_PREFERENCE_MAP.get(
+            kwargs['read_preference'],
+            kwargs['read_preference']
+        )
+
+        read_preference = getattr(ReadPreference, read_preference, None)
         if read_preference is not None:
             kwargs['read_preference'] = read_preference
 
@@ -58,9 +76,14 @@ def connect_to_mongodb(
             wait_time=retry_wait_time
         )
 
+    # default the authSource to be whatever db we are connecting to (for backwards compatiblity)
+    authSource = db
+    if kwargs.get('authSource'):
+        authSource = kwargs.get('authSource')
+
     # If credentials were provided, authenticate the user.
     if user is not None and password is not None:
-        mongo_conn.authenticate(user, password)
+        mongo_conn.authenticate(user, password, authSource)
 
     return mongo_conn
 

lms/djangoapps/course_api/views.py

@@ -241,11 +241,6 @@ class CourseListView(DeveloperErrorViewMixin, ListAPIView):
     #   - https://github.com/elastic/elasticsearch/commit/8b0a863d427b4ebcbcfb1dcd69c996c52e7ae05e
     results_size_infinity = 10000
 
-    # Return all the results, 10K is the maximum allowed value for ElasticSearch.
-    # We should use 0 after upgrading to 1.1+:
-    #   - https://github.com/elastic/elasticsearch/commit/8b0a863d427b4ebcbcfb1dcd69c996c52e7ae05e
-    results_size_infinity = 10000
-
     def get_queryset(self):
         """
         Return a list of courses visible to the user.

lms/djangoapps/dashboard/git_import.py

@@ -342,5 +342,5 @@ def add_repo(repo, rdir_in, branch=None):
     )
     cil.save()
 
-    log.debug('saved CourseImportLog for %s', cil.course_id)
-    mdb.disconnect()
+    log.debug(u'saved CourseImportLog for %s', cil.course_id)
+    mdb.close()

lms/djangoapps/dashboard/sysadmin.py

@@ -652,7 +652,7 @@ def get(self, request, *args, **kwargs):
             page = min(max(1, given_page), paginator.num_pages)
             logs = paginator.page(page)
 
-        mdb.disconnect()
+        mdb.close()
         context = {
             'logs': logs,
             'course_id': text_type(course_id) if course_id else None,