Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 15 additions & 12 deletions common/lib/xmodule/xmodule/contentstore/mongo.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def close_connections(self):
"""
Closes any open connections to the underlying databases
"""
self.fs_files.database.connection.close()
self.fs_files.database.client.close()

def _drop_database(self, database=True, collections=True, connections=True):
"""
Expand All @@ -65,10 +65,10 @@ def _drop_database(self, database=True, collections=True, connections=True):

If connections is True, then close the connection to the database as well.
"""
connection = self.fs_files.database.connection
connection = self.fs_files.database.client

if database:
connection.drop_database(self.fs_files.database)
connection.drop_database(self.fs_files.database.name)
elif collections:
self.fs_files.drop()
self.chunks.drop()
Expand Down Expand Up @@ -297,15 +297,18 @@ def _get_all_content_for_course(self,
}
})

items = self.fs_files.aggregate(pipeline_stages)
if items['result']:
result = items['result'][0]
count = result['count']
assets = list(result['results'])
else:
# no results
count = 0
assets = []
cursor = self.fs_files.aggregate(pipeline_stages)
# Set values if result of query is empty
count = 0
assets = []
try:
result = cursor.next()
if result:
count = result['count']
assets = list(result['results'])
except StopIteration:
# Skip if no assets were returned
pass

# We're constructing the asset key immediately after retrieval from the database so that
# callers are insulated from knowing how our identifiers are stored.
Expand Down
2 changes: 1 addition & 1 deletion common/lib/xmodule/xmodule/course_metadata_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ def sorting_dates(start, advertised_start, announcement):
start = dateutil.parser.parse(advertised_start)
if start.tzinfo is None:
start = start.replace(tzinfo=utc)
except (ValueError, AttributeError):
except (TypeError, ValueError, AttributeError):
start = start

now = datetime.now(utc)
Expand Down
25 changes: 10 additions & 15 deletions common/lib/xmodule/xmodule/modulestore/mongo/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -599,14 +599,7 @@ def close_connections(self):
"""
Closes any open connections to the underlying database
"""
self.collection.database.connection.close()

def mongo_wire_version(self):
"""
Returns the wire version for mongo. Only used to unit tests which instrument the connection.
"""
self.database.connection._ensure_connected()
return self.database.connection.max_wire_version
self.collection.database.client.close()

def _drop_database(self, database=True, collections=True, connections=True):
"""
Expand All @@ -623,14 +616,14 @@ def _drop_database(self, database=True, collections=True, connections=True):
# drop the assets
super(MongoModuleStore, self)._drop_database(database, collections, connections)

connection = self.collection.database.connection
connection = self.collection.database.client

if database:
connection.drop_database(self.collection.database.proxied_object)
elif collections:
self.collection.drop()
else:
self.collection.remove({})
self.collection.delete_many({})

if connections:
connection.close()
Expand Down Expand Up @@ -1908,8 +1901,8 @@ def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
dest_course_key (CourseKey): identifier of course to copy to
"""
source_assets = self._find_course_assets(source_course_key)
dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': unicode(dest_course_key)}
self.asset_collection.remove({'course_id': unicode(dest_course_key)})
dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': six.text_type(dest_course_key)}
self.asset_collection.delete_many({'course_id': six.text_type(dest_course_key)})
# Update the document.
self.asset_collection.insert(dest_assets)

Expand Down Expand Up @@ -1981,7 +1974,7 @@ def delete_all_asset_metadata(self, course_key, user_id):
# A single document exists per course to store the course asset metadata.
try:
course_assets = self._find_course_assets(course_key)
self.asset_collection.remove(course_assets.doc_id)
self.asset_collection.delete_many({'_id': course_assets.doc_id})
except ItemNotFoundError:
# When deleting asset metadata, if a course's asset metadata is not present, no big deal.
pass
Expand All @@ -1990,9 +1983,11 @@ def heartbeat(self):
"""
Check that the db is reachable.
"""
if self.database.connection.alive():
try:
# The ismaster command is cheap and does not require auth.
self.database.client.admin.command('ismaster')
return {ModuleStoreEnum.Type.mongo: True}
else:
except pymongo.errors.ConnectionFailure:
raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo')

def ensure_indexes(self):
Expand Down
6 changes: 3 additions & 3 deletions common/lib/xmodule/xmodule/modulestore/mongo/draft.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def delete_course(self, course_key, user_id):

# delete all of the db records for the course
course_query = self._course_key_to_son(course_key)
self.collection.remove(course_query, multi=True)
self.collection.delete_many(course_query)
self.delete_all_asset_metadata(course_key, user_id)

self._emit_course_deleted_signal(course_key)
Expand Down Expand Up @@ -639,7 +639,7 @@ def _internal(tier):
if len(to_be_deleted) > 0:
bulk_record = self._get_bulk_ops_record(root_usages[0].course_key)
bulk_record.dirty = True
self.collection.remove({'_id': {'$in': to_be_deleted}}, safe=self.collection.safe)
self.collection.delete_many({'_id': {'$in': to_be_deleted}})

@memoize_in_request_cache('request_cache')
def has_changes(self, xblock):
Expand Down Expand Up @@ -743,7 +743,7 @@ def _internal_depth_first(item_location, is_root):
bulk_record = self._get_bulk_ops_record(course_key)
if len(to_be_deleted) > 0:
bulk_record.dirty = True
self.collection.remove({'_id': {'$in': to_be_deleted}})
self.collection.delete_many({'_id': {'$in': to_be_deleted}})

self._flag_publish_event(course_key)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -308,9 +308,11 @@ def heartbeat(self):
"""
Check that the db is reachable.
"""
if self.database.connection.alive():
try:
# The ismaster command is cheap and does not require auth.
self.database.client.admin.command('ismaster')
return True
else:
except pymongo.errors.ConnectionFailure:
raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo')

def get_structure(self, key, course_context=None):
Expand Down Expand Up @@ -601,13 +603,7 @@ def close_connections(self):
"""
Closes any open connections to the underlying databases
"""
self.database.connection.close()

def mongo_wire_version(self):
"""
Returns the wire version for mongo. Only used to unit tests which instrument the connection.
"""
return self.database.connection.max_wire_version
self.database.client.close()

def _drop_database(self, database=True, collections=True, connections=True):
"""
Expand All @@ -621,7 +617,7 @@ def _drop_database(self, database=True, collections=True, connections=True):

If connections is True, then close the connection to the database as well.
"""
connection = self.database.connection
connection = self.database.client

if database:
connection.drop_database(self.database.name)
Expand Down
6 changes: 0 additions & 6 deletions common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
Original file line number Diff line number Diff line change
Expand Up @@ -736,12 +736,6 @@ def close_connections(self):
"""
self.db_connection.close_connections()

def mongo_wire_version(self):
"""
Returns the wire version for mongo. Only used to unit tests which instrument the connection.
"""
return self.db_connection.mongo_wire_version

def _drop_database(self, database=True, collections=True, connections=True):
"""
A destructive operation to drop the underlying database and close all connections.
Expand Down
10 changes: 4 additions & 6 deletions common/lib/xmodule/xmodule/modulestore/tests/factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -606,8 +606,6 @@ def mongo_uses_error_check(store):
"""
Does mongo use the error check as a separate message?
"""
if hasattr(store, 'mongo_wire_version'):
return store.mongo_wire_version() <= 1
if hasattr(store, 'modulestores'):
return any([mongo_uses_error_check(substore) for substore in store.modulestores])
return False
Expand All @@ -626,16 +624,16 @@ def check_mongo_calls_range(max_finds=float("inf"), min_finds=0, max_sends=None,
:param min_sends: If non-none, make sure number of send calls are >=min_sends
"""
with check_sum_of_calls(
pymongo.message,
['query', 'get_more'],
pymongo.collection.Collection,
['find'],
max_finds,
min_finds,
):
if max_sends is not None or min_sends is not None:
with check_sum_of_calls(
pymongo.message,
pymongo.collection.Collection,
# mongo < 2.6 uses insert, update, delete and _do_batched_insert. >= 2.6 _do_batched_write
['insert', 'update', 'delete', '_do_batched_write_command', '_do_batched_insert', ],
['insert', 'update', 'bulk_write', '_delete'],
max_sends if max_sends is not None else float("inf"),
min_sends if min_sends is not None else 0,
):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -151,11 +151,11 @@ def _import_course(self, content_store, modulestore):
(MIXED_OLD_MONGO_MODULESTORE_BUILDER, 0, True, False, 359),
# The line below shows the way this traversal *should* be done
# (if you'll eventually access all the fields and load all the definitions anyway).
(MIXED_SPLIT_MODULESTORE_BUILDER, None, False, True, 4),
(MIXED_SPLIT_MODULESTORE_BUILDER, None, False, True, 3),
(MIXED_SPLIT_MODULESTORE_BUILDER, None, True, True, 38),
(MIXED_SPLIT_MODULESTORE_BUILDER, 0, False, True, 38),
(MIXED_SPLIT_MODULESTORE_BUILDER, 0, True, True, 38),
(MIXED_SPLIT_MODULESTORE_BUILDER, None, False, False, 4),
(MIXED_SPLIT_MODULESTORE_BUILDER, None, False, False, 3),
(MIXED_SPLIT_MODULESTORE_BUILDER, None, True, False, 3),
(MIXED_SPLIT_MODULESTORE_BUILDER, 0, False, False, 3),
(MIXED_SPLIT_MODULESTORE_BUILDER, 0, True, False, 3)
Expand All @@ -177,7 +177,7 @@ def test_number_mongo_calls(self, store_builder, depth, lazy, access_all_block_f

@ddt.data(
(MIXED_OLD_MONGO_MODULESTORE_BUILDER, 176),
(MIXED_SPLIT_MODULESTORE_BUILDER, 5),
(MIXED_SPLIT_MODULESTORE_BUILDER, 4),
)
@ddt.unpack
def test_lazy_when_course_previously_cached(self, store_builder, num_mongo_calls):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
""" Test the behavior of split_mongo/MongoConnection """
import unittest
from mock import patch
from pymongo.errors import ConnectionFailure

from xmodule.exceptions import HeartbeatFailure
from xmodule.modulestore.split_mongo.mongo_connection import MongoConnection
from xmodule.exceptions import HeartbeatFailure

Expand All @@ -14,7 +17,7 @@ class TestHeartbeatFailureException(unittest.TestCase):
def test_heartbeat_raises_exception_when_connection_alive_is_false(self, *calls):
# pylint: disable=W0613
with patch('mongodb_proxy.MongoProxy') as mock_proxy:
mock_proxy.return_value.alive.return_value = False
mock_proxy.return_value.admin.command.side_effect = ConnectionFailure('Test')
useless_conn = MongoConnection('useless', 'useless', 'useless')

with self.assertRaises(HeartbeatFailure):
Expand Down
26 changes: 22 additions & 4 deletions common/lib/xmodule/xmodule/mongo_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,20 @@
import logging

import pymongo
from pymongo import ReadPreference
from mongodb_proxy import MongoProxy
from pymongo.read_preferences import (
ReadPreference,
read_pref_mode_from_name,
_MONGOS_MODES,
_MODES
)


logger = logging.getLogger(__name__) # pylint: disable=invalid-name

# This will yeld a map of all available Mongo modes and their name
MONGO_READ_PREFERENCE_MAP = dict(zip(_MONGOS_MODES, _MODES))


# pylint: disable=bad-continuation
def connect_to_mongodb(
Expand All @@ -34,10 +43,19 @@ def connect_to_mongodb(
# No 'replicaSet' in kwargs - so no secondary reads.
mongo_client_class = pymongo.MongoClient

# If read_preference is given as a name of a valid ReadPreference.<NAME> constant
# such as "SECONDARY_PREFERRED", convert it. Otherwise pass it through unchanged.
# If the MongoDB server uses a separate authentication database that should be specified here
auth_source = kwargs.pop('auth_source', '') or None

# If read_preference is given as a name of a valid ReadPreference.<NAME>
# constant such as "SECONDARY_PREFERRED" or a mongo mode such as
# "secondaryPreferred", convert it. Otherwise pass it through unchanged.
if 'read_preference' in kwargs:
read_preference = getattr(ReadPreference, kwargs['read_preference'], None)
read_preference = MONGO_READ_PREFERENCE_MAP.get(
kwargs['read_preference'],
kwargs['read_preference']
)

read_preference = getattr(ReadPreference, read_preference, None)
if read_preference is not None:
kwargs['read_preference'] = read_preference

Expand Down
4 changes: 2 additions & 2 deletions lms/djangoapps/dashboard/git_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,5 +342,5 @@ def add_repo(repo, rdir_in, branch=None):
)
cil.save()

log.debug('saved CourseImportLog for %s', cil.course_id)
mdb.disconnect()
log.debug(u'saved CourseImportLog for %s', cil.course_id)
mdb.close()
2 changes: 1 addition & 1 deletion lms/djangoapps/dashboard/sysadmin.py
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,7 @@ def get(self, request, *args, **kwargs):
page = min(max(1, given_page), paginator.num_pages)
logs = paginator.page(page)

mdb.disconnect()
mdb.close()
context = {
'logs': logs,
'course_id': text_type(course_id) if course_id else None,
Expand Down
Loading