diff --git a/cms/envs/common.py b/cms/envs/common.py index fffae4eaba84..be038e08223a 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -882,7 +882,11 @@ ################################# CELERY ###################################### # Auto discover tasks fails to detect contentstore tasks -CELERY_IMPORTS = ('cms.djangoapps.contentstore.tasks') +CELERY_IMPORTS = ( + 'cms.djangoapps.contentstore.tasks', + 'openedx.core.djangoapps.bookmarks.tasks', + 'openedx.core.djangoapps.ccxcon.tasks', +) # Message configuration diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index a51bbbf20caa..fcfe6aeadd98 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -51,7 +51,7 @@ def close_connections(self): """ Closes any open connections to the underlying databases """ - self.fs_files.database.connection.close() + self.fs_files.database.client.close() def _drop_database(self, database=True, collections=True, connections=True): """ @@ -65,10 +65,10 @@ def _drop_database(self, database=True, collections=True, connections=True): If connections is True, then close the connection to the database as well. """ - connection = self.fs_files.database.connection + connection = self.fs_files.database.client if database: - connection.drop_database(self.fs_files.database) + connection.drop_database(self.fs_files.database.name) elif collections: self.fs_files.drop() self.chunks.drop() @@ -297,15 +297,18 @@ def _get_all_content_for_course(self, } }) - items = self.fs_files.aggregate(pipeline_stages) - if items['result']: - result = items['result'][0] - count = result['count'] - assets = list(result['results']) - else: - # no results - count = 0 - assets = [] + cursor = self.fs_files.aggregate(pipeline_stages) + # Set values if result of query is empty + count = 0 + assets = [] + try: + result = cursor.next() + if result: + count = result['count'] + assets = list(result['results']) + except StopIteration: + # Skip if no assets were returned + pass # We're constructing the asset key immediately after retrieval from the database so that # callers are insulated from knowing how our identifiers are stored. diff --git a/common/lib/xmodule/xmodule/course_metadata_utils.py b/common/lib/xmodule/xmodule/course_metadata_utils.py index fee7a2d058f9..36ed4d05c02f 100644 --- a/common/lib/xmodule/xmodule/course_metadata_utils.py +++ b/common/lib/xmodule/xmodule/course_metadata_utils.py @@ -159,7 +159,7 @@ def sorting_dates(start, advertised_start, announcement): start = dateutil.parser.parse(advertised_start) if start.tzinfo is None: start = start.replace(tzinfo=utc) - except (ValueError, AttributeError): + except (TypeError, ValueError, AttributeError): start = start now = datetime.now(utc) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 48978c6d7f01..58e61cc85ed3 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -599,14 +599,7 @@ def close_connections(self): """ Closes any open connections to the underlying database """ - self.collection.database.connection.close() - - def mongo_wire_version(self): - """ - Returns the wire version for mongo. Only used to unit tests which instrument the connection. - """ - self.database.connection._ensure_connected() - return self.database.connection.max_wire_version + self.collection.database.client.close() def _drop_database(self, database=True, collections=True, connections=True): """ @@ -623,14 +616,14 @@ def _drop_database(self, database=True, collections=True, connections=True): # drop the assets super(MongoModuleStore, self)._drop_database(database, collections, connections) - connection = self.collection.database.connection + connection = self.collection.database.client if database: connection.drop_database(self.collection.database.proxied_object) elif collections: self.collection.drop() else: - self.collection.remove({}) + self.collection.delete_many({}) if connections: connection.close() @@ -1908,8 +1901,8 @@ def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id): dest_course_key (CourseKey): identifier of course to copy to """ source_assets = self._find_course_assets(source_course_key) - dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': unicode(dest_course_key)} - self.asset_collection.remove({'course_id': unicode(dest_course_key)}) + dest_assets = {'assets': source_assets.asset_md.copy(), 'course_id': six.text_type(dest_course_key)} + self.asset_collection.delete_many({'course_id': six.text_type(dest_course_key)}) # Update the document. self.asset_collection.insert(dest_assets) @@ -1981,7 +1974,7 @@ def delete_all_asset_metadata(self, course_key, user_id): # A single document exists per course to store the course asset metadata. try: course_assets = self._find_course_assets(course_key) - self.asset_collection.remove(course_assets.doc_id) + self.asset_collection.delete_many({'_id': course_assets.doc_id}) except ItemNotFoundError: # When deleting asset metadata, if a course's asset metadata is not present, no big deal. pass @@ -1990,9 +1983,11 @@ def heartbeat(self): """ Check that the db is reachable. """ - if self.database.connection.alive(): + try: + # The ismaster command is cheap and does not require auth. + self.database.client.admin.command('ismaster') return {ModuleStoreEnum.Type.mongo: True} - else: + except pymongo.errors.ConnectionFailure: raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo') def ensure_indexes(self): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py index 14a7b55f4498..126073087d3a 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py @@ -166,7 +166,7 @@ def delete_course(self, course_key, user_id): # delete all of the db records for the course course_query = self._course_key_to_son(course_key) - self.collection.remove(course_query, multi=True) + self.collection.delete_many(course_query) self.delete_all_asset_metadata(course_key, user_id) self._emit_course_deleted_signal(course_key) @@ -639,7 +639,7 @@ def _internal(tier): if len(to_be_deleted) > 0: bulk_record = self._get_bulk_ops_record(root_usages[0].course_key) bulk_record.dirty = True - self.collection.remove({'_id': {'$in': to_be_deleted}}, safe=self.collection.safe) + self.collection.delete_many({'_id': {'$in': to_be_deleted}}) @memoize_in_request_cache('request_cache') def has_changes(self, xblock): @@ -743,7 +743,7 @@ def _internal_depth_first(item_location, is_root): bulk_record = self._get_bulk_ops_record(course_key) if len(to_be_deleted) > 0: bulk_record.dirty = True - self.collection.remove({'_id': {'$in': to_be_deleted}}) + self.collection.delete_many({'_id': {'$in': to_be_deleted}}) self._flag_publish_event(course_key) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index 3aae20e61fcf..35812dce3c3a 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -308,9 +308,11 @@ def heartbeat(self): """ Check that the db is reachable. """ - if self.database.connection.alive(): + try: + # The ismaster command is cheap and does not require auth. + self.database.client.admin.command('ismaster') return True - else: + except pymongo.errors.ConnectionFailure: raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo') def get_structure(self, key, course_context=None): @@ -601,13 +603,7 @@ def close_connections(self): """ Closes any open connections to the underlying databases """ - self.database.connection.close() - - def mongo_wire_version(self): - """ - Returns the wire version for mongo. Only used to unit tests which instrument the connection. - """ - return self.database.connection.max_wire_version + self.database.client.close() def _drop_database(self, database=True, collections=True, connections=True): """ @@ -621,7 +617,7 @@ def _drop_database(self, database=True, collections=True, connections=True): If connections is True, then close the connection to the database as well. """ - connection = self.database.connection + connection = self.database.client if database: connection.drop_database(self.database.name) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 2acd3a52e3a3..8101badbcf45 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -736,12 +736,6 @@ def close_connections(self): """ self.db_connection.close_connections() - def mongo_wire_version(self): - """ - Returns the wire version for mongo. Only used to unit tests which instrument the connection. - """ - return self.db_connection.mongo_wire_version - def _drop_database(self, database=True, collections=True, connections=True): """ A destructive operation to drop the underlying database and close all connections. diff --git a/common/lib/xmodule/xmodule/modulestore/tests/factories.py b/common/lib/xmodule/xmodule/modulestore/tests/factories.py index 0a6ec26ba3ec..0ab7f85029aa 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/factories.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/factories.py @@ -606,8 +606,6 @@ def mongo_uses_error_check(store): """ Does mongo use the error check as a separate message? """ - if hasattr(store, 'mongo_wire_version'): - return store.mongo_wire_version() <= 1 if hasattr(store, 'modulestores'): return any([mongo_uses_error_check(substore) for substore in store.modulestores]) return False @@ -626,16 +624,16 @@ def check_mongo_calls_range(max_finds=float("inf"), min_finds=0, max_sends=None, :param min_sends: If non-none, make sure number of send calls are >=min_sends """ with check_sum_of_calls( - pymongo.message, - ['query', 'get_more'], + pymongo.collection.Collection, + ['find'], max_finds, min_finds, ): if max_sends is not None or min_sends is not None: with check_sum_of_calls( - pymongo.message, + pymongo.collection.Collection, # mongo < 2.6 uses insert, update, delete and _do_batched_insert. >= 2.6 _do_batched_write - ['insert', 'update', 'delete', '_do_batched_write_command', '_do_batched_insert', ], + ['insert', 'update', 'bulk_write', '_delete'], max_sends if max_sends is not None else float("inf"), min_sends if min_sends is not None else 0, ): diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py index 4210ce172829..6388ab7c20eb 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py @@ -151,11 +151,11 @@ def _import_course(self, content_store, modulestore): (MIXED_OLD_MONGO_MODULESTORE_BUILDER, 0, True, False, 359), # The line below shows the way this traversal *should* be done # (if you'll eventually access all the fields and load all the definitions anyway). - (MIXED_SPLIT_MODULESTORE_BUILDER, None, False, True, 4), + (MIXED_SPLIT_MODULESTORE_BUILDER, None, False, True, 3), (MIXED_SPLIT_MODULESTORE_BUILDER, None, True, True, 38), (MIXED_SPLIT_MODULESTORE_BUILDER, 0, False, True, 38), (MIXED_SPLIT_MODULESTORE_BUILDER, 0, True, True, 38), - (MIXED_SPLIT_MODULESTORE_BUILDER, None, False, False, 4), + (MIXED_SPLIT_MODULESTORE_BUILDER, None, False, False, 3), (MIXED_SPLIT_MODULESTORE_BUILDER, None, True, False, 3), (MIXED_SPLIT_MODULESTORE_BUILDER, 0, False, False, 3), (MIXED_SPLIT_MODULESTORE_BUILDER, 0, True, False, 3) @@ -177,7 +177,7 @@ def test_number_mongo_calls(self, store_builder, depth, lazy, access_all_block_f @ddt.data( (MIXED_OLD_MONGO_MODULESTORE_BUILDER, 176), - (MIXED_SPLIT_MODULESTORE_BUILDER, 5), + (MIXED_SPLIT_MODULESTORE_BUILDER, 4), ) @ddt.unpack def test_lazy_when_course_previously_cached(self, store_builder, num_mongo_calls): diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_mongo_connection.py index c5337129600f..43c0433efd8c 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_mongo_connection.py @@ -1,6 +1,9 @@ """ Test the behavior of split_mongo/MongoConnection """ import unittest from mock import patch +from pymongo.errors import ConnectionFailure + +from xmodule.exceptions import HeartbeatFailure from xmodule.modulestore.split_mongo.mongo_connection import MongoConnection from xmodule.exceptions import HeartbeatFailure @@ -14,7 +17,7 @@ class TestHeartbeatFailureException(unittest.TestCase): def test_heartbeat_raises_exception_when_connection_alive_is_false(self, *calls): # pylint: disable=W0613 with patch('mongodb_proxy.MongoProxy') as mock_proxy: - mock_proxy.return_value.alive.return_value = False + mock_proxy.return_value.admin.command.side_effect = ConnectionFailure('Test') useless_conn = MongoConnection('useless', 'useless', 'useless') with self.assertRaises(HeartbeatFailure): diff --git a/common/lib/xmodule/xmodule/mongo_utils.py b/common/lib/xmodule/xmodule/mongo_utils.py index ea6a37c9f68d..f3bc62121c82 100644 --- a/common/lib/xmodule/xmodule/mongo_utils.py +++ b/common/lib/xmodule/xmodule/mongo_utils.py @@ -4,11 +4,20 @@ import logging import pymongo -from pymongo import ReadPreference from mongodb_proxy import MongoProxy +from pymongo.read_preferences import ( + ReadPreference, + read_pref_mode_from_name, + _MONGOS_MODES, + _MODES +) + logger = logging.getLogger(__name__) # pylint: disable=invalid-name +# This will yeld a map of all available Mongo modes and their name +MONGO_READ_PREFERENCE_MAP = dict(zip(_MONGOS_MODES, _MODES)) + # pylint: disable=bad-continuation def connect_to_mongodb( @@ -34,10 +43,19 @@ def connect_to_mongodb( # No 'replicaSet' in kwargs - so no secondary reads. mongo_client_class = pymongo.MongoClient - # If read_preference is given as a name of a valid ReadPreference. constant - # such as "SECONDARY_PREFERRED", convert it. Otherwise pass it through unchanged. + # If the MongoDB server uses a separate authentication database that should be specified here + auth_source = kwargs.pop('auth_source', '') or None + + # If read_preference is given as a name of a valid ReadPreference. + # constant such as "SECONDARY_PREFERRED" or a mongo mode such as + # "secondaryPreferred", convert it. Otherwise pass it through unchanged. if 'read_preference' in kwargs: - read_preference = getattr(ReadPreference, kwargs['read_preference'], None) + read_preference = MONGO_READ_PREFERENCE_MAP.get( + kwargs['read_preference'], + kwargs['read_preference'] + ) + + read_preference = getattr(ReadPreference, read_preference, None) if read_preference is not None: kwargs['read_preference'] = read_preference @@ -58,9 +76,14 @@ def connect_to_mongodb( wait_time=retry_wait_time ) + # default the authSource to be whatever db we are connecting to (for backwards compatiblity) + authSource = db + if kwargs.get('authSource'): + authSource = kwargs.get('authSource') + # If credentials were provided, authenticate the user. if user is not None and password is not None: - mongo_conn.authenticate(user, password) + mongo_conn.authenticate(user, password, authSource) return mongo_conn diff --git a/lms/djangoapps/course_api/views.py b/lms/djangoapps/course_api/views.py index e1326d5aa55f..3ca878440db9 100644 --- a/lms/djangoapps/course_api/views.py +++ b/lms/djangoapps/course_api/views.py @@ -241,11 +241,6 @@ class CourseListView(DeveloperErrorViewMixin, ListAPIView): # - https://github.com/elastic/elasticsearch/commit/8b0a863d427b4ebcbcfb1dcd69c996c52e7ae05e results_size_infinity = 10000 - # Return all the results, 10K is the maximum allowed value for ElasticSearch. - # We should use 0 after upgrading to 1.1+: - # - https://github.com/elastic/elasticsearch/commit/8b0a863d427b4ebcbcfb1dcd69c996c52e7ae05e - results_size_infinity = 10000 - def get_queryset(self): """ Return a list of courses visible to the user. diff --git a/lms/djangoapps/dashboard/git_import.py b/lms/djangoapps/dashboard/git_import.py index 347cfdec5489..28dcfa0137b1 100644 --- a/lms/djangoapps/dashboard/git_import.py +++ b/lms/djangoapps/dashboard/git_import.py @@ -342,5 +342,5 @@ def add_repo(repo, rdir_in, branch=None): ) cil.save() - log.debug('saved CourseImportLog for %s', cil.course_id) - mdb.disconnect() + log.debug(u'saved CourseImportLog for %s', cil.course_id) + mdb.close() diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py index bf0d1bd7a13c..685d959e2e29 100644 --- a/lms/djangoapps/dashboard/sysadmin.py +++ b/lms/djangoapps/dashboard/sysadmin.py @@ -652,7 +652,7 @@ def get(self, request, *args, **kwargs): page = min(max(1, given_page), paginator.num_pages) logs = paginator.page(page) - mdb.disconnect() + mdb.close() context = { 'logs': logs, 'course_id': text_type(course_id) if course_id else None, diff --git a/lms/djangoapps/verify_student/management/commands/tests/test_send_verification_expiry_email.py b/lms/djangoapps/verify_student/management/commands/tests/test_send_verification_expiry_email.py new file mode 100644 index 000000000000..a43e91859fc1 --- /dev/null +++ b/lms/djangoapps/verify_student/management/commands/tests/test_send_verification_expiry_email.py @@ -0,0 +1,267 @@ +""" +Tests for django admin command `send_verification_expiry_email` in the verify_student module +""" + +from __future__ import absolute_import + +from datetime import timedelta + +import boto +from django.conf import settings +from django.contrib.sites.models import Site +from django.core import mail +from django.core.management import call_command, CommandError +from django.test.utils import override_settings +from django.utils.timezone import now +from mock import patch +from student.tests.factories import CourseEnrollmentFactory, UserFactory +from testfixtures import LogCapture +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + +from common.test.utils import MockS3Mixin, py2_only +from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification +from lms.djangoapps.verify_student.tests.test_models import FAKE_SETTINGS, mock_software_secure_post + +LOGGER_NAME = 'lms.djangoapps.verify_student.management.commands.send_verification_expiry_email' + + +@patch.dict(settings.VERIFY_STUDENT, FAKE_SETTINGS) +@patch('lms.djangoapps.verify_student.models.requests.post', new=mock_software_secure_post) +class TestSendVerificationExpiryEmail(MockS3Mixin, ModuleStoreTestCase): + """ Tests for django admin command `send_verification_expiry_email` in the verify_student module """ + + def setUp(self): + """ Initial set up for tests """ + super(TestSendVerificationExpiryEmail, self).setUp() + connection = boto.connect_s3() + connection.create_bucket(FAKE_SETTINGS['SOFTWARE_SECURE']['S3_BUCKET']) + Site.objects.create(domain='edx.org', name='edx.org') + self.resend_days = settings.VERIFICATION_EXPIRY_EMAIL['RESEND_DAYS'] + self.days = settings.VERIFICATION_EXPIRY_EMAIL['DAYS_RANGE'] + self.default_no_of_emails = settings.VERIFICATION_EXPIRY_EMAIL['DEFAULT_EMAILS'] + + def create_and_submit(self, user): + """ Helper method that lets us create new SoftwareSecurePhotoVerifications """ + attempt = SoftwareSecurePhotoVerification(user=user) + attempt.upload_face_image("Fake Data") + attempt.upload_photo_id_image("More Fake Data") + attempt.mark_ready() + attempt.submit() + return attempt + + def test_expiry_date_range(self): + """ + Test that the verifications are filtered on the given range. Email is not sent for any verification with + expiry date out of range + """ + user = UserFactory.create() + verification_in_range = self.create_and_submit(user) + verification_in_range.status = 'approved' + verification_in_range.expiry_date = now() - timedelta(days=self.days) + verification_in_range.save() + + user = UserFactory.create() + verification = self.create_and_submit(user) + verification.status = 'approved' + verification.expiry_date = now() - timedelta(days=self.days + 1) + verification.save() + + call_command('send_verification_expiry_email') + + # Check that only one email is sent + self.assertEqual(len(mail.outbox), 1) + + # Verify that the email is not sent to the out of range verification + expiry_email_date = SoftwareSecurePhotoVerification.objects.get(pk=verification.pk).expiry_email_date + self.assertIsNone(expiry_email_date) + + def test_expiry_email_date_range(self): + """ + Test that the verifications are filtered if the expiry_email_date has reached the time specified for + resending email + """ + user = UserFactory.create() + today = now().replace(hour=0, minute=0, second=0, microsecond=0) + verification_in_range = self.create_and_submit(user) + verification_in_range.status = 'approved' + verification_in_range.expiry_date = today - timedelta(days=self.days + 1) + verification_in_range.expiry_email_date = today - timedelta(days=self.resend_days) + verification_in_range.save() + + call_command('send_verification_expiry_email') + + # Check that email is sent even if the verification is not in expiry_date range but matches the criteria + # to resend email + self.assertEqual(len(mail.outbox), 1) + + def test_most_recent_verification(self): + """ + Test that the SoftwareSecurePhotoVerification object is not filtered if it is outdated. A verification is + outdated if it's expiry_date and expiry_email_date is set NULL + """ + # For outdated verification the expiry_date and expiry_email_date is set NULL verify_student/views.py:1164 + user = UserFactory.create() + outdated_verification = self.create_and_submit(user) + outdated_verification.status = 'approved' + outdated_verification.save() + + # Check that the expiry_email_date is not set for the outdated verification + expiry_email_date = SoftwareSecurePhotoVerification.objects.get(pk=outdated_verification.pk).expiry_email_date + self.assertIsNone(expiry_email_date) + + def test_send_verification_expiry_email(self): + """ + Test that checks for valid criteria the email is sent and expiry_email_date is set + """ + user = UserFactory.create() + verification = self.create_and_submit(user) + verification.status = 'approved' + verification.expiry_date = now() - timedelta(days=self.days) + verification.save() + + call_command('send_verification_expiry_email') + + expected_date = now() + attempt = SoftwareSecurePhotoVerification.objects.get(user_id=verification.user_id) + self.assertEquals(attempt.expiry_email_date.date(), expected_date.date()) + self.assertEqual(len(mail.outbox), 1) + + def test_email_already_sent(self): + """ + Test that if email is already sent as indicated by expiry_email_date then don't send again if it has been less + than resend_days + """ + user = UserFactory.create() + verification = self.create_and_submit(user) + verification.status = 'approved' + verification.expiry_date = now() - timedelta(days=self.days) + verification.expiry_email_date = now() + verification.save() + + call_command('send_verification_expiry_email') + + self.assertEqual(len(mail.outbox), 0) + + def test_no_verification_found(self): + """ + Test that if no approved and expired verifications are found the management command terminates gracefully + """ + start_date = now() - timedelta(days=self.days) # using default days + with LogCapture(LOGGER_NAME) as logger: + call_command('send_verification_expiry_email') + logger.check( + (LOGGER_NAME, + 'INFO', u"No approved expired entries found in SoftwareSecurePhotoVerification for the " + u"date range {} - {}".format(start_date.date(), now().date())) + ) + + def test_dry_run_flag(self): + """ + Test that the dry run flags sends no email and only logs the the number of email sent in each batch + """ + user = UserFactory.create() + verification = self.create_and_submit(user) + verification.status = 'approved' + verification.expiry_date = now() - timedelta(days=self.days) + verification.save() + + start_date = now() - timedelta(days=self.days) # using default days + count = 1 + + with LogCapture(LOGGER_NAME) as logger: + call_command('send_verification_expiry_email', '--dry-run') + logger.check( + (LOGGER_NAME, + 'INFO', + u"For the date range {} - {}, total Software Secure Photo verification filtered are {}" + .format(start_date.date(), now().date(), count) + ), + (LOGGER_NAME, + 'INFO', + u"This was a dry run, no email was sent. For the actual run email would have been sent " + u"to {} learner(s)".format(count) + )) + self.assertEqual(len(mail.outbox), 0) + + def test_not_enrolled_in_verified_course(self): + """ + Test that if the user is not enrolled in verified track, then after sending the default no of + emails, `expiry_email_date` is updated to None so that it's not filtered in the future for + sending emails + """ + user = UserFactory.create() + today = now().replace(hour=0, minute=0, second=0, microsecond=0) + verification = self.create_and_submit(user) + verification.status = 'approved' + verification.expiry_date = now() - timedelta(days=self.resend_days * (self.default_no_of_emails - 1)) + verification.expiry_email_date = today - timedelta(days=self.resend_days) + verification.save() + + call_command('send_verification_expiry_email') + + # check that after sending the default number of emails, the expiry_email_date is set to none for a + # user who is not enrolled in verified track + attempt = SoftwareSecurePhotoVerification.objects.get(pk=verification.id) + self.assertEqual(len(mail.outbox), 1) + self.assertIsNone(attempt.expiry_email_date) + + @py2_only + def test_user_enrolled_in_verified_course(self): + """ + Test that if the user is enrolled in verified track, then after sending the default no of + emails, `expiry_email_date` is updated to now() so that it's filtered in the future to send + email again. + + Does not work on python 3 with the latest mongo driver version due to class inheritance issues. + """ + user = UserFactory.create() + course = CourseFactory() + CourseEnrollmentFactory.create(user=user, course_id=course.id, mode='verified') + today = now().replace(hour=0, minute=0, second=0, microsecond=0) + verification = self.create_and_submit(user) + verification.status = 'approved' + verification.expiry_date = now() - timedelta(days=self.resend_days * (self.default_no_of_emails - 1)) + verification.expiry_email_date = today - timedelta(days=self.resend_days) + verification.save() + + call_command('send_verification_expiry_email') + + attempt = SoftwareSecurePhotoVerification.objects.get(pk=verification.id) + self.assertEqual(attempt.expiry_email_date, today) + + def test_number_of_emails_sent(self): + """ + Tests that the number of emails sent in case the user is only enrolled in audit track are same + as DEFAULT_EMAILS set in the settings + """ + user = UserFactory.create() + verification = self.create_and_submit(user) + verification.status = 'approved' + + verification.expiry_date = now() - timedelta(days=1) + verification.save() + call_command('send_verification_expiry_email') + + # running the loop one extra time to verify that after sending DEFAULT_EMAILS no extra emails are sent and + # for this reason expiry_email_date is set to None + for i in range(1, self.default_no_of_emails + 1): + if SoftwareSecurePhotoVerification.objects.get(pk=verification.id).expiry_email_date: + today = now().replace(hour=0, minute=0, second=0, microsecond=0) + verification.expiry_date = today - timedelta(days=self.resend_days * i + 1) + verification.expiry_email_date = today - timedelta(days=self.resend_days) + verification.save() + call_command('send_verification_expiry_email') + else: + break + + # expiry_email_date set to None means it no longer will be filtered hence no emails will be sent in future + self.assertIsNone(SoftwareSecurePhotoVerification.objects.get(pk=verification.id).expiry_email_date) + self.assertEqual(len(mail.outbox), self.default_no_of_emails) + + @override_settings(VERIFICATION_EXPIRY_EMAIL={'RESEND_DAYS': 15, 'DAYS_RANGE': 1, 'DEFAULT_EMAILS': 0}) + def test_command_error(self): + err_string = u"DEFAULT_EMAILS must be a positive integer. If you do not wish to send " \ + u"emails use --dry-run flag instead." + with self.assertRaisesRegexp(CommandError, err_string): + call_command('send_verification_expiry_email') diff --git a/lms/djangoapps/verify_student/tests/test_services.py b/lms/djangoapps/verify_student/tests/test_services.py index 43999288bf32..481f49992697 100644 --- a/lms/djangoapps/verify_student/tests/test_services.py +++ b/lms/djangoapps/verify_student/tests/test_services.py @@ -29,7 +29,7 @@ @patch.dict(settings.VERIFY_STUDENT, FAKE_SETTINGS) @ddt.ddt -class TestIDVerificationService(MockS3Mixin, ModuleStoreTestCase): +class TestIDVerificationService(ModuleStoreTestCase, MockS3Mixin): """ Tests for IDVerificationService. """ diff --git a/lms/lib/access_control_backends.py b/lms/lib/access_control_backends.py index 5bbfd4a90042..d494a0851f4b 100644 --- a/lms/lib/access_control_backends.py +++ b/lms/lib/access_control_backends.py @@ -18,14 +18,15 @@ class AccessControlBackends(object): Meant to be instantiated by this module, so use the `access_control_backends` object. """ SUPPORTED_ACTIONS = { + 'course.enroll', + 'course.instructor', 'course.load', + 'course.load_forum', 'course.load_mobile', - 'course.enroll', + 'course.see_about_page', 'course.see_exists', - 'course.staff', - 'course.instructor', 'course.see_in_catalog', - 'course.see_about_page', + 'course.staff', } UNSUPPORTED_ERROR_FMT = '`AccessControlBackends` does not support the action `{action}` yet'.format diff --git a/lms/urls.py b/lms/urls.py index c964eca9b01e..6b2743e042f9 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -1100,8 +1100,3 @@ include('openedx.core.djangoapps.appsembler.api.urls', namespace='tahoe-api')), ) - -if 'figures' in settings.INSTALLED_APPS: - urlpatterns += ( - url(r'^figures/', include('figures.urls', namespace='figures')), - ) diff --git a/openedx/core/djangoapps/ccxcon/tasks.py b/openedx/core/djangoapps/ccxcon/tasks.py index 3ff0391ce6d7..cee3d9430f14 100644 --- a/openedx/core/djangoapps/ccxcon/tasks.py +++ b/openedx/core/djangoapps/ccxcon/tasks.py @@ -12,7 +12,7 @@ log = get_task_logger(__name__) -@task() +@task(name='openedx.core.djangoapps.ccxcon.tasks.update_ccxcon') def update_ccxcon(course_id, cur_retry=0): """ Pass through function to update course information on CCXCon. diff --git a/openedx/core/djangoapps/content/course_overviews/models.py b/openedx/core/djangoapps/content/course_overviews/models.py index 7914f5960cac..99b74ee28864 100644 --- a/openedx/core/djangoapps/content/course_overviews/models.py +++ b/openedx/core/djangoapps/content/course_overviews/models.py @@ -7,6 +7,7 @@ from django.conf import settings from django.db import models, transaction +from django.db.models import Q from django.db.models.fields import BooleanField, DateTimeField, DecimalField, TextField, FloatField, IntegerField from django.db.utils import IntegrityError from django.template import defaultfilters @@ -576,7 +577,10 @@ def get_all_courses(cls, orgs=None, filter_=None): # In rare cases, courses belonging to the same org may be accidentally assigned # an org code with a different casing (e.g., Harvardx as opposed to HarvardX). # Case-insensitive matching allows us to deal with this kind of dirty data. - course_overviews = course_overviews.filter(org__iregex=r'(' + '|'.join(orgs) + ')') + org_filter = Q() # Avoiding the `reduce()` for more readability, so a no-op filter starter is needed. + for org in orgs: + org_filter |= Q(org__iexact=org) + course_overviews = course_overviews.filter(org_filter) if filter_: course_overviews = course_overviews.filter(**filter_) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 8e88184b0270..3d2e1d1b7fb7 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -124,7 +124,7 @@ edx-opaque-keys[django]==0.4.4 git+https://github.com/appsembler/edx-organizations.git@0.4.12-appsembler4 # edx-organizations==0.4.12 edx-proctoring==1.4.0 edx-rest-api-client==1.7.1 -edx-search==1.2.1 +-e git+https://github.com/appsembler/edx-search.git@appsembler-beta-release-2020-01-07_4#egg=edx-search # edx-search==1.2.1 edx-submissions==2.0.12 edx-user-state-client==1.0.4 edxval==0.1.16 @@ -192,10 +192,10 @@ pygments==2.2.0 pygraphviz==1.1 pyjwkest==1.3.2 pyjwt==1.5.2 -pymongo==2.9.1 -pynliner==0.5.2 -pyparsing==2.2.0 -pysrt==0.4.7 +pymongo==3.9.0 +pynliner==0.8.0 +pyparsing==2.2.0 # via pycontracts +pysrt==1.1.1 python-dateutil==2.4.0 python-levenshtein==0.12.0 python-memcached==1.48 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 049a981c46be..1a84b45094f3 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -145,7 +145,7 @@ edx-opaque-keys[django]==0.4.4 git+https://github.com/appsembler/edx-organizations.git@0.4.12-appsembler4 # edx-organizations==0.4.12 edx-proctoring==1.4.0 edx-rest-api-client==1.7.1 -edx-search==1.2.1 +-e git+https://github.com/appsembler/edx-search.git@appsembler-beta-release-2020-01-07_4#egg=edx-search # edx-search==1.2.1 edx-sphinx-theme==1.3.0 edx-submissions==2.0.12 edx-user-state-client==1.0.4 @@ -257,10 +257,9 @@ pyjwt==1.5.2 pylint-celery==0.3 pylint-django==0.7.2 pylint-plugin-utils==0.3 -pylint==1.7.1 -pymongo==2.9.1 -pynliner==0.5.2 -pyopenssl==18.0.0 +pylint==1.7.6 +pymongo==3.9.0 +pynliner==0.8.0 pyparsing==2.2.0 pyquery==1.4.0 pysqlite==2.8.3 diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in index e51f84e944d3..c07ed4909dc9 100644 --- a/requirements/edx/paver.in +++ b/requirements/edx/paver.in @@ -15,9 +15,9 @@ mock==1.0.1 # Stub out code with mock objects and make a path.py==8.2.1 # Easier manipulation of filesystem paths paver # Build, distribution and deployment scripting tool psutil==1.2.1 # Library for retrieving information on running processes and system utilization -pymongo==2.9.1 # via edx-opaque-keys -python-memcached==1.48 # Python interface to the memcached memory cache daemon -requests==2.9.1 # Simple interface for making HTTP requests -stevedore==1.10.0 # via edx-opaque-keys +pymongo==3.9.0 # via edx-opaque-keys +python-memcached # Python interface to the memcached memory cache daemon +requests # Simple interface for making HTTP requests +stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins watchdog # Used in paver watch_assets wrapt==1.10.5 # Decorator utilities used in the @timed paver task decorator diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index 9a6a6a4b3ee2..3697fde6602c 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -16,11 +16,12 @@ pathtools==0.1.2 # via watchdog paver==1.3.4 pbr==4.0.4 # via stevedore psutil==1.2.1 -pymongo==2.9.1 -python-memcached==1.48 -pyyaml==3.12 # via watchdog -requests==2.9.1 -six==1.11.0 # via edx-opaque-keys, libsass, paver, stevedore -stevedore==1.10.0 -watchdog==0.8.3 +pymongo==3.9.0 +python-memcached==1.59 +pyyaml==5.2 # via watchdog +requests==2.22.0 +six==1.13.0 # via edx-opaque-keys, libsass, mock, paver, python-memcached, stevedore +stevedore==1.31.0 +urllib3==1.25.7 # via requests +watchdog==0.9.0 wrapt==1.10.5 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 47d90de790a2..05a6037e105e 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -140,7 +140,7 @@ edx-opaque-keys[django]==0.4.4 git+https://github.com/appsembler/edx-organizations.git@0.4.12-appsembler4 # edx-organizations==0.4.12 edx-proctoring==1.4.0 edx-rest-api-client==1.7.1 -edx-search==1.2.1 +-e git+https://github.com/appsembler/edx-search.git@appsembler-beta-release-2020-01-07_4#egg=edx-search # edx-search==1.2.1 edx-submissions==2.0.12 edx-user-state-client==1.0.4 edxval==0.1.16 @@ -246,10 +246,9 @@ pyjwt==1.5.2 pylint-celery==0.3 # via edx-lint pylint-django==0.7.2 # via edx-lint pylint-plugin-utils==0.3 # via pylint-celery, pylint-django -pylint==1.7.1 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils -pymongo==2.9.1 -pynliner==0.5.2 -pyopenssl==18.0.0 # via scrapy, service-identity +pylint==1.7.6 # via edx-lint, pylint-celery, pylint-django +pymongo==3.9.0 +pynliner==0.8.0 pyparsing==2.2.0 pyquery==1.4.0 pysqlite==2.8.3 diff --git a/tox.ini b/tox.ini index 9ae04c75171e..d82678c70f02 100644 --- a/tox.ini +++ b/tox.ini @@ -68,7 +68,8 @@ commands = bash scripts/upgrade_pysqlite.sh {env:TRAVIS_FIXES} pytest \ - common/djangoapps/util/tests/test_milestones_helpers.py + common/djangoapps/util/tests/test_milestones_helpers.py \ + common/lib/xmodule/xmodule/modulestore/tests/test_split_mongo_mongo_connection.py [testenv:py27-studio] commands = @@ -122,6 +123,7 @@ commands = lms/djangoapps/courseware/tests/test_access_control_backends_integration.py \ lms/djangoapps/grades/tests/integration/test_events.py \ lms/djangoapps/instructor/tests/test_certificates.py::CertificatesInstructorApiTest \ + lms/djangoapps/verify_student/tests/test_services.py \ lms/lib/tests/test_access_control_backends.py \ openedx/core/djangoapps/appsembler \ openedx/core/djangoapps/site_configuration/tests/test_tahoe_changes.py \