diff --git a/common/lib/mandrill_client/client.py b/common/lib/mandrill_client/client.py index ef9ff8cc720f..725b1b37de17 100644 --- a/common/lib/mandrill_client/client.py +++ b/common/lib/mandrill_client/client.py @@ -7,6 +7,7 @@ class MandrillClient(object): + ACUMEN_DATA_TEMPLATE = 'acumen-data' PASSWORD_RESET_TEMPLATE = 'template-60' USER_ACCOUNT_ACTIVATION_TEMPLATE = 'template-61' ORG_ADMIN_ACTIVATION_TEMPLATE = 'org-admin-identified' @@ -24,7 +25,7 @@ class MandrillClient(object): def __init__(self): self.mandrill_client = mandrill.Mandrill(settings.MANDRILL_API_KEY) - def send_mail(self, template_name, user_email, context): + def send_mail(self, template_name, user_email, context, attachments=[]): """ calls the mandrill API for the specific template and email @@ -42,7 +43,8 @@ def send_mail(self, template_name, user_email, context): message={ 'from_email': settings.NOTIFICATION_FROM_EMAIL, 'to': [{'email': user_email}], - 'global_merge_vars': global_merge_vars + 'global_merge_vars': global_merge_vars, + 'attachments': attachments, }, ) log.info(result) diff --git a/lms/djangoapps/courseware/context_processor.py b/lms/djangoapps/courseware/context_processor.py index 07f98191a094..8124ffc91e2a 100644 --- a/lms/djangoapps/courseware/context_processor.py +++ b/lms/djangoapps/courseware/context_processor.py @@ -44,3 +44,4 @@ def user_timezone_locale_prefs(request): cached_value.update(user_prefs) return cached_value + diff --git a/lms/djangoapps/onboarding/helpers.py b/lms/djangoapps/onboarding/helpers.py index d76148ac37c6..649ad2d978f4 100644 --- a/lms/djangoapps/onboarding/helpers.py +++ b/lms/djangoapps/onboarding/helpers.py @@ -53,7 +53,6 @@ 'CN': 'China', 'CO': 'Colombia', 'CR': 'Costa Rica', - 'CU': 'Cuba', 'CUW': 'Curacao', 'CV': 'Cape Verde', 'CX': 'Christmas Island', @@ -110,7 +109,6 @@ 'IN': 'India', 'IO': 'British Indian Ocean Territory', 'IQ': 'Iraq', - 'IR': 'Iran', 'IS': 'Iceland', 'IT': 'Italy', 'JE': 'Jersey', @@ -123,7 +121,6 @@ 'KI': 'Kiribati', 'KM': 'Comoros', 'KN': 'Saint Kitts and Nevis', - 'KP': "North Korea", 'KR': 'South Korea', 'KW': 'Kuwait', 'KY': 'Cayman Islands', diff --git a/lms/djangoapps/philu_overrides/constants.py b/lms/djangoapps/philu_overrides/constants.py new file mode 100644 index 000000000000..38836c45fe26 --- /dev/null +++ b/lms/djangoapps/philu_overrides/constants.py @@ -0,0 +1,2 @@ +ACTIVATION_ERROR = 'Your account has not been activated. Please check your email to activate your account.' +ACTIVATION_ALERT_TYPE = 'activation' diff --git a/lms/djangoapps/philu_overrides/context_processor.py b/lms/djangoapps/philu_overrides/context_processor.py new file mode 100644 index 000000000000..6b3f183492fe --- /dev/null +++ b/lms/djangoapps/philu_overrides/context_processor.py @@ -0,0 +1,19 @@ +from lms.djangoapps.philu_overrides.constants import ACTIVATION_ERROR, ACTIVATION_ALERT_TYPE + + +def get_global_alert_messages(request): + + """ + function to get application wide messages + :param request: + :return: returns list of global messages" + """ + + alert_messages = [] + if not request.is_ajax(): + if request.user.is_authenticated() and not request.user.is_active and '/activate/' not in request.path: + alert_messages.append({ + "type": ACTIVATION_ALERT_TYPE, + "alert": ACTIVATION_ERROR + }) + return {"alert_messages": alert_messages} diff --git a/lms/djangoapps/philu_overrides/courseware/views/views.py b/lms/djangoapps/philu_overrides/courseware/views/views.py index f41286ee3c1d..e65fda5d2d11 100644 --- a/lms/djangoapps/philu_overrides/courseware/views/views.py +++ b/lms/djangoapps/philu_overrides/courseware/views/views.py @@ -70,7 +70,7 @@ def generate_user_cert(request, course_id): # mark the certificate with "error" status, so it can be re-run # with a management command. From the user's perspective, # it will appear that the certificate task was submitted successfully. - base_url = settings.LMS_ROOT_URL[:-3] + base_url = settings.LMS_ROOT_URL MandrillClient().send_mail( MandrillClient.COURSE_COMPLETION_TEMPLATE, student.email, diff --git a/lms/envs/common.py b/lms/envs/common.py index 08356ef78369..214175f16523 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -563,6 +563,9 @@ # Timezone processor (sends language and time_zone preference) 'courseware.context_processor.user_timezone_locale_prefs', + # Alert message processor + 'philu_overrides.context_processor.get_global_alert_messages', + # Allows the open edX footer to be leveraged in Django Templates. 'edxmako.shortcuts.footer_context_processor', @@ -2229,6 +2232,9 @@ # student_dashboard App 'lms.djangoapps.student_dashboard', + + # Data extraction App + 'openedx.features.data_extract', ) ######################### CSRF ######################################### @@ -2692,6 +2698,7 @@ 'enterprise', # Required by the Enterprise App 'django_object_actions', # https://github.com/crccheck/django-object-actions + ) for app_name in OPTIONAL_APPS: diff --git a/openedx/core/djangoapps/timed_notification/core.py b/openedx/core/djangoapps/timed_notification/core.py index f5472366ba1d..6c554966b1ea 100644 --- a/openedx/core/djangoapps/timed_notification/core.py +++ b/openedx/core/djangoapps/timed_notification/core.py @@ -93,5 +93,5 @@ def send_course_notification_email(course, template_name, context, to_list=None) def get_course_link(course_id): course_link = reverse("about_course", args=[course_id]) - base_url = settings.LMS_ROOT_URL[:-3] + base_url = settings.LMS_ROOT_URL return base_url + course_link diff --git a/openedx/features/__init__.py b/openedx/features/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/__init__.py b/openedx/features/data_extract/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/admin.py b/openedx/features/data_extract/admin.py new file mode 100644 index 000000000000..d716c34546e3 --- /dev/null +++ b/openedx/features/data_extract/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from openedx.features.data_extract.models import CourseDataExtraction + +admin.site.register(CourseDataExtraction) + diff --git a/openedx/features/data_extract/app.py b/openedx/features/data_extract/app.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/helpers.py b/openedx/features/data_extract/helpers.py new file mode 100644 index 000000000000..c8e24f9e57df --- /dev/null +++ b/openedx/features/data_extract/helpers.py @@ -0,0 +1,201 @@ +import requests +import json + +from django.conf import settings + +from certificates.models import GeneratedCertificate +from courseware.models import StudentModule +from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGrade +from lms.djangoapps.onboarding.models import Organization +from lms.djangoapps.teams.models import CourseTeamMembership, CourseTeam +from student.models import AnonymousUserId +from lms.djangoapps.teams.models import CourseTeamMembership, CourseTeam +from openassessment.fileupload import api as ora_file_upload_api +from openedx.core.djangoapps.content.course_structures.models import CourseStructure +from submissions.models import StudentItem, Submission, Score + + +def get_file_url(answer): + if answer.get('file_key'): + return ora_file_upload_api.get_download_url(answer['file_key']) + return None + + +def get_course_structure(course_key): + """ + Returns data about the course structure to course_data dict + + Arguments: + course_key (CourseKey): CourseKey object for specified course + """ + course_structure = CourseStructure.objects.get(course_id=course_key) + + return { + 'created': course_structure.created.__str__(), + 'modified': course_structure.modified.__str__(), + 'course_id': course_structure.course_id.to_deprecated_string(), + 'structure_json': course_structure.structure_json, + 'discussion_id_map_json': course_structure.discussion_id_map_json, + } + + +def get_teams_data(course_key): + """ + Returns data about all the teams in a course + + Arguments: + course_key (CourseKey): CourseKey object for specified course + """ + course_teams = CourseTeam.objects.filter(course_id=course_key) + team_data = [] + for team in course_teams: + team_data.push({ + 'team_id': course_team.team_id, + 'name': course_team.name, + 'course_id': course_team.course_id.to_deprecated_string(), + 'topic_id': course_team.topic_id, + 'date_created': course_team.date_created.__str__(), + 'description': course_team.description, + 'country': course_team.country, + 'language': course_team.language, + 'last_activity_at': course_team.last_activity_at.__str__(), + 'team_size': course_team.team_size, + }) + return team_data + + +def get_user_demographic_data(profile): + """ + Returns the demographic data for a single user + + Arguments: + profile (UserProfile): UserProfile object for the learner + """ + # get the user community profile data from NodeBB API + data_endpoint = settings.NODEBB_ENDPOINT + '/api/v2/users/data' + headers = {'Authorization': 'Bearer ' + settings.NODEBB_MASTER_TOKEN} + response = requests.post(data_endpoint, + data={'_uid': 1, 'username': profile.user.username}, + headers=headers) + + user_community_data = json.loads(response._content)['payload'] + + return { + 'student_id': profile.user.id, + 'email': profile.user.email, + 'first_name': profile.user.first_name, + 'last_name': profile.user.last_name, + 'date_joined': profile.user.date_joined.__str__(), + 'bio': profile.bio, + 'city': profile.city, + 'country': profile.country.__str__(), + 'language': profile.language, + 'english_proficiency': profile.user.extended_profile.english_proficiency, + 'organization_label': profile.user.extended_profile.organization.label if + profile.user.extended_profile.organization else '', + 'reputation': user_community_data['reputation'], + 'postcount': user_community_data['postcount'], + } + + +def get_user_progress_data(course_key, profile, anonymous_user_id): + """ + Returns all the data regarding the progress the user has made in a course + + Arguments: + course_key (CourseKey): CourseKey object for specified course + profile (UserProfile): UserProfile object for the learner + """ + + return { + 'team_memberships': [{ + 'team_id': membership.team_id, + 'date_joined': membership.date_joined.__str__(), + 'last_activity_at': membership.last_activity_at.__str__(), + } for membership in CourseTeamMembership.objects.filter(user_id=profile.user.id)], + + 'student_modules': [{ + 'module_type': module.module_type, + 'module_id': module.module_state_key.to_deprecated_string(), + 'course_id': module.course_id.to_deprecated_string(), + 'state': module.state, + 'grade': module.grade, + 'max_grade': module.max_grade, + 'done': module.done, + 'created': module.created.__str__(), + 'modified': module.modified.__str__(), + } for module in StudentModule.objects.filter(student_id=profile.user.id)], + + 'persistent_course_grades': [{ + 'created': course_grade.created.__str__(), + 'modified': course_grade.modified.__str__(), + 'course_id': course_grade.course_id.to_deprecated_string(), + 'course_edited_timestamp': course_grade.course_edited_timestamp.__str__(), + 'course_version': course_grade.course_version, + 'grading_policy_hash': course_grade.grading_policy_hash, + 'percent_grade': course_grade.percent_grade, + 'letter_grade': course_grade.letter_grade, + 'passed_timestamp': course_grade.passed_timestamp.__str__(), + } for course_grade in PersistentCourseGrade.objects.filter(user_id=profile.user.id)], + + 'persistent_subsection_grades': [{ + 'created': subsection_grade.created.__str__(), + 'modified': subsection_grade.modified.__str__(), + 'course_id': subsection_grade.course_id.to_deprecated_string(), + 'usage_key': subsection_grade.full_usage_key.to_deprecated_string(), + 'subtree_edited_timestamp': subsection_grade.subtree_edited_timestamp.__str__(), + 'course_version': subsection_grade.course_version, + 'earned_all': subsection_grade.earned_all, + 'possible_all': subsection_grade.possible_all, + 'earned_graded': subsection_grade.earned_graded, + 'possible_graded': subsection_grade.possible_graded, + 'visible_blocks': subsection_grade.visible_blocks.blocks_json, + 'first_attempted': subsection_grade.first_attempted.__str__(), + } for subsection_grade in PersistentSubsectionGrade.objects.filter(user_id=profile.user.id)], + + 'generated_certificates': [{ + 'course_id': certificate.course_id.to_deprecated_string(), + 'verify_uuid': certificate.verify_uuid, + 'download_uuid': certificate.download_uuid, + 'download_url': certificate.download_url, + 'grade': certificate.grade, + 'key': certificate.key, + 'distinction': certificate.distinction, + 'status': certificate.status, + 'mode': certificate.mode, + 'name': certificate.name, + 'created_date': certificate.created_date.__str__(), + 'modified_date': certificate.modified_date.__str__(), + 'error_reason': certificate.error_reason, + } for certificate in GeneratedCertificate.objects.filter(user_id=profile.user.id)], + + 'course_submission_data': { + 'student_items': [{ + 'course_id': item.course_id, + 'item_id': item.item_id, + 'item_type': item.item_type, + } for item in StudentItem.objects.filter(student_id=anonymous_user_id)], + + 'submissions': [{ + 'uuid': submission.uuid, + 'attempt_number': submission.attempt_number, + 'submitted_at': submission.submitted_at.__str__(), + 'created_at': submission.created_at.__str__(), + 'answer': submission.answer, + 'answer_file_url': get_file_url(submission.answer), + 'student_item_id': submission.student_item_id, + 'status': submission.status, + } for submission in Submission.objects.filter( + student_item__student_id=anonymous_user_id + )], + + 'student_scores': [{ + 'points_earned': score.points_earned, + 'points_possible': score.points_possible, + 'created_at': score.created_at.__str__(), + 'reset': score.reset, + 'student_item_id': score.student_item_id, + 'submission_id': score.submission_id, + } for score in Score.objects.filter(student_item__student_id=anonymous_user_id)], + } + } diff --git a/openedx/features/data_extract/management/__init__.py b/openedx/features/data_extract/management/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/management/commands/__init__.py b/openedx/features/data_extract/management/commands/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/management/commands/get_course_analytics.py b/openedx/features/data_extract/management/commands/get_course_analytics.py new file mode 100644 index 000000000000..56f60c22ba36 --- /dev/null +++ b/openedx/features/data_extract/management/commands/get_course_analytics.py @@ -0,0 +1,65 @@ +import base64 +import json +import tempfile + +from django.conf import settings +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand, CommandError + +from common.lib.mandrill_client.client import MandrillClient +from lms.djangoapps.mailing.management.commands.mailchimp_sync_course import get_enrolled_students +from opaque_keys.edx.keys import CourseKey +from openedx.features.data_extract.models import CourseDataExtraction + +from openedx.features.data_extract.helpers import ( + get_course_structure, + get_teams_data, + get_user_demographic_data, + get_user_progress_data, +) +from student.models import AnonymousUserId + + +class Command(BaseCommand): + help = 'Generates the analytics data for each course_id in coursedataextraction table' + + def handle(self, **options): + target_courses = CourseDataExtraction.objects.all() + + for target_course in target_courses: + emails = map(unicode.strip, target_course.emails.split(',')) + course_key = CourseKey.from_string(target_course.course_id) + + course_data = { + 'course_structure': get_course_structure(course_key), + 'team_data': get_teams_data(course_key), + 'user_data': [] + } + + user_profiles = get_enrolled_students(target_course.course_id) + anon_user_ids = dict(list(map(lambda x: (x.user.username, x.anonymous_user_id,), + AnonymousUserId.objects.filter(course_id=course_key)))) + + for profile in user_profiles: + demographic_data = get_user_demographic_data(profile) + progress_data = get_user_progress_data(course_key, profile, anon_user_ids[profile.user.username]) + + course_data['user_data'].append({ + 'demographic_data': demographic_data, + 'progress_data': progress_data, + }) + + with tempfile.TemporaryFile() as tmp: + for email in emails: + MandrillClient().send_mail( + template_name=MandrillClient.ACUMEN_DATA_TEMPLATE, + user_email=email, + context={}, + attachments=[ + { + "type": "text/plain", + "name": "data.json", + "content": base64.encodestring(json.dumps(course_data)) + } + ] + ) diff --git a/openedx/features/data_extract/migrations/0001_initial.py b/openedx/features/data_extract/migrations/0001_initial.py new file mode 100644 index 000000000000..cd6552cc9a25 --- /dev/null +++ b/openedx/features/data_extract/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='CourseDataExtraction', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('course_id', models.CharField(max_length=255)), + ('emails', models.TextField()), + ], + ), + ] diff --git a/openedx/features/data_extract/migrations/__init__.py b/openedx/features/data_extract/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/models.py b/openedx/features/data_extract/models.py new file mode 100644 index 000000000000..b2908eed57a3 --- /dev/null +++ b/openedx/features/data_extract/models.py @@ -0,0 +1,14 @@ +from django.db import models + + +class CourseDataExtraction(models.Model): + """ + These courses are marked for data extraction. These will be used to determine data of which courses is to be extracted + + @course_id: (string) id of the course. can be gotten through to_deprecated_string() method of CourseKey object + """ + course_id = models.CharField(max_length=255) + emails = models.TextField() + + def __unicode__(self): + return '{}'.format(self.course_id) diff --git a/openedx/features/data_extract/urls.py b/openedx/features/data_extract/urls.py new file mode 100644 index 000000000000..42397ff70986 --- /dev/null +++ b/openedx/features/data_extract/urls.py @@ -0,0 +1,4 @@ +from django.conf.urls import url + + +urlpatterns = []