Sprint 30

common/lib/mandrill_client/client.py

@@ -7,6 +7,7 @@
 
 
 class MandrillClient(object):
+    ACUMEN_DATA_TEMPLATE = 'acumen-data'
     PASSWORD_RESET_TEMPLATE = 'template-60'
     USER_ACCOUNT_ACTIVATION_TEMPLATE = 'template-61'
     ORG_ADMIN_ACTIVATION_TEMPLATE = 'org-admin-identified'
@@ -24,7 +25,7 @@ class MandrillClient(object):
     def __init__(self):
         self.mandrill_client = mandrill.Mandrill(settings.MANDRILL_API_KEY)
 
-    def send_mail(self, template_name, user_email, context):
+    def send_mail(self, template_name, user_email, context, attachments=[]):
         """
         calls the mandrill API for the specific template and email
 
@@ -42,7 +43,8 @@ def send_mail(self, template_name, user_email, context):
                 message={
                     'from_email': settings.NOTIFICATION_FROM_EMAIL,
                     'to': [{'email': user_email}],
-                    'global_merge_vars': global_merge_vars
+                    'global_merge_vars': global_merge_vars,
+                    'attachments': attachments,
                 },
             )
             log.info(result)

lms/djangoapps/courseware/context_processor.py

@@ -44,3 +44,4 @@ def user_timezone_locale_prefs(request):
 
         cached_value.update(user_prefs)
     return cached_value
+

lms/djangoapps/onboarding/helpers.py

@@ -53,7 +53,6 @@
     'CN': 'China',
     'CO': 'Colombia',
     'CR': 'Costa Rica',
-    'CU': 'Cuba',
     'CUW': 'Curacao',
     'CV': 'Cape Verde',
     'CX': 'Christmas Island',
@@ -110,7 +109,6 @@
     'IN': 'India',
     'IO': 'British Indian Ocean Territory',
     'IQ': 'Iraq',
-    'IR': 'Iran',
     'IS': 'Iceland',
     'IT': 'Italy',
     'JE': 'Jersey',
@@ -123,7 +121,6 @@
     'KI': 'Kiribati',
     'KM': 'Comoros',
     'KN': 'Saint Kitts and Nevis',
-    'KP': "North Korea",
     'KR': 'South Korea',
     'KW': 'Kuwait',
     'KY': 'Cayman Islands',

lms/djangoapps/philu_overrides/constants.py

@@ -0,0 +1,2 @@
+ACTIVATION_ERROR = 'Your account has not been activated. Please check your email to activate your account.'
+ACTIVATION_ALERT_TYPE = 'activation'

lms/djangoapps/philu_overrides/context_processor.py

@@ -0,0 +1,19 @@
+from lms.djangoapps.philu_overrides.constants import ACTIVATION_ERROR, ACTIVATION_ALERT_TYPE
+
+
+def get_global_alert_messages(request):
+
+    """
+    function to get application wide messages
+    :param request:
+    :return: returns list of global messages"
+    """
+
+    alert_messages = []
+    if not request.is_ajax():
+        if request.user.is_authenticated() and not request.user.is_active and '/activate/' not in request.path:
+            alert_messages.append({
+                "type": ACTIVATION_ALERT_TYPE,
+                "alert": ACTIVATION_ERROR
+            })
+    return {"alert_messages": alert_messages}

lms/envs/common.py

@@ -563,6 +563,9 @@
                 # Timezone processor (sends language and time_zone preference)
                 'courseware.context_processor.user_timezone_locale_prefs',
 
+                # Alert message processor
+                'philu_overrides.context_processor.get_global_alert_messages',
+
                 # Allows the open edX footer to be leveraged in Django Templates.
                 'edxmako.shortcuts.footer_context_processor',
 
@@ -2229,6 +2232,9 @@
 
     # student_dashboard App
     'lms.djangoapps.student_dashboard',
+
+    # Data extraction App
+    'openedx.features.data_extract',
 )
 
 ######################### CSRF #########################################
@@ -2692,6 +2698,7 @@
     'enterprise',
     # Required by the Enterprise App
     'django_object_actions',  # https://github.com/crccheck/django-object-actions
+
 )
 
 for app_name in OPTIONAL_APPS:

openedx/features/data_extract/admin.py

@@ -0,0 +1,5 @@
+from django.contrib import admin
+from openedx.features.data_extract.models import CourseDataExtraction
+
+admin.site.register(CourseDataExtraction)
+

openedx/features/data_extract/helpers.py

@@ -0,0 +1,201 @@
+import requests
+import json
+
+from django.conf import settings
+
+from certificates.models import GeneratedCertificate
+from courseware.models import StudentModule
+from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGrade
+from lms.djangoapps.onboarding.models import Organization
+from lms.djangoapps.teams.models import CourseTeamMembership, CourseTeam
+from student.models import AnonymousUserId
+from lms.djangoapps.teams.models import CourseTeamMembership, CourseTeam
+from openassessment.fileupload import api as ora_file_upload_api
+from openedx.core.djangoapps.content.course_structures.models import CourseStructure
+from submissions.models import StudentItem, Submission, Score
+
+
+def get_file_url(answer):
+    if answer.get('file_key'):
+        return ora_file_upload_api.get_download_url(answer['file_key'])
+    return None
+
+
+def get_course_structure(course_key):
+    """
+    Returns data about the course structure to course_data dict
+
+    Arguments:
+    course_key (CourseKey): CourseKey object for specified course
+    """
+    course_structure = CourseStructure.objects.get(course_id=course_key)
+
+    return {
+        'created': course_structure.created.__str__(),
+        'modified': course_structure.modified.__str__(),
+        'course_id': course_structure.course_id.to_deprecated_string(),
+        'structure_json': course_structure.structure_json,
+        'discussion_id_map_json': course_structure.discussion_id_map_json,
+    }
+
+
+def get_teams_data(course_key):
+    """
+    Returns data about all the teams in a course
+
+    Arguments:
+    course_key (CourseKey): CourseKey object for specified course
+    """
+    course_teams = CourseTeam.objects.filter(course_id=course_key)
+    team_data = []
+    for team in course_teams:
+        team_data.push({
+            'team_id': course_team.team_id,
+            'name': course_team.name,
+            'course_id': course_team.course_id.to_deprecated_string(),
+            'topic_id': course_team.topic_id,
+            'date_created': course_team.date_created.__str__(),
+            'description': course_team.description,
+            'country': course_team.country,
+            'language': course_team.language,
+            'last_activity_at': course_team.last_activity_at.__str__(),
+            'team_size': course_team.team_size,
+        })
+    return team_data
+
+
+def get_user_demographic_data(profile):
+    """
+    Returns the demographic data for a single user
+
+    Arguments:
+    profile (UserProfile): UserProfile object for the learner
+    """
+    # get the user community profile data from NodeBB API
+    data_endpoint = settings.NODEBB_ENDPOINT + '/api/v2/users/data'
+    headers = {'Authorization': 'Bearer ' + settings.NODEBB_MASTER_TOKEN}
+    response = requests.post(data_endpoint,
+                             data={'_uid': 1, 'username': profile.user.username},
+                             headers=headers)
+
+    user_community_data = json.loads(response._content)['payload']
+
+    return {
+        'student_id': profile.user.id,
+        'email': profile.user.email,
+        'first_name': profile.user.first_name,
+        'last_name': profile.user.last_name,
+        'date_joined': profile.user.date_joined.__str__(),
+        'bio': profile.bio,
+        'city': profile.city,
+        'country': profile.country.__str__(),
+        'language': profile.language,
+        'english_proficiency': profile.user.extended_profile.english_proficiency,
+        'label': profile.user.extended_profile.organization.label if
+        profile.user.extended_profile.organization else '',
+        'reputation': user_community_data['reputation'],
+        'postcount': user_community_data['postcount'],
+    }
+
+
+def get_user_progress_data(course_key, profile, anonymous_user_id):
+    """
+    Returns all the data regarding the progress the user has made in a course
+
+    Arguments:
+    course_key (CourseKey): CourseKey object for specified course
+    profile (UserProfile): UserProfile object for the learner
+    """
+
+    return {
+        'team_memberships': [{
+            'team_id': membership.team_id,
+            'date_joined': membership.date_joined.__str__(),
+            'last_activity_at': membership.last_activity_at.__str__(),
+        } for membership in CourseTeamMembership.objects.filter(user_id=profile.user.id)],
+
+        'student_modules': [{
+            'module_type': module.module_type,
+            'module_id': module.module_state_key.to_deprecated_string(),
+            'course_id': module.course_id.to_deprecated_string(),
+            'state': module.state,
+            'grade': module.grade,
+            'max_grade': module.max_grade,
+            'done': module.done,
+            'created': module.created.__str__(),
+            'modified': module.modified.__str__(),
+        } for module in StudentModule.objects.filter(student_id=profile.user.id)],
+
+        'persistent_course_grades': [{
+            'created': course_grade.created.__str__(),
+            'modified': course_grade.modified.__str__(),
+            'course_id': course_grade.course_id.to_deprecated_string(),
+            'course_edited_timestamp': course_grade.course_edited_timestamp.__str__(),
+            'course_version': course_grade.course_version,
+            'grading_policy_hash': course_grade.grading_policy_hash,
+            'percent_grade': course_grade.percent_grade,
+            'letter_grade': course_grade.letter_grade,
+            'passed_timestamp': course_grade.passed_timestamp.__str__(),
+        } for course_grade in PersistentCourseGrade.objects.filter(user_id=profile.user.id)],
+
+        'persistent_subsection_grades': [{
+            'created': subsection_grade.created.__str__(),
+            'modified': subsection_grade.modified.__str__(),
+            'course_id': subsection_grade.course_id.to_deprecated_string(),
+            'usage_key': subsection_grade.full_usage_key.to_deprecated_string(),
+            'subtree_edited_timestamp': subsection_grade.subtree_edited_timestamp.__str__(),
+            'course_version': subsection_grade.course_version,
+            'earned_all': subsection_grade.earned_all,
+            'possible_all': subsection_grade.possible_all,
+            'earned_graded': subsection_grade.earned_graded,
+            'possible_graded': subsection_grade.possible_graded,
+            'visible_blocks': subsection_grade.visible_blocks.blocks_json,
+            'first_attempted': subsection_grade.first_attempted.__str__(),
+        } for subsection_grade in PersistentSubsectionGrade.objects.filter(user_id=profile.user.id)],
+
+        'generated_certificates': [{
+            'course_id': certificate.course_id.to_deprecated_string(),
+            'verify_uuid': certificate.verify_uuid,
+            'download_uuid': certificate.download_uuid,
+            'download_url': certificate.download_url,
+            'grade': certificate.grade,
+            'key': certificate.key,
+            'distinction': certificate.distinction,
+            'status': certificate.status,
+            'mode': certificate.mode,
+            'name': certificate.name,
+            'created_date': certificate.created_date.__str__(),
+            'modified_date': certificate.modified_date.__str__(),
+            'error_reason': certificate.error_reason,
+        } for certificate in GeneratedCertificate.objects.filter(user_id=profile.user.id)],
+
+        'course_submission_data': {
+            'student_items': [{
+                'course_id': item.course_id,
+                'item_id': item.item_id,
+                'item_type': item.item_type,
+            } for item in StudentItem.objects.filter(student_id=anonymous_user_id)],
+
+            'submissions': [{
+                'uuid': submission.uuid,
+                'attempt_number': submission.attempt_number,
+                'submitted_at': submission.submitted_at.__str__(),
+                'created_at': submission.created_at.__str__(),
+                'answer': submission.answer,
+                'answer_file_url': get_file_url(submission.answer),
+                'student_item_id': submission.student_item_id,
+                'status': submission.status,
+            } for submission in Submission.objects.filter(
+                student_item__student_id=anonymous_user_id
+            )],
+
+            'student_scores': [{
+                'points_earned': score.points_earned,
+                'points_possible': score.points_possible,
+                'created_at': score.created_at.__str__(),
+                'reset': score.reset,
+                'student_item_id': score.student_item_id,
+                'submission_id': score.submission_id,
+            } for score in Score.objects.filter(student_item__student_id=anonymous_user_id)],
+        }
+    }

openedx/features/data_extract/management/commands/get_course_analytics.py

@@ -0,0 +1,65 @@
+import base64
+import json
+import tempfile
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand, CommandError
+
+from common.lib.mandrill_client.client import MandrillClient
+from lms.djangoapps.mailing.management.commands.mailchimp_sync_course import get_enrolled_students
+from opaque_keys.edx.keys import CourseKey
+from openedx.features.data_extract.models import CourseDataExtraction
+
+from openedx.features.data_extract.helpers import (
+    get_course_structure,
+    get_teams_data,
+    get_user_demographic_data,
+    get_user_progress_data,
+)
+from student.models import AnonymousUserId
+
+
+class Command(BaseCommand):
+    help = 'Generates the analytics data for each course_id in coursedataextraction table'
+
+    def handle(self, **options):
+        target_courses = CourseDataExtraction.objects.all()
+
+        for target_course in target_courses:
+            emails = map(unicode.strip, target_course.emails.split(','))
+            course_key = CourseKey.from_string(target_course.course_id)
+
+            course_data = {
+                'course_structure': get_course_structure(course_key),
+                'team_data': get_teams_data(course_key),
+                'user_data': []
+            }
+
+            user_profiles = get_enrolled_students(target_course.course_id)
+            anon_user_ids = dict(list(map(lambda x: (x.user.username, x.anonymous_user_id,),
+                                          AnonymousUserId.objects.filter(course_id=course_key))))
+
+            for profile in user_profiles:
+                demographic_data = get_user_demographic_data(profile)
+                progress_data = get_user_progress_data(course_key, profile, anon_user_ids[profile.user.username])
+
+                course_data['user_data'].append({
+                    'demographic_data': demographic_data,
+                    'progress_data': progress_data,
+                })
+
+            with tempfile.TemporaryFile() as tmp:
+                for email in emails:
+                    MandrillClient().send_mail(
+                        template_name=MandrillClient.ACUMEN_DATA_TEMPLATE,
+                        user_email=email,
+                        context={},
+                        attachments=[
+                            {
+                                "type": "text/plain",
+                                "name": "data.json",
+                                "content": base64.encodestring(json.dumps(course_data))
+                            }
+                        ]
+                    )