Improve: add data extraction management command

common/lib/mandrill_client/client.py

@@ -7,6 +7,7 @@
 
 
 class MandrillClient(object):
+    ACUMEN_DATA_TEMPLATE = 'acumen-data'
     PASSWORD_RESET_TEMPLATE = 'template-60'
     USER_ACCOUNT_ACTIVATION_TEMPLATE = 'template-61'
     ORG_ADMIN_ACTIVATION_TEMPLATE = 'org-admin-identified'
@@ -24,7 +25,7 @@ class MandrillClient(object):
     def __init__(self):
         self.mandrill_client = mandrill.Mandrill(settings.MANDRILL_API_KEY)
 
-    def send_mail(self, template_name, user_email, context):
+    def send_mail(self, template_name, user_email, context, attachments=[]):
         """
         calls the mandrill API for the specific template and email
 
@@ -42,7 +43,8 @@ def send_mail(self, template_name, user_email, context):
                 message={
                     'from_email': settings.NOTIFICATION_FROM_EMAIL,
                     'to': [{'email': user_email}],
-                    'global_merge_vars': global_merge_vars
+                    'global_merge_vars': global_merge_vars,
+                    'attachments': attachments,
                 },
             )
             log.info(result)

lms/envs/common.py

@@ -2229,6 +2229,9 @@
 
     # student_dashboard App
     'lms.djangoapps.student_dashboard',
+
+    # Data extraction App
+    'openedx.features.data_extract',
 )
 
 ######################### CSRF #########################################
@@ -2692,6 +2695,7 @@
     'enterprise',
     # Required by the Enterprise App
     'django_object_actions',  # https://github.com/crccheck/django-object-actions
+
 )
 
 for app_name in OPTIONAL_APPS:

openedx/features/data_extract/admin.py

@@ -0,0 +1,5 @@
+from django.contrib import admin
+from openedx.features.data_extract.models import CourseDataExtraction
+
+admin.site.register(CourseDataExtraction)
+

openedx/features/data_extract/helpers.py

@@ -0,0 +1,204 @@
+import requests
+import json
+
+from django.conf import settings
+
+from certificates.models import GeneratedCertificate
+from courseware.models import StudentModule
+from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGrade
+from lms.djangoapps.onboarding.models import Organization
+from lms.djangoapps.teams.models import CourseTeamMembership, CourseTeam
+from student.models import AnonymousUserId
+from lms.djangoapps.teams.models import CourseTeamMembership, CourseTeam
+from openassessment.fileupload import api as ora_file_upload_api
+from openedx.core.djangoapps.content.course_structures.models import CourseStructure
+from submissions.models import StudentItem, Submission, Score
+
+
+def get_file_url(answer):
+    try:
+        return ora_file_upload_api.get_download_url(answer.file_key)
+    except AttributeError:
+        return ""
+    else:
+        return ""
+
+
+def get_course_structure(course_key):
+    """
+    Returns data about the course structure to course_data dict
+
+    Arguments:
+    course_key (CourseKey): CourseKey object for specified course
+    """
+    course_structure = CourseStructure.objects.get(course_id=course_key)
+
+    return {
+        'created': course_structure.created.__str__(),
+        'modified': course_structure.modified.__str__(),
+        'course_id': course_structure.course_id.to_deprecated_string(),
+        'structure_json': course_structure.structure_json,
+        'discussion_id_map_json': course_structure.discussion_id_map_json,
+    }
+
+
+def get_teams_data(course_key):
+    """
+    Returns data about all the teams in a course
+
+    Arguments:
+    course_key (CourseKey): CourseKey object for specified course
+    """
+    course_teams = CourseTeam.objects.filter(course_id=course_key)
+    team_data = []
+    for team in course_teams:
+        team_data.push({
+            'team_id': course_team.team_id,
+            'name': course_team.name,
+            'course_id': course_team.course_id.to_deprecated_string(),
+            'topic_id': course_team.topic_id,
+            'date_created': course_team.date_created.__str__(),
+            'description': course_team.description,
+            'country': course_team.country,
+            'language': course_team.language,
+            'last_activity_at': course_team.last_activity_at.__str__(),
+            'team_size': course_team.team_size,
+        })
+    return team_data
+
+
+def get_user_demographic_data(profile):
+    """
+    Returns the demographic data for a single user
+
+    Arguments:
+    profile (UserProfile): UserProfile object for the learner
+    """
+    # get the user community profile data from NodeBB API
+    data_endpoint = settings.NODEBB_ENDPOINT + '/api/v2/users/data'
+    headers = {'Authorization': 'Bearer ' + settings.NODEBB_MASTER_TOKEN}
+    response = requests.post(data_endpoint,
+                             data={'_uid': 1, 'username': profile.user.username},
+                             headers=headers)
+
+    user_community_data = json.loads(response._content)['payload']
+
+    return {
+        'student_id': profile.user.id,
+        'email': profile.user.email,
+        'first_name': profile.user.first_name,
+        'last_name': profile.user.last_name,
+        'date_joined': profile.user.date_joined.__str__(),
+        'bio': profile.bio,
+        'city': profile.city,
+        'country': profile.country.__str__(),
+        'language': profile.language,
+        'english_proficiency': profile.user.extended_profile.english_proficiency,
+        'label': profile.user.extended_profile.organization.label if
+        profile.user.extended_profile.organization else '',
+        'reputation': user_community_data['reputation'],
+        'postcount': user_community_data['postcount'],
+    }
+
+
+def get_user_progress_data(course_key, profile, anonymous_user_id):
+    """
+    Returns all the data regarding the progress the user has made in a course
+
+    Arguments:
+    course_key (CourseKey): CourseKey object for specified course
+    profile (UserProfile): UserProfile object for the learner
+    """
+
+    return {
+        'team_memberships': [{
+            'team_id': membership.team_id,
+            'date_joined': membership.date_joined.__str__(),
+            'last_activity_at': membership.last_activity_at.__str__(),
+        } for membership in CourseTeamMembership.objects.filter(user_id=profile.user.id)],
+
+        'student_modules': [{
+            'module_type': module.module_type,
+            'module_id': module.module_state_key.to_deprecated_string(),
+            'course_id': module.course_id.to_deprecated_string(),
+            'state': module.state,
+            'grade': module.grade,
+            'max_grade': module.max_grade,
+            'done': module.done,
+            'created': module.created.__str__(),
+            'modified': module.modified.__str__(),
+        } for module in StudentModule.objects.filter(student_id=profile.user.id)],
+
+        'persistent_course_grades': [{
+            'created': course_grade.created.__str__(),
+            'modified': course_grade.modified.__str__(),
+            'course_id': course_grade.course_id.to_deprecated_string(),
+            'course_edited_timestamp': course_grade.course_edited_timestamp.__str__(),
+            'course_version': course_grade.course_version,
+            'grading_policy_hash': course_grade.grading_policy_hash,
+            'percent_grade': course_grade.percent_grade,
+            'letter_grade': course_grade.letter_grade,
+            'passed_timestamp': course_grade.passed_timestamp.__str__(),
+        } for course_grade in PersistentCourseGrade.objects.filter(user_id=profile.user.id)],
+
+        'persistent_subsection_grades': [{
+            'created': subsection_grade.created.__str__(),
+            'modified': subsection_grade.modified.__str__(),
+            'course_id': subsection_grade.course_id.to_deprecated_string(),
+            'usage_key': subsection_grade.full_usage_key.to_deprecated_string(),
+            'subtree_edited_timestamp': subsection_grade.subtree_edited_timestamp.__str__(),
+            'course_version': subsection_grade.course_version,
+            'earned_all': subsection_grade.earned_all,
+            'possible_all': subsection_grade.possible_all,
+            'earned_graded': subsection_grade.earned_graded,
+            'possible_graded': subsection_grade.possible_graded,
+            'visible_blocks': subsection_grade.visible_blocks.blocks_json,
+            'first_attempted': subsection_grade.first_attempted.__str__(),
+        } for subsection_grade in PersistentSubsectionGrade.objects.filter(user_id=profile.user.id)],
+
+        'generated_certificates': [{
+            'course_id': certificate.course_id.to_deprecated_string(),
+            'verify_uuid': certificate.verify_uuid,
+            'download_uuid': certificate.download_uuid,
+            'download_url': certificate.download_url,
+            'grade': certificate.grade,
+            'key': certificate.key,
+            'distinction': certificate.distinction,
+            'status': certificate.status,
+            'mode': certificate.mode,
+            'name': certificate.name,
+            'created_date': certificate.created_date.__str__(),
+            'modified_date': certificate.modified_date.__str__(),
+            'error_reason': certificate.error_reason,
+        } for certificate in GeneratedCertificate.objects.filter(user_id=profile.user.id)],
+
+        'course_submission_data': {
+            'student_items': [{
+                'course_id': item.course_id,
+                'item_id': item.item_id,
+                'item_type': item.item_type,
+            } for item in StudentItem.objects.filter(student_id=anonymous_user_id)],
+
+            'submissions': [{
+                'uuid': submission.uuid,
+                'attempt_number': submission.attempt_number,
+                'submitted_at': submission.submitted_at.__str__(),
+                'created_at': submission.created_at.__str__(),
+                'answer': submission.answer,
+                'answer_file_url': get_file_url(submission.answer),
+                'student_item_id': submission.student_item_id,
+                'status': submission.status,
+            } for submission in Submission.objects.filter(
+                student_item__student_id=anonymous_user_id
+            )],
+
+            'student_scores': [{
+                'points_earned': score.points_earned,
+                'points_possible': score.points_possible,
+                'created_at': score.created_at.__str__(),
+                'reset': score.reset,
+                'student_item_id': score.student_item_id,
+                'submission_id': score.submission_id,
+            } for score in Score.objects.filter(student_item__student_id=anonymous_user_id)],
+        }
+    }

openedx/features/data_extract/management/commands/get_course_analytics.py

@@ -0,0 +1,65 @@
+import base64
+import json
+import tempfile
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand, CommandError
+
+from common.lib.mandrill_client.client import MandrillClient
+from lms.djangoapps.mailing.management.commands.mailchimp_sync_course import get_enrolled_students
+from opaque_keys.edx.keys import CourseKey
+from openedx.features.data_extract.models import CourseDataExtraction
+
+from openedx.features.data_extract.helpers import (
+    get_course_structure,
+    get_teams_data,
+    get_user_demographic_data,
+    get_user_progress_data,
+)
+from student.models import AnonymousUserId
+
+
+class Command(BaseCommand):
+    help = 'Generates the analytics data for each course_id in coursedataextraction table'
+
+    def handle(self, **options):
+        target_courses = CourseDataExtraction.objects.all()
+
+        for target_course in target_courses:
+            emails = map(unicode.strip, target_course.emails.split(','))
+            course_key = CourseKey.from_string(target_course.course_id)
+
+            course_data = {
+                'course_structure': get_course_structure(course_key),
+                'team_data': get_teams_data(course_key),
+                'user_data': []
+            }
+
+            user_profiles = get_enrolled_students(target_course.course_id)
+            anon_user_ids = dict(list(map(lambda x: (x.user.username, x.anonymous_user_id,),
+                                          AnonymousUserId.objects.filter(course_id=course_key))))
+
+            for profile in user_profiles:
+                demographic_data = get_user_demographic_data(profile)
+                progress_data = get_user_progress_data(course_key, profile, anon_user_ids[profile.user.username])
+
+                course_data['user_data'].append({
+                    'demographic_data': demographic_data,
+                    'progress_data': progress_data,
+                })
+
+            with tempfile.TemporaryFile() as tmp:
+                for email in emails:
+                    MandrillClient().send_mail(
+                        template_name=MandrillClient.ACUMEN_DATA_TEMPLATE,
+                        user_email=email,
+                        context={},
+                        attachments=[
+                            {
+                                "type": "text/plain",
+                                "name": "data.json",
+                                "content": base64.encodestring(json.dumps(course_data))
+                            }
+                        ]
+                    )

openedx/features/data_extract/migrations/0001_initial.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CourseDataExtraction',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('course_id', models.CharField(max_length=255)),
+                ('emails', models.TextField()),
+            ],
+        ),
+    ]

openedx/features/data_extract/models.py

@@ -0,0 +1,14 @@
+from django.db import models
+
+
+class CourseDataExtraction(models.Model):
+    """
+    These courses are marked for data extraction. These will be used to determine data of which courses is to be extracted
+
+    @course_id: (string) id of the course. can be gotten through to_deprecated_string() method of CourseKey object
+    """
+    course_id = models.CharField(max_length=255)
+    emails = models.TextField()
+
+    def __unicode__(self):
+        return '{}'.format(self.course_id)

openedx/features/data_extract/urls.py

@@ -0,0 +1,4 @@
+from django.conf.urls import url
+
+
+urlpatterns = []