diff --git a/common/lib/mandrill_client/client.py b/common/lib/mandrill_client/client.py index ef9ff8cc720f..725b1b37de17 100644 --- a/common/lib/mandrill_client/client.py +++ b/common/lib/mandrill_client/client.py @@ -7,6 +7,7 @@ class MandrillClient(object): + ACUMEN_DATA_TEMPLATE = 'acumen-data' PASSWORD_RESET_TEMPLATE = 'template-60' USER_ACCOUNT_ACTIVATION_TEMPLATE = 'template-61' ORG_ADMIN_ACTIVATION_TEMPLATE = 'org-admin-identified' @@ -24,7 +25,7 @@ class MandrillClient(object): def __init__(self): self.mandrill_client = mandrill.Mandrill(settings.MANDRILL_API_KEY) - def send_mail(self, template_name, user_email, context): + def send_mail(self, template_name, user_email, context, attachments=[]): """ calls the mandrill API for the specific template and email @@ -42,7 +43,8 @@ def send_mail(self, template_name, user_email, context): message={ 'from_email': settings.NOTIFICATION_FROM_EMAIL, 'to': [{'email': user_email}], - 'global_merge_vars': global_merge_vars + 'global_merge_vars': global_merge_vars, + 'attachments': attachments, }, ) log.info(result) diff --git a/lms/envs/common.py b/lms/envs/common.py index 08356ef78369..9bba93f5df32 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2229,6 +2229,9 @@ # student_dashboard App 'lms.djangoapps.student_dashboard', + + # Data extraction App + 'openedx.features.data_extract', ) ######################### CSRF ######################################### @@ -2692,6 +2695,7 @@ 'enterprise', # Required by the Enterprise App 'django_object_actions', # https://github.com/crccheck/django-object-actions + ) for app_name in OPTIONAL_APPS: diff --git a/openedx/features/__init__.py b/openedx/features/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/__init__.py b/openedx/features/data_extract/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/admin.py b/openedx/features/data_extract/admin.py new file mode 100644 index 000000000000..d716c34546e3 --- /dev/null +++ b/openedx/features/data_extract/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from openedx.features.data_extract.models import CourseDataExtraction + +admin.site.register(CourseDataExtraction) + diff --git a/openedx/features/data_extract/app.py b/openedx/features/data_extract/app.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/helpers.py b/openedx/features/data_extract/helpers.py new file mode 100644 index 000000000000..7f4dd10c9843 --- /dev/null +++ b/openedx/features/data_extract/helpers.py @@ -0,0 +1,204 @@ +import requests +import json + +from django.conf import settings + +from certificates.models import GeneratedCertificate +from courseware.models import StudentModule +from lms.djangoapps.grades.models import PersistentCourseGrade, PersistentSubsectionGrade +from lms.djangoapps.onboarding.models import Organization +from lms.djangoapps.teams.models import CourseTeamMembership, CourseTeam +from student.models import AnonymousUserId +from lms.djangoapps.teams.models import CourseTeamMembership, CourseTeam +from openassessment.fileupload import api as ora_file_upload_api +from openedx.core.djangoapps.content.course_structures.models import CourseStructure +from submissions.models import StudentItem, Submission, Score + + +def get_file_url(answer): + try: + return ora_file_upload_api.get_download_url(answer.file_key) + except AttributeError: + return "" + else: + return "" + + +def get_course_structure(course_key): + """ + Returns data about the course structure to course_data dict + + Arguments: + course_key (CourseKey): CourseKey object for specified course + """ + course_structure = CourseStructure.objects.get(course_id=course_key) + + return { + 'created': course_structure.created.__str__(), + 'modified': course_structure.modified.__str__(), + 'course_id': course_structure.course_id.to_deprecated_string(), + 'structure_json': course_structure.structure_json, + 'discussion_id_map_json': course_structure.discussion_id_map_json, + } + + +def get_teams_data(course_key): + """ + Returns data about all the teams in a course + + Arguments: + course_key (CourseKey): CourseKey object for specified course + """ + course_teams = CourseTeam.objects.filter(course_id=course_key) + team_data = [] + for team in course_teams: + team_data.push({ + 'team_id': course_team.team_id, + 'name': course_team.name, + 'course_id': course_team.course_id.to_deprecated_string(), + 'topic_id': course_team.topic_id, + 'date_created': course_team.date_created.__str__(), + 'description': course_team.description, + 'country': course_team.country, + 'language': course_team.language, + 'last_activity_at': course_team.last_activity_at.__str__(), + 'team_size': course_team.team_size, + }) + return team_data + + +def get_user_demographic_data(profile): + """ + Returns the demographic data for a single user + + Arguments: + profile (UserProfile): UserProfile object for the learner + """ + # get the user community profile data from NodeBB API + data_endpoint = settings.NODEBB_ENDPOINT + '/api/v2/users/data' + headers = {'Authorization': 'Bearer ' + settings.NODEBB_MASTER_TOKEN} + response = requests.post(data_endpoint, + data={'_uid': 1, 'username': profile.user.username}, + headers=headers) + + user_community_data = json.loads(response._content)['payload'] + + return { + 'student_id': profile.user.id, + 'email': profile.user.email, + 'first_name': profile.user.first_name, + 'last_name': profile.user.last_name, + 'date_joined': profile.user.date_joined.__str__(), + 'bio': profile.bio, + 'city': profile.city, + 'country': profile.country.__str__(), + 'language': profile.language, + 'english_proficiency': profile.user.extended_profile.english_proficiency, + 'label': profile.user.extended_profile.organization.label if + profile.user.extended_profile.organization else '', + 'reputation': user_community_data['reputation'], + 'postcount': user_community_data['postcount'], + } + + +def get_user_progress_data(course_key, profile, anonymous_user_id): + """ + Returns all the data regarding the progress the user has made in a course + + Arguments: + course_key (CourseKey): CourseKey object for specified course + profile (UserProfile): UserProfile object for the learner + """ + + return { + 'team_memberships': [{ + 'team_id': membership.team_id, + 'date_joined': membership.date_joined.__str__(), + 'last_activity_at': membership.last_activity_at.__str__(), + } for membership in CourseTeamMembership.objects.filter(user_id=profile.user.id)], + + 'student_modules': [{ + 'module_type': module.module_type, + 'module_id': module.module_state_key.to_deprecated_string(), + 'course_id': module.course_id.to_deprecated_string(), + 'state': module.state, + 'grade': module.grade, + 'max_grade': module.max_grade, + 'done': module.done, + 'created': module.created.__str__(), + 'modified': module.modified.__str__(), + } for module in StudentModule.objects.filter(student_id=profile.user.id)], + + 'persistent_course_grades': [{ + 'created': course_grade.created.__str__(), + 'modified': course_grade.modified.__str__(), + 'course_id': course_grade.course_id.to_deprecated_string(), + 'course_edited_timestamp': course_grade.course_edited_timestamp.__str__(), + 'course_version': course_grade.course_version, + 'grading_policy_hash': course_grade.grading_policy_hash, + 'percent_grade': course_grade.percent_grade, + 'letter_grade': course_grade.letter_grade, + 'passed_timestamp': course_grade.passed_timestamp.__str__(), + } for course_grade in PersistentCourseGrade.objects.filter(user_id=profile.user.id)], + + 'persistent_subsection_grades': [{ + 'created': subsection_grade.created.__str__(), + 'modified': subsection_grade.modified.__str__(), + 'course_id': subsection_grade.course_id.to_deprecated_string(), + 'usage_key': subsection_grade.full_usage_key.to_deprecated_string(), + 'subtree_edited_timestamp': subsection_grade.subtree_edited_timestamp.__str__(), + 'course_version': subsection_grade.course_version, + 'earned_all': subsection_grade.earned_all, + 'possible_all': subsection_grade.possible_all, + 'earned_graded': subsection_grade.earned_graded, + 'possible_graded': subsection_grade.possible_graded, + 'visible_blocks': subsection_grade.visible_blocks.blocks_json, + 'first_attempted': subsection_grade.first_attempted.__str__(), + } for subsection_grade in PersistentSubsectionGrade.objects.filter(user_id=profile.user.id)], + + 'generated_certificates': [{ + 'course_id': certificate.course_id.to_deprecated_string(), + 'verify_uuid': certificate.verify_uuid, + 'download_uuid': certificate.download_uuid, + 'download_url': certificate.download_url, + 'grade': certificate.grade, + 'key': certificate.key, + 'distinction': certificate.distinction, + 'status': certificate.status, + 'mode': certificate.mode, + 'name': certificate.name, + 'created_date': certificate.created_date.__str__(), + 'modified_date': certificate.modified_date.__str__(), + 'error_reason': certificate.error_reason, + } for certificate in GeneratedCertificate.objects.filter(user_id=profile.user.id)], + + 'course_submission_data': { + 'student_items': [{ + 'course_id': item.course_id, + 'item_id': item.item_id, + 'item_type': item.item_type, + } for item in StudentItem.objects.filter(student_id=anonymous_user_id)], + + 'submissions': [{ + 'uuid': submission.uuid, + 'attempt_number': submission.attempt_number, + 'submitted_at': submission.submitted_at.__str__(), + 'created_at': submission.created_at.__str__(), + 'answer': submission.answer, + 'answer_file_url': get_file_url(submission.answer), + 'student_item_id': submission.student_item_id, + 'status': submission.status, + } for submission in Submission.objects.filter( + student_item__student_id=anonymous_user_id + )], + + 'student_scores': [{ + 'points_earned': score.points_earned, + 'points_possible': score.points_possible, + 'created_at': score.created_at.__str__(), + 'reset': score.reset, + 'student_item_id': score.student_item_id, + 'submission_id': score.submission_id, + } for score in Score.objects.filter(student_item__student_id=anonymous_user_id)], + } + } diff --git a/openedx/features/data_extract/management/__init__.py b/openedx/features/data_extract/management/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/management/commands/__init__.py b/openedx/features/data_extract/management/commands/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/management/commands/get_course_analytics.py b/openedx/features/data_extract/management/commands/get_course_analytics.py new file mode 100644 index 000000000000..56f60c22ba36 --- /dev/null +++ b/openedx/features/data_extract/management/commands/get_course_analytics.py @@ -0,0 +1,65 @@ +import base64 +import json +import tempfile + +from django.conf import settings +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand, CommandError + +from common.lib.mandrill_client.client import MandrillClient +from lms.djangoapps.mailing.management.commands.mailchimp_sync_course import get_enrolled_students +from opaque_keys.edx.keys import CourseKey +from openedx.features.data_extract.models import CourseDataExtraction + +from openedx.features.data_extract.helpers import ( + get_course_structure, + get_teams_data, + get_user_demographic_data, + get_user_progress_data, +) +from student.models import AnonymousUserId + + +class Command(BaseCommand): + help = 'Generates the analytics data for each course_id in coursedataextraction table' + + def handle(self, **options): + target_courses = CourseDataExtraction.objects.all() + + for target_course in target_courses: + emails = map(unicode.strip, target_course.emails.split(',')) + course_key = CourseKey.from_string(target_course.course_id) + + course_data = { + 'course_structure': get_course_structure(course_key), + 'team_data': get_teams_data(course_key), + 'user_data': [] + } + + user_profiles = get_enrolled_students(target_course.course_id) + anon_user_ids = dict(list(map(lambda x: (x.user.username, x.anonymous_user_id,), + AnonymousUserId.objects.filter(course_id=course_key)))) + + for profile in user_profiles: + demographic_data = get_user_demographic_data(profile) + progress_data = get_user_progress_data(course_key, profile, anon_user_ids[profile.user.username]) + + course_data['user_data'].append({ + 'demographic_data': demographic_data, + 'progress_data': progress_data, + }) + + with tempfile.TemporaryFile() as tmp: + for email in emails: + MandrillClient().send_mail( + template_name=MandrillClient.ACUMEN_DATA_TEMPLATE, + user_email=email, + context={}, + attachments=[ + { + "type": "text/plain", + "name": "data.json", + "content": base64.encodestring(json.dumps(course_data)) + } + ] + ) diff --git a/openedx/features/data_extract/migrations/0001_initial.py b/openedx/features/data_extract/migrations/0001_initial.py new file mode 100644 index 000000000000..cd6552cc9a25 --- /dev/null +++ b/openedx/features/data_extract/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='CourseDataExtraction', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('course_id', models.CharField(max_length=255)), + ('emails', models.TextField()), + ], + ), + ] diff --git a/openedx/features/data_extract/migrations/__init__.py b/openedx/features/data_extract/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/data_extract/models.py b/openedx/features/data_extract/models.py new file mode 100644 index 000000000000..b2908eed57a3 --- /dev/null +++ b/openedx/features/data_extract/models.py @@ -0,0 +1,14 @@ +from django.db import models + + +class CourseDataExtraction(models.Model): + """ + These courses are marked for data extraction. These will be used to determine data of which courses is to be extracted + + @course_id: (string) id of the course. can be gotten through to_deprecated_string() method of CourseKey object + """ + course_id = models.CharField(max_length=255) + emails = models.TextField() + + def __unicode__(self): + return '{}'.format(self.course_id) diff --git a/openedx/features/data_extract/urls.py b/openedx/features/data_extract/urls.py new file mode 100644 index 000000000000..42397ff70986 --- /dev/null +++ b/openedx/features/data_extract/urls.py @@ -0,0 +1,4 @@ +from django.conf.urls import url + + +urlpatterns = []