From f99951eee0213494fd242de836c0ae9c2190f777 Mon Sep 17 00:00:00 2001 From: Vasyl Nakvasiuk Date: Fri, 14 Jun 2013 15:35:22 +0300 Subject: [PATCH 01/30] import subtitles into studio: add python side functionality --- cms/djangoapps/contentstore/utils.py | 164 +++++++++++++++++++++- cms/djangoapps/contentstore/views/item.py | 71 +++++++++- cms/urls.py | 1 + 3 files changed, 230 insertions(+), 6 deletions(-) diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index c9c40ab95dac..f66b45c614d3 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -1,11 +1,28 @@ +"""Helpers functions.""" + +#pylint: disable=E1103 + +from __future__ import division + +import copy +import logging +import re +import json +from functools import wraps +import HTMLParser + +import requests +from lxml import etree from django.conf import settings +from django.core.urlresolvers import reverse + +from cache_toolbox.core import del_cached_content +from django_comment_client.utils import JsonResponse +from xmodule.contentstore.content import StaticContent +from xmodule.contentstore.django import contentstore from xmodule.modulestore import Location from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import ItemNotFoundError -from django.core.urlresolvers import reverse -import copy -import logging -import re from xmodule.modulestore.draft import DIRECT_ONLY_CATEGORIES log = logging.getLogger(__name__) @@ -255,3 +272,142 @@ def remove_extra_panel_tab(tab_type, course): course_tabs = [ct for ct in course_tabs if ct != tab_panel] changed = True return changed, course_tabs + + +def return_ajax_status(view_function): + """Except, that view function return True/False, and convert + response to JSON HTTP response: + {"success": true} or {"success": false} + """ + @wraps(view_function) + def new_view_function(request, *args, **kwargs): + """New view functions for decorator result.""" + result = view_function(request, *args, **kwargs) + if isinstance(result, tuple): + status = result[0] + response_data = result[1] + else: + status = result + response_data = {} + response_data.update({'success': status}) + return JsonResponse(response_data) + return new_view_function + + +def generate_subs(speed, source_speed, source_subs): + """Generate and return subtitles dictionary for speed equal to + `speed` value, using `source_speed` and `source_subs`.""" + if speed == source_speed: + return source_subs + + coefficient = speed / source_speed + subs = { + 'start': [ + int(round(timestamp * coefficient)) for + timestamp in source_subs['start'] + ], + 'end': [ + int(round(timestamp * coefficient)) for + timestamp in source_subs['end'] + ], + 'text': source_subs['text']} + return subs + + +def save_subs_to_store(subs, subs_id, item): + """Save subtitles into `StaticContent`.""" + filedata = json.dumps(subs, indent=2) + mime_type = 'application/json' + filename = 'subs_{0}.srt.sjson'.format(subs_id) + + content_location = StaticContent.compute_location( + item.location.org, item.location.course, filename) + content = StaticContent(content_location, filename, mime_type, filedata) + contentstore().save(content) + del_cached_content(content_location) + return content_location + + +def download_youtube_subs(youtube_subs, item): + """Download subtitles from Youtube using `youtube_ids`, and + save them to assets for `item` module.""" + html_parser = HTMLParser.HTMLParser() + status_dict = {} + + # Iterate from lowest to highest speed and try to do download subtitles + # from the Youtube service. + for speed, youtube_id in sorted(youtube_subs.iteritems()): + data = requests.get( + "http://video.google.com/timedtext", + params={'lang': 'en', 'v': youtube_id}) + + if data.status_code != 200 or not data.text: + status_dict.update({speed: False}) + log.error("Can't recieved correct subtitles from Youtube.") + continue + + sub_starts = [] + sub_ends = [] + sub_texts = [] + + xmltree = etree.fromstring(str(data.text)) + for element in xmltree: + if element.tag == "text": + start = float(element.get("start")) + duration = float(element.get("dur")) + text = element.text + end = start + duration + + if text: + # Start and end are an int representing the + # millisecond timestamp. + sub_starts.append(int(start * 1000)) + sub_ends.append(int((end + 0.0001) * 1000)) + sub_texts.append( + html_parser.unescape(text.replace('\n', ' '))) + + available_speed = speed + subs = { + 'start': sub_starts, + 'end': sub_ends, + 'text': sub_texts} + + save_subs_to_store(subs, youtube_id, item) + + log.info( + """Subtitles for Youtube ID {0} (speed {1}) + are downloaded from Youtube and + saved.""".format(youtube_id, speed) + ) + + status_dict.update({speed: True}) + + if not any(status_dict.itervalues()): + log.error("Can't find any subtitles on the Youtube service.") + return False + + # When we exit from the previous loop, `available_speed` and `subs` + # are the subtitles data with the highest speed available on the + # Youtube service. We use the highest speed as main speed for the + # generation other subtitles, cause during calculation timestamps + # for lower speeds we just use multiplication istead of division. + + # Generate subtitles for missed speeds. + for speed, status in status_dict.iteritems(): + if not status: + save_subs_to_store( + generate_subs(speed, available_speed, subs), + youtube_subs[speed], + item) + + log.info( + """Subtitles for Youtube ID {0} (speed {1}) + are generated from Youtube ID {2} (speed {3}) and + saved.""".format( + youtube_subs[speed], + speed, + youtube_subs[available_speed], + available_speed) + ) + + return True diff --git a/cms/djangoapps/contentstore/views/item.py b/cms/djangoapps/contentstore/views/item.py index abc5f48564e6..1fc5837752ea 100644 --- a/cms/djangoapps/contentstore/views/item.py +++ b/cms/djangoapps/contentstore/views/item.py @@ -1,5 +1,9 @@ +"""Views for items (modules).""" + import json +import logging from uuid import uuid4 +from lxml import etree from django.core.exceptions import PermissionDenied from django.http import HttpResponse @@ -8,13 +12,16 @@ from xmodule.modulestore import Location from xmodule.modulestore.django import modulestore from xmodule.modulestore.inheritance import own_metadata +from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError from util.json_request import expect_json -from ..utils import get_modulestore +from ..utils import get_modulestore, download_youtube_subs, return_ajax_status from .access import has_access from .requests import _xmodule_recurse -__all__ = ['save_item', 'clone_item', 'delete_item'] +__all__ = ['save_item', 'clone_item', 'delete_item', 'import_subtitles'] + +log = logging.getLogger(__name__) # cdodge: these are categories which should not be parented, they are detached from the hierarchy DETACHED_CATEGORIES = ['about', 'static_tab', 'course_info'] @@ -23,6 +30,7 @@ @login_required @expect_json def save_item(request): + """View saving items.""" item_location = request.POST['id'] # check permissions for this user within this course @@ -71,9 +79,67 @@ def save_item(request): return HttpResponse() +@login_required +@expect_json +@return_ajax_status +def import_subtitles(request): + """Try to import subtitles from Youtube for current modules.""" + + # This view return True/False, cause we use `return_ajax_status` + # view decorator. + + item_location = request.POST.get('id') + if not item_location: + log.error('POST data without "id" property.') + return False + + try: + item = modulestore().get_item(item_location) + except (ItemNotFoundError, InvalidLocationError): + log.error("Can't find item by location.") + return False + + # Check permissions for this user within this course. + if not has_access(request.user, item_location): + raise PermissionDenied() + + if item.category != 'videoalpha': + log.error('Subtitles are supported only for videoalpha" modules.') + return False + + try: + xmltree = etree.fromstring(item.data) + except etree.XMLSyntaxError: + log.error("Can't parse source XML.") + return False + + youtube = xmltree.get('youtube') + if not youtube: + log.error('Missing or blank "youtube" attribute.') + return False + + try: + youtube_subs = dict([ + (float(i.split(':')[0]), i.split(':')[1]) + for i in youtube.split(',') + ]) + except (IndexError, ValueError): + # Get `IndexError` if after splitting by ':' we have one item + # (missing ':' in the "youtube" attribute value). + # Get `ValueError` when after splitting by ':' key can't convert + # to float. + log.error('Bad "youtube" attribute.') + return False + + status = download_youtube_subs(youtube_subs, item) + + return status + + @login_required @expect_json def clone_item(request): + """View for cloning items.""" parent_location = Location(request.POST['parent_location']) template = Location(request.POST['template']) @@ -102,6 +168,7 @@ def clone_item(request): @login_required @expect_json def delete_item(request): + """View for removing items.""" item_location = request.POST['id'] item_location = Location(item_location) diff --git a/cms/urls.py b/cms/urls.py index d04c31116132..eeb2ebf898c8 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -23,6 +23,7 @@ url(r'^unpublish_unit$', 'contentstore.views.unpublish_unit', name='unpublish_unit'), url(r'^create_new_course', 'contentstore.views.create_new_course', name='create_new_course'), url(r'^reorder_static_tabs', 'contentstore.views.reorder_static_tabs', name='reorder_static_tabs'), + url(r'^import_subtitles$', 'contentstore.views.import_subtitles', name='import_subtitles'), url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)$', 'contentstore.views.course_index', name='course_index'), From 5feea5191299573435d86447e483e1c05bf2f558 Mon Sep 17 00:00:00 2001 From: Vasyl Nakvasiuk Date: Tue, 18 Jun 2013 12:58:02 +0300 Subject: [PATCH 02/30] import subtitles into studio: add python integration tests --- .../tests/test_course_settings.py | 4 +- .../contentstore/tests/test_item.py | 182 +++++++++++++++++- 2 files changed, 182 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_course_settings.py b/cms/djangoapps/contentstore/tests/test_course_settings.py index 40ec2ed3c734..8e764d5a17cd 100644 --- a/cms/djangoapps/contentstore/tests/test_course_settings.py +++ b/cms/djangoapps/contentstore/tests/test_course_settings.py @@ -54,8 +54,8 @@ def setUp(self): self.client = Client() self.client.login(username=uname, password=password) - course = CourseFactory.create(template='i4x://edx/templates/course/Empty', org='MITx', number='999', display_name='Robot Super Course') - self.course_location = course.location + self.course = CourseFactory.create(template='i4x://edx/templates/course/Empty', org='MITx', number='999', display_name='Robot Super Course') + self.course_location = self.course.location class CourseDetailsTestCase(CourseTestCase): diff --git a/cms/djangoapps/contentstore/tests/test_item.py b/cms/djangoapps/contentstore/tests/test_item.py index 1831a5769a61..bc1638792283 100644 --- a/cms/djangoapps/contentstore/tests/test_item.py +++ b/cms/djangoapps/contentstore/tests/test_item.py @@ -1,15 +1,25 @@ +"""Tests for items views.""" +import json + +from lxml import etree +from django.core.urlresolvers import reverse + from contentstore.tests.test_course_settings import CourseTestCase from xmodule.modulestore.tests.factories import CourseFactory -from django.core.urlresolvers import reverse +from xmodule.modulestore.django import modulestore +from xmodule.contentstore.django import contentstore +from xmodule.contentstore.content import StaticContent +from xmodule.exceptions import NotFoundError class DeleteItem(CourseTestCase): + """Tests for '/delete_item' url.""" def setUp(self): """ Creates the test course with a static page in it. """ super(DeleteItem, self).setUp() self.course = CourseFactory.create(org='mitX', number='333', display_name='Dummy Course') - def testDeleteStaticPage(self): + def test_delete_static_page(self): # Add static tab data = { 'parent_location': 'i4x://mitX/333/course/Dummy_Course', @@ -24,5 +34,173 @@ def testDeleteStaticPage(self): self.assertEqual(resp.status_code, 200) +class BaseSubtitles(CourseTestCase): + """Base test class for subtitles tests.""" + + org = 'MITx' + number = '999' + + def clear_subs_content(self): + """Remove, if subtitles content exists.""" + for youtube_id in self.get_youtube_ids().values(): + filename = 'subs_{0}.srt.sjson'.format(youtube_id) + content_location = StaticContent.compute_location( + self.org, self.number, filename) + try: + content = contentstore().find(content_location) + contentstore().delete(content.get_id()) + except NotFoundError: + pass + + def setUp(self): + """Create initial data.""" + super(BaseSubtitles, self).setUp() + + # Add videoalpha module + data = { + 'parent_location': str(self.course_location), + 'template': 'i4x://edx/templates/videoalpha/Video_Alpha' + } + resp = self.client.post(reverse('clone_item'), data) + self.item_location = json.loads(resp.content).get('id') + self.assertEqual(resp.status_code, 200) + + # hI10vDNYz4M - valid Youtube ID with subtitles. + # JMD_ifUUfsU, AKqURZnYqpk, DYpADpL7jAY - valid Youtube IDs + # without subtitles. + data = '' + modulestore().update_item(self.item_location, data) + + self.item = modulestore().get_item(self.item_location) + + # Remove all subtitles for current module. + self.clear_subs_content() + + def get_youtube_ids(self): + """Return youtube speeds and ids.""" + xmltree = etree.fromstring(self.item.data) + youtube = xmltree.get('youtube') + return dict([ + (float(i.split(':')[0]), i.split(':')[1]) + for i in youtube.split(',') + ]) + + +class ImportSubtitles(BaseSubtitles): + """Tests for '/import_subtitles' url.""" + + def test_success_videoalpha_module_subs_importing(self): + # Import subtitles. + resp = self.client.post( + reverse('import_subtitles'), {'id': self.item_location}) + + self.assertEqual(resp.status_code, 200) + self.assertTrue(json.loads(resp.content).get('success')) + + # Check assets status after importing subtitles. + for youtube_id in self.get_youtube_ids().values(): + filename = 'subs_{0}.srt.sjson'.format(youtube_id) + content_location = StaticContent.compute_location( + self.org, self.number, filename) + self.assertTrue(contentstore().find(content_location)) + + def test_fail_data_without_id(self): + resp = self.client.post( + reverse('import_subtitles'), {}) + + self.assertEqual(resp.status_code, 200) + self.assertFalse(json.loads(resp.content).get('success')) + + def test_fail_data_with_bad_location(self): + # Test for raising `InvalidLocationError` exception. + resp = self.client.post( + reverse('import_subtitles'), {'id': 'BAD_LOCATION'}) + + self.assertEqual(resp.status_code, 200) + self.assertFalse(json.loads(resp.content).get('success')) + + # Test for raising `ItemNotFoundError` exception. + resp = self.client.post( + reverse('import_subtitles'), + {'id': '{0}_{1}'.format(self.item_location, 'BAD_LOCATION')} + ) + + self.assertEqual(resp.status_code, 200) + self.assertFalse(json.loads(resp.content).get('success')) + + def test_fail_for_non_videoalpha_module(self): + # Video module: setup + data = { + 'parent_location': str(self.course_location), + 'template': 'i4x://edx/templates/video/default' + } + resp = self.client.post(reverse('clone_item'), data) + item_location = json.loads(resp.content).get('id') + data = '