diff --git a/cms/djangoapps/content_testing/__init__.py b/cms/djangoapps/content_testing/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/content_testing/models.py b/cms/djangoapps/content_testing/models.py new file mode 100644 index 000000000000..e8da47138aec --- /dev/null +++ b/cms/djangoapps/content_testing/models.py @@ -0,0 +1,688 @@ +""" +Django models used to store and manipulate content tests +""" +from xmodule.modulestore.django import modulestore +from xmodule.modulestore import Location +from contentstore.views.preview import load_preview_module +from lxml import etree +from copy import deepcopy +from difflib import SequenceMatcher + +# dear god why do I need to do this? +from xmodule.modulestore.mongo.draft import as_draft + +# pylint: disable=E1101 + + +def hash_xml(tree): + """ + create a hash of the etree xml element solely based on 'meaningful' parts of the xml string + """ + tree = deepcopy(tree) + remove_ids(tree, (lambda k: k[-2:] == 'id' or k == 'size')) + return etree.tostring(tree).__hash__() + + +def remove_ids(tree, should_be_removed): + """ + remove all keys for which `should_be_removed(attrib)` returns true + """ + for attrib in tree.attrib: + if should_be_removed(attrib): + tree.attrib.pop(attrib) + + # do the same to all the children + for child in tree: + remove_ids(child, should_be_removed) + + +def hash_xml_structure(tree): + """ + create hash of xml that ignores all attributes except ones involving `id` + """ + tree = deepcopy(tree) + remove_ids(tree, (lambda k: True)) + return etree.tostring(tree).__hash__() + + +def condense_attributes(tree): + """ + take an XML tree and collect all `meaningful` attributes into single dict + """ + + tree = deepcopy(tree) + remove_ids(tree, (lambda k: k[-2:] == 'id' or k == 'size')) + attrib = tree.attrib + + # add in childrens attributes + for child in tree: + attrib.update(condense_attributes(child)) + + return attrib + + +def remove_xml_wrapper(tree, name): + """ + Remove all elements by the name of `name` in `tree` but keep + any children by inserting them into the location of `name`. + + Return a new tree. + + Accepts either lxml.etree.Element objects or strings. + """ + + # we want to return the same type we were given + tree = deepcopy(tree) + return_string = False + if isinstance(tree, basestring): + tree = etree.XML(tree) + return_string = True + + for item in tree.iterfind('.//'+name): + # reverse children for inserting + children = [elts for elts in item] + children.reverse() + + # index for insertion + index = item.getparent().index(item) + + # insert the form contents + for child in children: + item.getparent().insert(index, child) + + # remove item + item.getparent().remove(item) + + # return a string if that is what we were given + if return_string: + tree = etree.tostring(tree) + + return tree + + +def condense_dict(dictionary): + """ + returns a string ondensation of the dictionary for %% comparison purposes. + + {'tree': 3, 'apple': 'hello'} -> 'tree3applehello' + """ + + return ''.join([str(key)+str(dictionary[key]) for key in dictionary]) + + +def closeness(model, responder): + """ + Return a value between 0 and ~1 representing how good a match these two are. + 0 = Terribad. 1 = identical. 1.01 = identical in original location. + """ + + # no match if the structure is different + if model.structure_hash != hash_xml_structure(responder.xml): + return 0 + + # almost all the xml will be the same since they have the same structure anyway. + # Thus, we look at just the attributes that are meaningful + model_xml = etree.XML(model.xml_string) + resp_xml = responder.xml + + model_string = condense_dict(condense_attributes(model_xml)) + responder_string = condense_dict(condense_attributes(resp_xml)) + + # use difflib to calculate string closeness + seq = SequenceMatcher(None, model_string, responder_string) + ratio = seq.ratio() + + # favor matches that are in the same location so that if two + # are identically close, it will choose not to move things. The + # way it does this is designed to have a much greater effect + # (proportionally) when the match is close. + if model.string_id == responder.id: + ratio = 1.01 * (1 - (1 - ratio) ** (1.2)) + return ratio + + +class ContentTest(object): + """ + Model for a user-created test for a capa-problem + """ + + ERROR = "error" + PASS = "Pass" + FAIL = "Fail" + NONE = " - Not Yet Run - " + + SHOULD_BE = { + "correct": "correct", + "incorrect": "incorrect", + "error": "error" + } + + def __init__( + self, + location, + should_be=SHOULD_BE['correct'], + response_dict=None, + verdict=NONE, + message=NONE, + override_state=None, + responses=None, + module=None, + id=0 + ): + """ + To instantiate a content_test, we use the following information: + - location -- locaiton of capa_problem() being tested + - should_be -- what the expected return of the grader is (Correct, Incorrect, etc.) + - response_dict -- dictionary to be turned into the grade + - verdict -- result of the test (pass, fail, not run, Error) + - message -- message about test (error message, etc) + - override_state -- dictionary of values to define the state of the lcp (seed, etc) + - module -- CapaModule + - id -- 'unique' number within the descriptor. + uniqueness requires using the view function `add_test_to_ + """ + self.location = location + self.should_be = should_be + self.response_dict = response_dict or {} + self.verdict = verdict + self.message = message + self.override_state = override_state or {'seed': 1} + self.module = module + self.id = id + + # list of children + # the None case is handled in self._create_children() + self.responses = responses + + # used to detect edits + self._old_resp_dict = self.response_dict + self._create_children() + self.rematch_if_necessary() + + @classmethod + def construct_preview_module(cls, location): + """ + construct a new preview capa module + """ + # For some reasone, it sometimes comes out as draft, and other times not. + # This is an issue if the form to create a test was generated with a different + # revision than what comes up the first time the ContentTest is instantiated (if + # revision changes after that, rematching will take care of things, but it needs + # to be instantiated once for anything to work), because the id's in the response_dict + # will not match anything in the CapaProblem. + + # This next line solves the issue outlined above as far as I can tell. + # For some reasone, it also appears to be unecessary. However, I am leaving it + # commented in for safety. Historically, one could optionally pass in a descriptor to + # this function, and than no call would be made to modulestore().get_item. This was + # purely for the purpose of minimizing database access. However, the descriptor loaded + # by preview.py would never result in a module with @draft. + # location = as_draft(location) + + # if descriptor is None: + descriptor = modulestore().get_item(Location(location)) + + preview_module = load_preview_module(str(0), descriptor) + + return preview_module + + @classmethod + def is_valid_dict(cls, test_dict): + """ + Returnes true if the dict `test_dict` is a valid dict to instantiate + a ContentTest from. + """ + keys = test_dict.keys() + + ATTRS = { + 'location': object, + 'should_be': basestring, + 'response_dict': dict, + 'verdict': basestring, + 'message': basestring, + 'override_state': dict, + 'id': int, + 'responses': list + } + + # we know there needs to be some information + if len(keys) < 1: + return False + + # it always needs location + if 'location' not in keys: + return False + + # make sure it is a valid location + try: + Location(test_dict['location']) + except: + return False + + # make sure optional attributes are of the right type + for key in keys: + if key not in ATTRS: + return False + + if not isinstance(test_dict[key], ATTRS[key]): + return False + + if key == 'responses': + for resp_dict in test_dict[key]: + if not ResponseTest.is_valid_dict(resp_dict): + return False + + return True + + def capa_problem(self): + """ + create the capa_problem(). + """ + + # create a LoncapaProblem with the right state + new_lcp_state = self.capa_module().get_state_for_lcp() # pylint: disable=E1103 + new_lcp_state.update(self.override_state) + lcp = self.capa_module().new_lcp(new_lcp_state) # pylint: disable=E1103 + + return lcp + + def capa_module(self): + """ + resturns a preview instance of the capa module pointed to by + self.location + """ + # fetch from mongo if we don't already have a module + if self.module is None: + self.module = ContentTest.construct_preview_module(self.location) + + return self.module + + def todict(self, *arg, **kwargs): + """ + Returns a dict describing this content test + """ + + # if we changing the dictionary, update the verdict to NONE + if self._old_resp_dict != self.response_dict: + self.verdict = self.NONE + + return { + 'location': self.location, + 'should_be': self.should_be, + 'response_dict': self.response_dict, + 'verdict': self.verdict, + 'message': self.message, + 'override_state': self.override_state, + 'responses': [resp_test.todict() for resp_test in self.responses], + 'id': self.id + } + + def run(self): + """ + run the test, and see if it passes + """ + + # process dictionary that is the response from grading + grade_dict = self._evaluate(self.response_dict) + + # compare the result with what is should be + self.verdict = self._make_verdict(grade_dict) + + # on error, the message gets set when that error is handled + # (in self._evaluate()) + if self.verdict == self.FAIL: + if self.should_be == self.SHOULD_BE["correct"]: + self.message = "%s: Input got evaluated as %s" % (self.verdict, self.SHOULD_BE["incorrect"]) + else: + self.message = "%s: Input got evaluated as %s" % (self.verdict, self.SHOULD_BE["correct"]) + elif self.verdict == self.PASS: + self.message = self.verdict + " :)" + + # write the change to the database and return the result + return self.verdict + + def rematch_if_necessary(self): + """ + Rematches itself to its problem if it no longer matches. + Reassigns hashes to response models if they no longer + match but the structure still does (so future matching + can happen). + """ + + if not self._still_matches(): + self._rematch() + else: + self._reassign_hashes_if_necessary() + +#======= Private Methods =======# + + def _still_matches(self): + """ + Returns true if the test still corresponds to the structure of the + problem + """ + + # if there are no longer the same number, not a match. + if not(len(self.responses) == len(self.capa_problem().responders)): # pylint: disable=E1101 + return False + + # loop through response models, and check that they match + all_match = True + for resp_model in self.responses: # pylint: disable=E1101 + if not resp_model.still_matches(): + all_match = False + break + + return all_match + + def _reassign_hashes_if_necessary(self): + """ + Iterate through the response models, and rematch their + xml_string and xml_hashes if they have changed in the capa problem + """ + + for response in self.responses: + response.rematch(response.capa_response) + + def _rematch(self): + """ + Corrects structure to reflect the state of the capa problem. + + The algorithm proceeds by looking at all possible parings from the two lists + and calculating the closeness value of the match. It then begins popping off + the closest matches and making the match if the clioseness is above some + threshold value (assuming that neither object has been involved + in any previous (and therefore better) matches). + """ + + # how desperate are we to make matches? + cutoff = 0.90 + + # copy lists + unmatched_models = list(self.responses) + unmatched_responders = list(self.capa_problem().responders.values()) + + # make a sorted list of triples of all possible matches and their closeness value + potential_matches = sorted([(closeness(model, responder), model, responder) for model in unmatched_models for responder in unmatched_responders]) + + while potential_matches: + match = potential_matches.pop() + + # interpret the tuple + percent_match = match[0] + model = match[1] + responder = match[2] + + # if it is not good enought of a match, just stop + if percent_match < cutoff: + break + + # only make the match if neither object has been matched + elif (model in unmatched_models) and (responder in unmatched_responders): + + # make the match and mark as used + model.rematch(responder) + unmatched_models.remove(model) + unmatched_responders.remove(responder) + + # delete unused models + for model in unmatched_models: + self.responses.remove(model) + + # create new models for unmatched responders + for responder in unmatched_responders: + self._create_child(responder) + + # remake dict + self._remake_dict_from_children() + + def _evaluate(self, response_dict): + """ + Give the capa_problem() the response dictionary and return the result + """ + + # instantiate the capa problem so it can grade itself + capa = self.capa_problem() + + try: + correct_map = capa.grade_answers(response_dict) + return correct_map.get_dict() + + # if there is any error, we just return None + except Exception as e: # pylint: disable=W0703 + + # log the error message + self.message = str(e) + return None + + def _make_verdict(self, grade_dict): + """ + compare what the result of the grading should be with the actual grading + and return the verdict + """ + + # if there was an error + if grade_dict is None: + # if we want error, return pass + if self.should_be == self.SHOULD_BE["error"]: + return self.PASS + return self.ERROR + + # see that they all are the expected value (if not blank) + passing = True + for response in self.responses: + passing = passing and (response.passing(self.should_be, grade_dict)) + + if passing: + return self.PASS + else: + return self.FAIL + + def _remake_dict_from_children(self): + """ + build the response dictionary by getting the values from the children + """ + + # refetch the answers from all the children + resp_dict = {} + for response in self.responses: + resp_dict.update(response.dict_slice()) + + # update the dictionary + self.response_dict = resp_dict + + def _create_children(self): + """ + create child responses and input entries + """ + + # the first time loaded, we make the responses from + # the capa moudle + if self.responses is None: + self.responses = [] + # create a preview capa problem + problem_capa = self.capa_problem() + + # go through responder objects + for responder in problem_capa.responders.itervalues(): + self._create_child(responder, self.response_dict) + + # if the dictionary was incomplete, we remake it so we + # have all the blank entries. + self._remake_dict_from_children() + + # else, we just instantiate from the saved versions + else: + self.responses = [ResponseTest(**dict(resp_test_dict, content_test=self, response_dict=self.response_dict)) for resp_test_dict in self.responses] + + def _create_child(self, responder, response_dict=dict()): + """ + from a responder object, create the associated child response model + """ + + self.responses.append(ResponseTest(self, responder.id, etree.tostring(responder.xml), response_dict)) + + +class ResponseTest(object): + """ + Object that corresponds to the <_____response> fields + """ + + @classmethod + def is_valid_dict(cls, test_dict): + """ + Returnes true if the dict `resp_test_dict` is a valid dict to instantiate + a ResponseTest from. + """ + + keys = test_dict.keys() + + ATTRS = { + 'string_id': basestring, + 'xml_string': basestring, + 'inputs': dict + } + + # we know there needs to be some information + if len(keys) < 2: + return False + + # it always needs string_id and xml_string + if 'string_id' not in keys or 'xml_string' not in keys: + return False + + # ensure valid xml + try: + etree.XML(test_dict['xml_string']) + except: + return False + + # make sure optional attributes are of the right type + for key in keys: + if key not in ATTRS: + return False + + if not isinstance(test_dict[key], ATTRS[key]): + return False + + return True + + def __init__(self, content_test, string_id, xml_string, response_dict={}, inputs=None): + """ + To instantiate the response sub-object of a contentTest, we use the following + - content_test -- parent object + - string_id -- id for the response + - xml_string -- string xml definition of the response + """ + + self.content_test = content_test + self.string_id = string_id + self.xml_string = xml_string + self.inputs = inputs + + # we store hashes of various properties about the xml for faster future processing + self.structure_hash = hash_xml_structure(etree.XML(self.xml_string)) + self.xml_hash = hash_xml(etree.XML(self.xml_string)) + + # store the inputs keyd by their order in this response + if self.inputs is None: + self.inputs = dict() + for entry in self.capa_response.inputfields: + answer = response_dict.get(entry.attrib['id'], '') + self.inputs[entry.attrib['answer_id']] = {'id': entry.attrib['id'], 'answer': answer} + + def rematch(self, responder): + """ + reassociates the ids with this new responder object. + If the hashes match, then all that needs + changing are the ids. If not, we recalculate hashes. + (It is assumed that structure_hash's match). + + Note: structure hash is never changed. + """ + + # if the ids and hashes match, we are done + if self.string_id == responder.id: + if self.xml_hash == hash_xml(responder.xml): + return + + # if just the hashes don't match, + # only update the response model + # (not the children) + else: + self.xml_string = etree.tostring(responder.xml) + self.xml_hash = hash_xml(responder.xml) + return + + # The id's don't match, so we re-associate them + self.string_id = responder.id + + # rematch xml if necessary + if self.xml_hash != hash_xml(responder.xml): + self.xml_string = etree.tostring(responder.xml) + self.xml_hash = hash_xml(responder.xml) + + # rematch all the childrens ids + for entry in responder.inputfields: + + # reassign the other ids + index = entry.attrib['answer_id'] + new_id = entry.attrib['id'] + self.inputs[index]['id'] = new_id + + @property + def capa_response(self): + """ + get the capa-response object to which this response model corresponds + """ + + parent_capa = self.content_test.capa_problem() # pylint: disable=E1101 + self_capa = parent_capa.responders_by_id[self.string_id] + + return self_capa + + def still_matches(self): + """ + check that the model has the same structure as corresponding responder object + """ + + try: + return self.structure_hash == hash_xml_structure(self.capa_response.xml) + except KeyError: + return False + + def dict_slice(self): + """ + Returns the slice of the total response_dict that this response contains. + """ + + dict_slice = {} + + for entry in self.inputs.values(): + dict_slice[entry['id']] = entry['answer'] + + return dict_slice + + def passing(self, should_be, grade_dict): + """ + For each response whose answer isn't blank, the grade in `grade_dict` + must be `should_be` + """ + + passes = True + for entry in self.inputs.values(): + if entry['answer'] != '': + passes = passes and (grade_dict[entry['id']]['correctness'].lower() == should_be.lower()) + + return passes + + def todict(self): + """ + Serializes object to dictionary + """ + + return { + "string_id": self.string_id, + "xml_string": self.xml_string, + "inputs": self.inputs, + } diff --git a/cms/djangoapps/content_testing/tests/__init__.py b/cms/djangoapps/content_testing/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/content_testing/tests/test_model.py b/cms/djangoapps/content_testing/tests/test_model.py new file mode 100644 index 000000000000..08a4d45f7f06 --- /dev/null +++ b/cms/djangoapps/content_testing/tests/test_model.py @@ -0,0 +1,869 @@ +""" +Unit tests on the models that make up automated content testing +""" + +from django.test import TestCase +from django.test.utils import override_settings +from django.conf import settings + +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, mongo_store_config +from xmodule.modulestore.django import modulestore +from content_testing.models import ContentTest, ResponseTest, hash_xml, hash_xml_structure, \ + condense_attributes, remove_xml_wrapper, condense_dict + +from capa.tests.response_xml_factory import CustomResponseXMLFactory + +from lxml import etree +from mock import patch +from textwrap import dedent + +# disable sillly pylint violations +# pylint: disable=W0212 +# pylint: disable=W0201 +MONGO_CONFIG = mongo_store_config(settings.COMMON_TEST_DATA_ROOT) + + +@override_settings(MODULESTORE=MONGO_CONFIG) +class ContentTestTestCase(ModuleStoreTestCase): + """ + set up a content test to test + """ + + SCRIPT = dedent(""" + def is_prime (n): + primality = True + for i in range(2,int(math.sqrt(n))+1): + if n%i == 0: + primality = False + break + return primality + + def test_prime(expect,ans): + a1=int(ans[0]) + a2=int(ans[1]) + return is_prime(a1) and is_prime(a2)""").strip() + NUM_INPUTS = 2 # tied to script + + HTML_SUMMARY = dedent(""" + + + + + + + + + + + +
+ Inputs: + + Should Be: + + Verdict: +
+
    +
  1. 5
  2. +
  3. 174440041
  4. +
+
+ correct + + - Not Yet Run - +
""").strip() + + VERDICT_PASS = "Pass" + VERDICT_FAIL = "Fail" + VERDICT_ERROR = ContentTest.ERROR + VERDICT_NONE = ContentTest.NONE + + def setUp(self): + """ + create all the tools to test content_tests + """ + + #course in which to put the problem + self.course = CourseFactory.create() + assert self.course + + # make the problem + problem_xml = CustomResponseXMLFactory().build_xml( + script=self.SCRIPT, + cfn='test_prime', + num_inputs=self.NUM_INPUTS) + + self.problem = ItemFactory.create( + parent_location=self.course.location, + data=problem_xml, + category='problem') + + # convert urls to ids + self.input_id_base = self.problem.id.replace('://', '-').replace('/', '-') + + # saved responses for making tests + self.response_dict_correct = { + self.input_id_base + '_2_1': '5', + self.input_id_base + '_2_2': '174440041' + } + self.response_dict_incorrect = { + self.input_id_base + '_2_1': '4', + self.input_id_base + '_2_2': '541098' + } + + self.response_dict_error = { + self.input_id_base + '_2_1': 'anyone lived', + self.input_id_base + '_2_2': 'in a pretty how town' + } + assert self.problem + + # Make a collection of ContentTests to test + self.pass_correct = ContentTest( + location=self.problem.location, + should_be='correct', + response_dict=self.response_dict_correct + ) + + self.pass_incorrect = ContentTest( + location=self.problem.location, + should_be='incorrect', + response_dict=self.response_dict_incorrect + ) + + self.fail_correct = ContentTest( + location=self.problem.location, + should_be='incorrect', + response_dict=self.response_dict_correct + ) + + self.fail_incorrect = ContentTest( + location=self.problem.location, + should_be='correct', + response_dict=self.response_dict_incorrect + ) + + self.fail_error = ContentTest( + location=self.problem.location, + should_be='correct', + response_dict=self.response_dict_error + ) + + self.pass_error = ContentTest( + location=self.problem.location, + should_be="error", + response_dict=self.response_dict_error) + + +class WhiteBoxTestCase(ContentTestTestCase): + """ + test that inner methods are working + """ + + def test_make_capa(self): + """ + test that the capa instantiation happens properly + """ + test_model = ContentTest( + location=self.problem.location, + should_be='Correct') + + capa = test_model.capa_problem() + + #assert no error + assert self.SCRIPT in capa.problem_text + + def test_create_children(self): + """ + test that the ContentTest is created with the right structure + """ + + test_model = ContentTest( + location=str(self.problem.location), + should_be='Correct') + + #check that the response created properly + responses = test_model.responses + self.assertEqual(len(responses), 1) + + #and the input + self.assertEqual(len(responses[0].inputs), self.NUM_INPUTS) + + def test_create_dictionary(self): + """ + tests the constructions of the response dictionary + """ + + test_model = ContentTest( + location=self.problem.location, + should_be='Correct', + response_dict=self.response_dict_correct + ) + + created_dict = test_model.response_dict + + self.assertEqual(self.response_dict_correct, created_dict) + + def test_remake_dict(self): + """ + tests the internal functionality of remaking the dictionary through the children + """ + test_model = self.pass_correct + + # delete the dict attribute + del test_model.response_dict + + #remake the attribute + test_model._remake_dict_from_children() + + # make sure they match + self.assertEqual(self.response_dict_correct, test_model.response_dict) + + +class MakeVerdictTestCase(ContentTestTestCase): + """ + a few tests for the _make_verdict method + """ + + def setUp(self): + super(MakeVerdictTestCase, self).setUp() + response_dict_with_blank = { + self.input_id_base + '_2_1': '', + self.input_id_base + '_2_2': '174440041' + } + + self.pass_correct_with_blank = ContentTest( + location=self.problem.location, + should_be='Correct', + response_dict=response_dict_with_blank + ) + + self.mockup_correctmap_mixed = { + self.input_id_base + '_2_1': {'correctness': 'incorrect'}, + self.input_id_base + '_2_2': {'correctness': 'correct'} + } + + def test_pass_with_blank(self): + """ + tests that blank entries are ignored + """ + + verdict = self.pass_correct_with_blank._make_verdict(self.mockup_correctmap_mixed) + self.assertEqual(verdict, self.VERDICT_PASS) + + def test_mixed_fails_correct(self): + """ + test that a mixed dictionary is not correct + """ + + test_model = self.pass_correct + verdict = test_model._make_verdict(self.mockup_correctmap_mixed) + self.assertEqual(verdict, self.VERDICT_FAIL) + + def test_mixed_fails_incorrect(self): + """ + test that a mixed dictionary is not incorrect + """ + + test_model = self.pass_incorrect + verdict = test_model._make_verdict(self.mockup_correctmap_mixed) + self.assertEqual(verdict, self.VERDICT_FAIL) + + def test_mixed_fails_error(self): + """ + test that a mixed dictionary is not error + """ + + test_model = self.pass_error + verdict = test_model._make_verdict(self.mockup_correctmap_mixed) + self.assertEqual(verdict, self.VERDICT_FAIL) + + +class BlackBoxTestCase(ContentTestTestCase): + """ + test overall behavior of the ContentTest model + """ + + def test_pass_correct(self): + """ + test that it passes with correct answers when it should + """ + + # run the test + self.pass_correct.run() + + # make sure it passed + self.assertEqual(self.VERDICT_PASS, self.pass_correct.verdict) + + def test_fail_incorrect(self): + """ + test that it fails with incorrect answers + """ + + # run the testcase + self.fail_incorrect.run() + + # make sure it failed + self.assertEqual(self.VERDICT_FAIL, self.fail_incorrect.verdict) + assert 'incorrect' in self.fail_incorrect.message + + def test_pass_incorrect(self): + """test that it passes with incorrect""" + + # run the test + self.pass_incorrect.run() + + # make sure it passed + self.assertEqual(self.VERDICT_PASS, self.pass_incorrect.verdict) + + def test_fail_correct(self): + """test that it fails with correct answers""" + + # run the testcase + self.fail_correct.run() + + # make sure it failed + self.assertEqual(self.VERDICT_FAIL, self.fail_correct.verdict) + assert 'correct' in self.fail_correct.message + + def test_pass_error(self): + """ + test that we get a pass when it expects and gets an error + """ + # run the testcae + self.pass_error.run() + + # make sure it passed + self.assertEqual(self.VERDICT_PASS, self.pass_error.verdict) + + def test_fail_error(self): + """ + Test that a badly formatted dictionary results in error + """ + + test_model = self.fail_error + test_model.run() + + self.assertEqual(self.VERDICT_ERROR, test_model.verdict) + + def test_reset_verdict(self): + """test that changing things resets the verdict""" + + test_model = self.pass_correct + + # run the testcase (generates verdict) + test_model.run() + + # update test + test_model.response_dict = self.response_dict_incorrect + test_model.todict() + + #ensure that verdict is now null + self.assertEqual(self.VERDICT_NONE, test_model.verdict) + + def test_change_dict(self): + """test that the verdict changes with the new dictionary on new run""" + + test_model = self.pass_correct + + # update test + test_model.response_dict = self.response_dict_incorrect + test_model.todict() + + # run the test + test_model.run() + + # assert that the verdict is now self.VERDICT_FAIL + self.assertEqual(self.VERDICT_FAIL, test_model.verdict) + + def test_todict_idempotent(self): + """ + tests that we get the same object after instantiating from dict + """ + test_dict = self.pass_correct.todict() + new_test = ContentTest(**test_dict) + + self.assertEqual(new_test.todict(), self.pass_correct.todict()) + + @patch('content_testing.models.ContentTest.capa_problem') + @patch('content_testing.models.ContentTest.rematch_if_necessary') + def test_instantiate_from_todict(self, capa_problem, rematch_if_necessary): + """ + test that other than the structure matching (which always will + require fetching the capa somehow), no capa is used for instantiation + from saved dict. This test should fail if .capa_problem() is ever + called, not result in error. + """ + test_dict = self.pass_correct.todict() + new_test = ContentTest(**test_dict) + + assert not (new_test.capa_problem.called) + + def test_partial_dict(self): + """ + test that a model instantiated with a incomplete dict will + fill in the remaining values with blanks + """ + + self.response_dict_correct.popitem() + incomplete_dict = self.response_dict_correct + incomplete_test = ContentTest( + location=self.problem.location, + response_dict=incomplete_dict + ) + + assert '' in incomplete_test.response_dict.values() + + +class RematchingTestCase(ContentTestTestCase): + """ + tests the ability to rematch itself to an edited problem + """ + + def setUp(self): + """ + create new sructure to test smart restructuring capabilities + """ + + super(RematchingTestCase, self).setUp() + + self.new_xml = CustomResponseXMLFactory().build_xml( + script=self.SCRIPT, + cfn='test_prime', + num_inputs=self.NUM_INPUTS + 1) + + self.new_problem = ItemFactory.create( + parent_location=self.course.location, + data=self.new_xml, + category='problem') + + self.test_model = self.pass_correct + + def update_problem_xml(self, new_xml_string): + """ + update the problem xml and do the other acrobatics to update everything + consistantly + """ + + # update the problem + modulestore().update_item(self.problem.location, new_xml_string) + + # this gets rid of the _draft nonsense, which makes hard-coded dicts easier. + # modulestore().publish(self.problem.location, 0) + + # force ContentTest to refetch module + # If we just set .module=None, then we force the ContentTest object + # to refetch, and thus effectively test the rematching capabilities. + # Hoerver, this only tests for when the ContentTest isn't being reloaded + # from the database, which would be most of the time. Thus, we test with + # both. + self.test_model2 = ContentTest(**self.test_model.todict()) + self.test_model.module = None + + def test_matches(self): + """ + test that the model knows when it still matches the problem + """ + + assert self.pass_correct._still_matches() + + def test_not_matches_new_xml(self): + """ + test that when the xml of the capa problem gets updated + the model knows + """ + + # change the problem by adding another textline + self.update_problem_xml(self.new_xml) + + assert not(self.test_model._still_matches()) + + def test_new_dict_blank(self): + """ + test rebuilding the dictionary with a different response + """ + + # the dictioanry, after fixing, should have blank answers + new_dict = { + self.input_id_base + '_2_1': '', + self.input_id_base + '_2_2': '', + self.input_id_base + '_2_3': '' + } + + # change the problem by adding another textline + self.update_problem_xml(self.new_xml) + + self.test_model.rematch_if_necessary() + # self.assertEqual(new_dict, self.test_model.response_dict) + self.assertEqual(new_dict, self.test_model2.response_dict) + + def test_append(self): + """ + test adding a new response at the end and then rebuilding + """ + + # add a response at the end + new_response_xml = etree.XML("") + new_xml = self.pass_correct.capa_problem().tree + new_xml.append(new_response_xml) + new_xml_string = etree.tostring(new_xml) + # the response dict should look like + two_responses_dict = { + self.input_id_base + '_3_1': '', + self.input_id_base + '_3_2': '', + self.input_id_base + '_3_3': '' + } + two_responses_dict.update(self.response_dict_correct) + + # change the problem by adding another response at end + self.update_problem_xml(new_xml_string) + self.test_model.rematch_if_necessary() + self.assertEqual(two_responses_dict, self.test_model.response_dict) + self.assertEqual(two_responses_dict, self.test_model2.response_dict) + + def test_insert(self): + """ + adding response at beginning of problem + """ + + new_xml_string = dedent(""" + + + + +

Enter a prime number

+ + + + + + + + + +
""") + + # the response dict should look like + two_responses_dict = { + self.input_id_base + '_3_1': '5', + self.input_id_base + '_3_2': '174440041', + self.input_id_base + '_2_1': '', + self.input_id_base + '_2_2': '', + self.input_id_base + '_2_3': '' + } + + # change the problem by adding another response at end + self.update_problem_xml(new_xml_string) + + self.test_model.rematch_if_necessary() + self.assertEqual(two_responses_dict, self.test_model.response_dict) + + def test_change_attributes(self): + """ + test that changing the attributes of the mxl doesn't cuase any net restructuring + """ + + # add attribute values + test_model = self.pass_correct + xml = test_model.capa_problem().tree + for child in xml: + child.attrib['samba'] = 'deamon' + + # save these to the capa_problem + self.update_problem_xml(etree.tostring(xml)) + + # make sure that no restructuring happens + self.test_model.rematch_if_necessary() + self.assertEqual(self.response_dict_correct, self.test_model.response_dict) + + def test_delete_response(self): + """ + test removing responses that no longer match any in the problem + (changing problem location accomplishes this) + """ + + # change location on the test + test_model = self.pass_correct + test_model.location = self.new_problem.location + # force it to refetch from mongo + test_model.module = None + + # make it rematch itself + test_model.rematch_if_necessary() + + # assert that the new dictionary has no values + self.assertEqual(["", "", ""], test_model.response_dict.values()) + + def test_fuzzy_rematching_insert(self): + """ + test matching capabilities when things are slightly off + """ + + new_xml_string = dedent(""" + + + + +

Enter a prime number

+ + + + + + + + + +
""") + + # the response dict should look like + two_responses_dict = { + self.input_id_base + '_3_1': '5', + self.input_id_base + '_3_2': '174440041', + self.input_id_base + '_2_1': '', + self.input_id_base + '_2_2': '', + self.input_id_base + '_2_3': '' + } + + # change the problem by adding another response at end + self.update_problem_xml(new_xml_string) + self.test_model.rematch_if_necessary() + self.assertEqual(two_responses_dict, self.test_model.response_dict) + + +class HelperFunctionsTestCase(TestCase): + """ + tests for the xml helper functions + """ + + def test_hash_xml_same(self): + """ + test that the hash function ignors the right things + """ + + xml1 = etree.XML("") + xml2 = etree.XML("") + + self.assertEqual(hash_xml(xml1), hash_xml(xml2)) + + def test_hash_xml_different(self): + """ + test that the hash function includes the right things + """ + + xml1 = etree.XML("") + xml2 = etree.XML("") + + self.assertNotEqual(hash_xml(xml1), hash_xml(xml2)) + + def test_hash_xml_structure_same(self): + """ + Test that structure hash ignores all attributes + """ + + xml1 = etree.XML("") + xml2 = etree.XML("") + + self.assertEqual(hash_xml_structure(xml1), hash_xml_structure(xml2)) + + def test_hash_xml_structure_different(self): + """ + Test that structure hash ignores all attributes + """ + + xml1 = etree.XML("") + xml2 = etree.XML("") + + self.assertNotEqual(hash_xml_structure(xml1), hash_xml_structure(xml2)) + + def test_remove_wrapper_xml(self): + """ + test the function used to strip out forms with lxml input + """ + + xml1 = etree.XML("") + xml2 = etree.XML("") + + processed_xml = remove_xml_wrapper(xml1, 'a') + + self.assertEqual(etree.tostring(xml2), etree.tostring(processed_xml)) + + def test_remove_wrapper_string(self): + """ + test the function used to strip out forms with text input + """ + + xml1 = "" + xml2 = "" + + processed_xml = remove_xml_wrapper(xml1, 'a') + + self.assertEqual(xml2, processed_xml) + + def test_condense_attributes(self): + """ + tests a helper function for recursively generating a attribute dictionary + """ + + xml = etree.XML(""" + + """) + + dictionary = {'cfn': 'test_csv', 'expect': '0, 1, 2, 3, 3', 'correct_answer': '0, 1, 2, 3, 3'} + + self.assertEqual(dictionary, condense_attributes(xml)) + + def test_condense_dict(self): + """ + test helper function for squashing a dictionary + """ + + to_squash = {'all in green ': 'went my love riding'} + squashed = 'all in green went my love riding' + + self.assertEqual(condense_dict(to_squash), squashed) + + +class ValidateDictTestCase(ContentTestTestCase): + """ + Test the validate dictionary methods of the ContentTest and ResponseTest objects + """ + + def test_todict_is_valid(self): + """ + Tests that the class method for checking valid dicts returns True + on the todict() + """ + test_dict = self.pass_correct.todict() + assert ContentTest.is_valid_dict(test_dict) + + def test_blank_ContentTest(self): + """ + Tests that blank dicts evaluate as invalid + """ + + assert not(ContentTest.is_valid_dict({})) + + def test_missing_loc_ContentTest(self): + """ + All good, but missing location + """ + + test_dict = self.pass_correct.todict() + test_dict.pop('location') + + assert not(ContentTest.is_valid_dict(test_dict)) + + def test_bad_location_ContentTest(self): + """ + Assert that it fails on a bad location + """ + + test_dict = self.pass_correct.todict() + test_dict['location'] = 'trolololol' + + assert not(ContentTest.is_valid_dict(test_dict)) + + def test_extra_kwarg_ContentTest(self): + """ + Assert that it fails on a bad location + """ + + test_dict = self.pass_correct.todict() + test_dict['new_field'] = 'trolololol' + + assert not(ContentTest.is_valid_dict(test_dict)) + + def test_bad_type_ContentTest(self): + """ + Assert that wrong types makes for invalid dicts + """ + + test_dict = self.pass_correct.todict() + test_dict['should_be'] = 3 + + assert not(ContentTest.is_valid_dict(test_dict)) + + def test_bad_response_subdicts_ContentTest(self): + """ + Assert that bad responseTest dicts cause it to fail + """ + + test_dict = self.pass_correct.todict() + test_dict['responses'][0] = {} + + assert not(ContentTest.is_valid_dict(test_dict)) + + def valid_resptest_dict(self): + """ + return a valid dict for ResponseTests + """ + + return { + 'string_id': 'this is an id :)', + 'xml_string': '', + 'inputs': {'1': {'id': 'id_string', 'answer': '42'}} + } + + def test_empty_dict_ResponsTest(self): + """ + Assert that empty dicts are invalid + """ + + assert not(ResponseTest.is_valid_dict({})) + + def test_lacking_id_ResponseTest(self): + + test_dict = self.valid_resptest_dict() + test_dict.pop('string_id') + + assert not(ResponseTest.is_valid_dict(test_dict)) + + def test_lacking_xml_ResponseTest(self): + + test_dict = self.valid_resptest_dict() + test_dict.pop('xml_string') + + assert not(ResponseTest.is_valid_dict(test_dict)) + + def test_broken_xml_ResponseTest(self): + + test_dict = self.valid_resptest_dict() + test_dict['xml_string'] = '' + + assert not(ResponseTest.is_valid_dict(test_dict)) + + def test_bad_type_ResponseTest(self): + + test_dict = self.valid_resptest_dict() + test_dict['inputs'] = [1, 2, 3] + + assert not(ResponseTest.is_valid_dict(test_dict)) diff --git a/cms/djangoapps/content_testing/tests/test_views.py b/cms/djangoapps/content_testing/tests/test_views.py new file mode 100644 index 000000000000..7e297b165d1d --- /dev/null +++ b/cms/djangoapps/content_testing/tests/test_views.py @@ -0,0 +1,302 @@ +""" +Tests for the views involved in content testing. +""" + +from contentstore.tests.test_course_settings import CourseTestCase +from xmodule.modulestore.tests.factories import ItemFactory +from capa.tests.response_xml_factory import CustomResponseXMLFactory +from content_testing.models import ContentTest +from content_testing.views import getprompt, add_contenttest_to_descriptor, delete_contenttest +from xmodule.modulestore.django import modulestore +from textwrap import dedent +from lxml import etree +from django.http import Http404 + + +# disable sillly pylint violations +# pylint: disable=W0212 +# pylint: disable=W0201 +class ContentTestViewTestCase (CourseTestCase): + """ + Tests for the views involved in the automated content testing + """ + + SCRIPT = dedent(""" + def is_prime (n): + primality = True + for i in range(2,int(math.sqrt(n))+1): + if n%i == 0: + primality = False + break + return primality + + def test_prime(expect,ans): + a1=int(ans) + return is_prime(a1)""").strip() + + def setUp(self): + """ + override parent setUp to put a problem in that course + """ + + super(ContentTestViewTestCase, self).setUp() + + #change the script if 1 + problem_xml = CustomResponseXMLFactory().build_xml( + script=self.SCRIPT, + cfn='test_prime') + + self.problem = ItemFactory.create( + parent_location=self.course.location, + data=problem_xml, + category='problem') + + # format as if it came from the form generated by the capa_problem + # sigh + self.input_id_base = self.problem.id.replace('://', '-').replace('/', '-') + + self.url = "/test_problem/" + self.loc = self.problem.location.url() + + # add a @draft thing, so it doesn't change to that half way through + modulestore().update_metadata(self.loc, {'tests': []}) + + def create_model(self): + """ + helper method to add a content test to the database + """ + + # saved responses for making tests + self.response_dict_correct = { + self.input_id_base + '-draft_2_1': '174440041' + } + + self.response_dict_incorrect = { + self.input_id_base + '-draft_2_1': '6' + } + + self.pass_correct = ContentTest( + location=self.loc, + should_be='correct', + response_dict=self.response_dict_correct + ) + + # save the test + modulestore().update_metadata(self.loc, {'tests': [self.pass_correct.todict()]}) + + # update self.problem to reflect the change + self.problem = modulestore().get_item(self.problem.location) + + def check_no_contenttests(self): + """ + check that there are no tests in this summary + """ + + descriptor = modulestore().get_item(self.problem.location) + self.assertEqual(len(descriptor.tests), 0) + + def check_exist_contenttest(self): + """ + check that there are tests in the summary view + """ + + descriptor = modulestore().get_item(self.problem.location) + self.assertEqual(len(descriptor.tests), 1) + + # also, chack that it was saved "fully", with all the data that gets + # calculated on instantiation (necessary for future rematching) + for test in descriptor.tests: + assert 'responses' in test + + + +class ContentTestDispatchTestCase(ContentTestViewTestCase): + + def test_no_tests(self): + """ + test that initially there are no tests for the problem + """ + + response = self.client.get(self.url, {'location': self.loc}) + self.check_no_contenttests() + self.assertEqual(response.status_code, 200) + + def test_create(self): + """ + test that saving a new test works + """ + + # format the response that the capa problem generates + input_id = 'input_' + self.input_id_base + '_2_1' + post_data = { + 'location': self.loc, + 'should_be': 'correct', + input_id: '5' + } + + response = self.client.post(self.url, post_data, follow=True) + + # ceck that there is now one test. + self.check_exist_contenttest() + self.assertEqual(response.status_code, 200) + + def test_delete(self): + """ + test that the delete works + """ + + self.create_model() + model_id = self.problem.tests[0]['id'] + + req_data = { + 'id_to_delete': model_id, + 'location': self.loc + } + + response = self.client.delete(self.url, req_data) + + self.check_no_contenttests() + self.assertEqual(response.status_code, 200) + + def test_summary(self): + """ + Test that the main render is done properly + """ + + self.create_model() + + response = self.client.get(self.url, {'location': self.loc}) + + assert ContentTest.NONE.lower() in response.content.lower() + + def test_run(self): + """ + Test running tests + """ + + self.create_model() + + # run the test, and see that result is persistant + response1 = self.client.post(self.url, {'location': self.loc, 'run': 'yup'}) + response2 = self.client.get(self.url, {'location': self.loc}) + + assert ContentTest.PASS.lower() in response2.content.lower() + self.assertEqual(response1.status_code, 200) + + def test_404_without_location(self): + """ + Test raises 404 when not passed location + """ + + response = self.client.post(self.url) + self.assertEqual(response.status_code, 404) + + +class HelpFuncTestCase(ContentTestViewTestCase): + + def test_getprompt_parent(self): + + xml_string = dedent(""" + + + + + + + """) + + xml = etree.XML(xml_string) + intro = xml[0] + blob = xml[1][0] + + self.assertEqual(getprompt(blob), intro) + + def test_getprompt_sib(self): + + xml_string = dedent(""" + + + + + + + """) + + xml = etree.XML(xml_string) + intro = xml[0] + child = xml[1] + + self.assertEqual(getprompt(child), intro) + + def test_getprompt_none(self): + + xml_string = dedent(""" + + + + + + """) + + xml = etree.XML(xml_string) + blob = xml[0][0] + + self.assertEqual(getprompt(blob), None) + + def test_add_to_descriptor(self): + """ + Test the function for incrementing id's. + """ + self.create_model() + + # add a few tests + num_tests = 5 + for i in range(num_tests-1): + add_contenttest_to_descriptor(self.problem, self.pass_correct.todict()) + + # refetch problem form database to see changes + problem = modulestore().get_item(self.problem.location) + + test_ids = [test['id'] for test in problem.tests] + + # asserts that there are `num_tests`, and each has different ids + self.assertEqual(len(problem.tests), num_tests) + assert len(test_ids) == len(set(test_ids)) + + def test_delete_contenttest(self): + """ + Test deleteing tests by id works + """ + self.create_model() + + # add a few tests + num_tests = 5 + for i in range(num_tests-1): + add_contenttest_to_descriptor(self.problem, self.pass_correct.todict()) + + # delete the test + delete_contenttest(self.problem, self.problem.tests[0]['id']) + + # re-fetch descriptor + self.problem = modulestore().get_item(self.problem.location) + + self.assertEqual(len(self.problem.tests), num_tests-1) + + def test_delete_404_no_tests(self): + """ + Test that delete complains when deleting tests when there are no tests + """ + + # delete the test + self.assertRaises(Http404, delete_contenttest, self.problem, 0) + + def test_delete_404_exist_tests(self): + """ + Test that when there are tests, deleting a test that doesn't exits + cuases error + """ + + self.create_model() + + # delete the test + self.assertRaises(Http404, delete_contenttest, self.problem, 3) diff --git a/cms/djangoapps/content_testing/views.py b/cms/djangoapps/content_testing/views.py new file mode 100644 index 000000000000..8f3f6dedabd0 --- /dev/null +++ b/cms/djangoapps/content_testing/views.py @@ -0,0 +1,332 @@ +""" +Views for using the atuomated content testing feature. These views should be considered +a mockup for demonstrating the usage of the ContentTest objects. +""" + +from django.http import HttpResponse, Http404 +from xmodule.modulestore.django import modulestore +from content_testing.models import ContentTest + +# csrf utilities because mako. +# Any form submitted with the post methodh should have a hidden input where +# name="csrfmiddlewaretoken" and value = csrf(request)['csrf_token'] +# This is not necesary if the form is only used to populate the data of an AJAX call. +from django_future.csrf import ensure_csrf_cookie +from django.core.context_processors import csrf +from django.contrib.auth.decorators import login_required +from django.core.exceptions import PermissionDenied +from contentstore.views.access import has_access + +from mitxmako.shortcuts import render_to_string +from lxml import etree +from copy import deepcopy + +# supported response types +RESPONSE_TYPES = ['customresponse'] + + +def dict_slice(data, string): + """ + Returns dict of that keys that start with "string" (and removing "string" from the keys) + + This function is used to pull a valid response_dict out of POST data. + """ + + # this can be done beautifully in one line, but pylint complains :( + d_slice = {} + for key, value in data.iteritems(): + if key.startswith(string): + d_slice.update([(key.replace(string, ''), value)]) + return d_slice + + +@login_required +@ensure_csrf_cookie +def problem_test(request): + """ + Main routing function for content_testing (and the only one accessible by + url). + + method == GET -- Returns summary of tests for this problem, grouped by + `should_be` values. + + method == POST -- If `run` is a key in the post data that maps to anything (including + the empty string), it runs the tests and saves the result. + + Else, it assumes that we are creating a new test, and assumes that the requisite data + for creating a ContentTest is in the POST datatest to list of tests. + + If an index is amung the post data, it replaces the test at that index instead + of creating a new one. + + method == DELETE -- Removes test at index from the list of tests. Index is given in + in the DELETE data. + """ + try: + location = request.GET['location'] + except KeyError: + try: + location = request.POST['location'] + except KeyError: + raise Http404 + + # check that logged in user has permissions to this item + if not has_access(request.user, location): + raise PermissionDenied() + + # fetch descriptor once + descriptor = modulestore().get_item(location) + + if request.method == 'GET': + html = testing_summary(request, descriptor) + return HttpResponse('{"html": '+html+'}') + + elif request.method == "POST": + run = request.POST.get('run', None) + if run is not None: + + # instantiate + tests = instantiate_tests(descriptor) + + # run the tests + for test in tests: + test.run() + + # update database + modulestore().update_metadata(location, {'tests': [test.todict() for test in tests]}) + + else: + response_dict = dict_slice(request.POST, 'input_') + should_be = request.POST['should_be'] + + # don't save a blank test + if any(response_dict.values()): + + # instantiate new test + new_test = { + 'location': location, + 'response_dict': response_dict, + 'should_be': should_be + } + # save + add_contenttest_to_descriptor(descriptor, new_test) + + return HttpResponse('') + + elif request.method == "DELETE": + + # get necessary data from GET data + id_to_delete = int(request.GET['id_to_delete']) + + # try to delete + delete_contenttest(descriptor, id_to_delete) + + return HttpResponse('') + + +def delete_contenttest(descriptor, id_to_delete): + """ + Deletes any content test with id `id_to_delete` from the descriptor. If one + with that id is not found, raises 404. + """ + + number_tests = len(descriptor.tests) + new_tests = [test for test in descriptor.tests if not (test['id'] == id_to_delete)] + + # if we didn't find anything, raise 404 + if (len(new_tests) >= number_tests) or (number_tests == 0): + raise Http404 + + # save changes to database + else: + modulestore().update_metadata(descriptor.location, {'tests': new_tests}) + + +def add_contenttest_to_descriptor(descriptor, test_dict): + """ + Adds the test to the descriptor, and creates an id for the test. + """ + + tests = descriptor.tests + + # give it an ID one more than the previous one, if there are + # previou tests + if tests: + new_id = tests[-1]['id']+1 + else: + new_id = 0 + + test_dict['id'] = new_id + + # we need to instantiate before saving so rematching will work + test_dict = ContentTest(**test_dict).todict() + + # add it to the descriptor + tests.append(test_dict) + + #save it to the database + modulestore().update_metadata(descriptor.location, {'tests': tests}) + + +def instantiate_tests(descriptor): + """ + Instantiate the tests of the descriptor such that no db access is needed per test. + + TODO: Validate the elements of test_dicts with ContentTest.is_valid_dict(dict) + """ + + test_dicts = descriptor.tests + + # instantiate preview module (so each test doesn't need to indevidually) + module = ContentTest.construct_preview_module(descriptor.location) + tests = [ContentTest(**dict(test_dict, module=module)) for test_dict in test_dicts] + + return tests + + +def testing_summary(request, descriptor): + """ + Render the testing summary for this descriptor + """ + tests = instantiate_tests(descriptor) + # sort tests by should_be value. + # The dictionary contains a key for every available `should_be`, and tests are just + # appended to the value of that key. + + # sorted_tests = {value: [] for value in ContentTest.SHOULD_BE} + sorted_tests = {} + for value in ContentTest.SHOULD_BE.values(): + sorted_tests[value] = [] + for test in tests: + sorted_tests[test.should_be].append(test) + + # for each `should_be` we generate a summary of the tests + test_summaries = {} + for should_be in sorted_tests: + test_summaries[should_be] = render_test_group(request, descriptor, sorted_tests[should_be], should_be) + + # render summaries + context = { + 'csrf': csrf(request)['csrf_token'], + 'summaries': test_summaries, + 'location': descriptor.location, + } + + return render_to_string('content_testing/test_summaries.html', context) + + +def render_test_group(request, descriptor, tests, should_be): + """ + Render the problem with sections left for the test summaries + """ + + HTML_SIG = '_test_summary' + + # make the lcp + lcp = ContentTest.construct_preview_module(descriptor.location).lcp + + # generate the summaries for each response + response_summaries = {} + for responder in lcp.responders.values(): + response_summaries[responder.id] = aggregate_response_summaries(responder.id, tests) + + section = etree.Element('section') + tree = deepcopy(lcp.tree) + + for resp in tree.xpath('//' + "|//".join(RESPONSE_TYPES)): + resp_id = resp.attrib['id'] + + # first add what is probably the prompt for the question + prompt = getprompt(resp) + if prompt is not None: + section.append(prompt) + + # construct the summary container + div = etree.Element('div') + div.set('class', "response-test-wrapper") + div.set('id', resp_id+HTML_SIG+'_'+should_be) + div.extend(response_summaries[resp.attrib['id']]) + + # append the create-new form + form = creation_form(request, descriptor.location, should_be) + div.append(form) + form.insert(0, resp) + + section.append(div) + + # use the lcp to render the xml we generated + lcp.tree = section + return lcp.get_html() + + +def getprompt(xml): + """ + Given an xml object, try to get the xml that looks like it came directly before. + """ + + # try just getting the previous + prompt = xml.getprevious() + + # if that's none, try the parent (unless that's none too) + if prompt is None: + parent = xml.getparent() + if parent is not None: + return getprompt(parent) + + return prompt + + +def aggregate_response_summaries(response_id, tests): + """ + For tests in the list `tests`, gets all response + summaries for the response id, and returns the concatenated html summaries. + """ + + # not efficiently searchable :( + # maybe I should store as dict by id... + summary = [] + for test in tests: + for response in test.responses: + if response.string_id == response_id: + + # only append if the summary is not None + if response_summary(response, test.message) is not None: + summary.append(response_summary(response, test.message)) + + return summary + + +def response_summary(response_test, msg): + """ + Given response test (child of ContentTest) and overall message of the test, + returns xml summary + """ + + # sorted list of answers + answers = [response_test.inputs[index]['answer'] for index in sorted(response_test.inputs)] + + if any(answers): + context = { + 'msg': msg, + 'answers': answers, + 'id': response_test.content_test.id, + 'location': response_test.content_test.location + } + + return etree.XML(render_to_string('content_testing/response_summary.html', context)) + + +def creation_form(request, location, should_be): + """ + Retruns xml form object for creating a new test + (this needs to be wrapped around the actual rendering of the + responder object) + """ + + context = { + 'csrf': csrf(request)['csrf_token'], + 'location': location, + 'should_be': should_be + } + + return etree.XML(render_to_string('content_testing/test_form.html', context)) diff --git a/cms/djangoapps/contentstore/views/preview.py b/cms/djangoapps/contentstore/views/preview.py index 7a3a224d8686..12b5d5e24fb8 100644 --- a/cms/djangoapps/contentstore/views/preview.py +++ b/cms/djangoapps/contentstore/views/preview.py @@ -20,7 +20,7 @@ from util.sandboxing import can_execute_unsafe_code import static_replace -from .session_kv_store import SessionKeyValueStore +from .session_kv_store import SessionKeyValueStore, StaticPreviewKeyValueStore from .requests import render_from_lms from .access import has_access from ..utils import get_course_for_item @@ -43,7 +43,7 @@ def preview_dispatch(request, preview_id, location, dispatch=None): """ descriptor = modulestore().get_item(location) - instance = load_preview_module(request, preview_id, descriptor) + instance = load_preview_module(preview_id, descriptor, request) # Let the module handle the AJAX try: ajax_return = instance.handle_ajax(dispatch, request.POST) @@ -82,12 +82,12 @@ def preview_component(request, location): ) return render_to_response('component.html', { - 'preview': get_preview_html(request, component, 0), + 'preview': get_preview_html(component, 0, request), 'editor': component.runtime.render(component, None, 'studio_view').content, }) -def preview_module_system(request, preview_id, descriptor): +def preview_module_system(preview_id, descriptor, request=None): """ Returns a ModuleSystem for the specified descriptor that is specialized for rendering module previews. @@ -97,10 +97,18 @@ def preview_module_system(request, preview_id, descriptor): descriptor: An XModuleDescriptor """ + if request is not None: + kvs = SessionKeyValueStore(request, descriptor._model_data) + user = request.user + + else: + kvs = StaticPreviewKeyValueStore(descriptor._model_data) + user = None + def preview_model_data(descriptor): "Helper method to create a DbModel from a descriptor" return DbModel( - SessionKeyValueStore(request, descriptor._model_data), + kvs, descriptor.module_class, preview_id, MongoUsage(preview_id, descriptor.location.url()), @@ -113,25 +121,26 @@ def preview_model_data(descriptor): # TODO (cpennington): Do we want to track how instructors are using the preview problems? track_function=lambda event_type, event: None, filestore=descriptor.system.resources_fs, - get_module=partial(load_preview_module, request, preview_id), + get_module=partial(load_preview_module, preview_id, request=request), render_template=render_from_lms, debug=True, replace_urls=partial(static_replace.replace_static_urls, data_directory=None, course_id=course_id), - user=request.user, + user=user, xblock_model_data=preview_model_data, can_execute_unsafe_code=(lambda: can_execute_unsafe_code(course_id)), ) -def load_preview_module(request, preview_id, descriptor): +def load_preview_module(preview_id, descriptor, request=None): """ - Return a preview XModule instantiated from the supplied descriptor. + Returns a preview XModule at the specified location. The preview_data is chosen arbitrarily + from the set of preview data for the descriptor specified by Location request: The active django request preview_id (str): An identifier specifying which preview this module is used for - descriptor: An XModuleDescriptor + location: A Location """ - system = preview_module_system(request, preview_id, descriptor) + system = preview_module_system(preview_id, descriptor, request) try: module = descriptor.xmodule(system) except: @@ -171,10 +180,10 @@ def load_preview_module(request, preview_id, descriptor): return module -def get_preview_html(request, descriptor, idx): +def get_preview_html(descriptor, idx, request): """ Returns the HTML returned by the XModule's student_view, specified by the descriptor and idx. """ - module = load_preview_module(request, str(idx), descriptor) - return module.runtime.render(module, None, "student_view").content + module = load_preview_module(str(idx), descriptor, request) + return module.runtime.render(module, None, "student_view").content \ No newline at end of file diff --git a/cms/djangoapps/contentstore/views/session_kv_store.py b/cms/djangoapps/contentstore/views/session_kv_store.py index 87a92a9e2e41..03a67290ae8a 100644 --- a/cms/djangoapps/contentstore/views/session_kv_store.py +++ b/cms/djangoapps/contentstore/views/session_kv_store.py @@ -26,3 +26,29 @@ def delete(self, key): def has(self, key): return key.field_name in self._descriptor_model_data or tuple(key) in self._session + +class StaticPreviewKeyValueStore(KeyValueStore): + '''Like the SessionKeyValueStore but session independent (this breaks randomization)''' + def __init__(self, model_data): + self._model_data = model_data + + def get(self, key): + try: + return self._model_data[key.field_name] + except: + raise KeyError + + def set(self, key, value): + try: + self._model_data[key.field_name] = value + except (KeyError, InvalidScopeError): + pass + + def delete(self, key): + try: + del self._model_data[key.field_name] + except: + raise KeyError + + def has(self, key): + return key.field_name in self._model_data \ No newline at end of file diff --git a/cms/envs/common.py b/cms/envs/common.py index 29e99b2551c2..f2b7605e2bf8 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -27,7 +27,6 @@ import lms.envs.common from lms.envs.common import USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL from path import path - ############################ FEATURE CONFIGURATION ############################# MITX_FEATURES = { @@ -55,7 +54,10 @@ # If set to True, new Studio users won't be able to author courses unless # edX has explicitly added them to the course creator group. - 'ENABLE_CREATOR_GROUP': False + 'ENABLE_CREATOR_GROUP': False, + + # disable content testing which is in development + 'CONTENT_TESTING': False, } ENABLE_JASMINE = False @@ -360,7 +362,10 @@ 'django.contrib.admin', # for managing course modes - 'course_modes' + 'course_modes', + + #automated content testing for custom response + 'content_testing', ) ################# EDX MARKETING SITE ################################## diff --git a/cms/envs/test.py b/cms/envs/test.py index ffbf9f5376d2..1106a49bf603 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -150,3 +150,6 @@ # This is to disable a test under the common directory that will not pass when run under CMS MITX_FEATURES['DISABLE_PASSWORD_RESET_EMAIL_TEST'] = True + +# this is disabled under normal dev since the feature is in development +MITX_FEATURES['CONTENT_TESTING'] = True diff --git a/cms/static/coffee/src/views/module_edit.coffee b/cms/static/coffee/src/views/module_edit.coffee index c45feecd419e..c9c30b7dfc48 100644 --- a/cms/static/coffee/src/views/module_edit.coffee +++ b/cms/static/coffee/src/views/module_edit.coffee @@ -2,6 +2,8 @@ class CMS.Views.ModuleEdit extends Backbone.View tagName: 'li' className: 'component' editorMode: 'editor-mode' + settingsMode: 'settings-mode' + testMode: 'test-mode' events: "click .component-editor .cancel-button": 'clickCancelButton' @@ -126,22 +128,43 @@ class CMS.Views.ModuleEdit extends Backbone.View selectMode: (mode) => dataEditor = @$el.find('.wrapper-comp-editor') settingsEditor = @$el.find('.wrapper-comp-settings') + testEditor = @$el.find('.wrapper-comp-testor') editorModeButton = @$el.find('#editor-mode').find("a") settingsModeButton = @$el.find('#settings-mode').find("a") + testModeButton = @$el.find('#test-mode').find("a") if mode == @editorMode # Because of CodeMirror editor, cannot hide the data editor when it is first loaded. Therefore # we have to use a class of is-inactive instead of is-active. dataEditor.removeClass('is-inactive') editorModeButton.addClass('is-set') + settingsEditor.removeClass('is-active') settingsModeButton.removeClass('is-set') - else + + testEditor.removeClass('is-active') + testModeButton.removeClass('is-set') + + else if mode == @settingsMode dataEditor.addClass('is-inactive') editorModeButton.removeClass('is-set') + settingsEditor.addClass('is-active') settingsModeButton.addClass('is-set') + testEditor.removeClass('is-active') + testModeButton.removeClass('is-set') + + else if mode == @testMode + dataEditor.addClass('is-inactive') + editorModeButton.removeClass('is-set') + + settingsEditor.removeClass('is-active') + settingsModeButton.removeClass('is-set') + + testEditor.addClass('is-active') + testModeButton.addClass('is-set') + hideDataEditor: => editorModeButtonParent = @$el.find('#editor-mode') editorModeButtonParent.addClass('inactive-mode') diff --git a/cms/templates/content_testing/response_summary.html b/cms/templates/content_testing/response_summary.html new file mode 100644 index 000000000000..ec365209c4b9 --- /dev/null +++ b/cms/templates/content_testing/response_summary.html @@ -0,0 +1,13 @@ +
  • +
      + % for value in answers: +
    • + ${value} +
    • + % endfor +
    + + + ${msg} + +
  • \ No newline at end of file diff --git a/cms/templates/content_testing/test_form.html b/cms/templates/content_testing/test_form.html new file mode 100644 index 000000000000..93cc3dbcbbcf --- /dev/null +++ b/cms/templates/content_testing/test_form.html @@ -0,0 +1,6 @@ +
    + + + + +
    \ No newline at end of file diff --git a/cms/templates/content_testing/test_summaries.html b/cms/templates/content_testing/test_summaries.html new file mode 100644 index 000000000000..dcdb137e7f64 --- /dev/null +++ b/cms/templates/content_testing/test_summaries.html @@ -0,0 +1,9 @@ +Run Tests +
      +% for should_be in summaries: +${should_be} +
    • + ${summaries[should_be]} +
    • +% endfor +
    diff --git a/cms/templates/content_testing/test_summary.html b/cms/templates/content_testing/test_summary.html new file mode 100644 index 000000000000..dde84840a05d --- /dev/null +++ b/cms/templates/content_testing/test_summary.html @@ -0,0 +1,42 @@ +## <%inherit file="../base.html" /> +## <%namespace name='static' file='../static_content.html'/> + +## +## <%block name="title">Test Summary + +## <%block name='content'> +##
    +##
    +##
    +##

    Testing Summary


    +% if tests: + % for test in tests: + ${test.get_html_summary()} +
    + + + +
    +
    + + +
    +
    + % endfor +
    + ## + + +
    +%else: +

    No Tests

    +%endif +
    + ## + + +
    +
    +
    +
    + diff --git a/cms/urls.py b/cms/urls.py index 8f396d374287..7ccca22c09cc 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -135,6 +135,11 @@ url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict), ) +if settings.MITX_FEATURES['CONTENT_TESTING']: + urlpatterns += ( + #for content testing + url(r'^test_problem/$', 'content_testing.views.problem_test', name='testing'), + ) if settings.ENABLE_JASMINE: urlpatterns += (url(r'^_jasmine/', include('django_jasmine.urls')),) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index c2bdeadc2143..70885228517b 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -474,7 +474,6 @@ def _extract_system_path(self, script): # find additional comma-separated modules search path path = [] - for dir in raw_path: if not dir: continue @@ -557,6 +556,7 @@ def _extract_html(self, problemtree): # private ''' if (problemtree.tag == 'script' and problemtree.get('type') and 'javascript' in problemtree.get('type')): + # leave javascript intact. return deepcopy(problemtree) @@ -645,10 +645,16 @@ def _preprocess_problem(self, tree): # private ''' response_id = 1 self.responders = {} + self.responders_by_id = {} for response in tree.xpath('//' + "|//".join(response_tag_dict)): response_id_str = self.problem_id + "_" + str(response_id) # create and save ID for this response response.set('id', response_id_str) + + # This nex line should occure at the end of the loop after the + # `for entry in inputfields:` loop. This is a bug. It has been + # decided that it will be fixed in the general restructuring of + # the capa_problem which is slated for down the road. response_id += 1 answer_id = 1 @@ -670,6 +676,7 @@ def _preprocess_problem(self, tree): # private self.context, self.system) # save in list in self self.responders[response] = responder + self.responders_by_id[responder.id] = responder # get responder answers (do this only once, since there may be a performance cost, # eg with externalresponse) diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index dbd535a471be..ddaaa8f0c049 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -18,7 +18,7 @@ from xmodule.x_module import XModule from xmodule.raw_module import RawDescriptor from xmodule.exceptions import NotFoundError, ProcessingError -from xblock.core import Scope, String, Boolean, Dict, Integer, Float +from xblock.core import Scope, String, Boolean, Dict, Integer, Float, List from .fields import Timedelta, Date from django.utils.timezone import UTC @@ -151,6 +151,7 @@ class CapaFields(object): help="Source code for LaTeX and Word problems. This feature is not well-supported.", scope=Scope.settings ) + tests = List(help="Tests cases for the LCP", default=[], scope=Scope.settings) class CapaModule(CapaFields, XModule): @@ -564,7 +565,7 @@ def handle_ajax(self, dispatch, data): 'problem_show': self.get_answer, 'score_update': self.update_score, 'input_ajax': self.handle_input_ajax, - 'ungraded_response': self.handle_ungraded_response + 'ungraded_response': self.handle_ungraded_response, } generic_error_message = ( @@ -1171,5 +1172,6 @@ def backcompat_paths(cls, path): def non_editable_metadata_fields(self): non_editable_fields = super(CapaDescriptor, self).non_editable_metadata_fields non_editable_fields.extend([CapaDescriptor.due, CapaDescriptor.graceperiod, - CapaDescriptor.force_save_button, CapaDescriptor.markdown]) + CapaDescriptor.force_save_button, CapaDescriptor.markdown, + CapaDescriptor.tests]) return non_editable_fields