Search in multivalue fields

.pylintrc

@@ -1,5 +1,7 @@
 [MESSAGES CONTROL]
 disable=too-many-lines
+# https://bitbucket.org/logilab/pylint/issue/214/the-duplicate-code-r0801-cant-be-disabled
+disable=duplicate-code
 
 [FORMAT]
 max-line-length=120

README.md

@@ -87,6 +87,12 @@ search_result = search_engine.search(field_dictionary=match_field_dict)
 
 _Notice the . notation for querying fields that are nested within the indexed object_
 
+**Important notice:** searching in multivalue fields (i.e. a lists) have a special semantics - if search term is a
+scalar value (i.e. string, number, etc.), search uses "contains in" predicate, so all documents containing specified
+search value as one of the elements of multivalue field are included in result set. If search term is vector value (i.e. 
+list, tuple, dictionary, etc.), search will result in undefined behavior, specific to underlying search engine; thus
+using iterable as filter field value is discouraged. 
+
 #### Search results
 The `search_result` object returned from a call to `search` is a python dict object that contains the following fields:
 ```
@@ -168,6 +174,8 @@ already_started = {
 search_result = search_engine.search(filter_dictionary=already_started)
 ```
 
+**Important notice:** same concerns about searching in multivalue fields apply here.
+
 ### Searches using a combination of these criteria
 All of these criteria can be combined to present results that are desired. Consider a search that wants to return objects that:
 

search/tests/mock_search_engine.py

@@ -2,6 +2,7 @@
 import copy
 from datetime import datetime
 import json
+import collections
 import os
 
 from django.conf import settings
@@ -47,6 +48,11 @@ def _find_field(doc, field_name):
         return field_value
 
 
+def _is_iterable(item):
+    """ Checks if an item is iterable (list, tuple, generator), but not string """
+    return isinstance(item, collections.Iterable) and not isinstance(item, basestring)
+
+
 def _filter_intersection(documents_to_search, dictionary_object, include_blanks=False):
     """
     Filters out documents that do not match all of the field values within the dictionary_object
@@ -73,6 +79,8 @@ def value_matches(doc, field_name, field_value):
                 (field_value.lower is None or compare_value >= field_value.lower) and
                 (field_value.upper is None or compare_value <= field_value.upper)
             )
+        elif _is_iterable(compare_value) and not _is_iterable(field_value):
+            return any((item == field_value for item in compare_value))
         else:
             return compare_value == field_value
 

search/tests/test_mock_search_engine.py

@@ -0,0 +1,81 @@
+""" Tests for MockSearchEngine specific features """
+from datetime import datetime
+from django.test import TestCase
+from django.test.utils import override_settings
+from search.tests.mock_search_engine import _find_field, _filter_intersection, json_date_to_datetime
+
+
+# Any class that inherits from TestCase will cause too-many-public-methods pylint error
+# pylint: disable=too-many-public-methods
+@override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine")
+@override_settings(ELASTIC_FIELD_MAPPINGS={"start_date": {"type": "date"}})
+class MockSpecificSearchTests(TestCase):
+    """ For testing pieces of the Mock Engine that have no equivalent in Elastic """
+
+    def test_find_field_arguments(self):
+        """ test that field argument validity is observed """
+        field_value = _find_field(
+            {
+                "name": "Come and listen to my story"
+            },
+            "name"
+        )
+        self.assertEqual(field_value, "Come and listen to my story")
+
+        field_value = _find_field(
+            {
+                "name": {
+                    "first": "Martyn",
+                    "last": "James"
+                }
+            },
+            "name.first"
+        )
+        self.assertEqual(field_value, "Martyn")
+
+        field_value = _find_field(
+            {
+                "name": {
+                    "first": "Monica",
+                    "last": {
+                        "one": "Parker",
+                        "two": "James"
+                    }
+                }
+            },
+            "name.last.two"
+        )
+        self.assertEqual(field_value, "James")
+
+        with self.assertRaises(ValueError):
+            field_value = _find_field(
+                {
+                    "name": "Come and listen to my story"
+                },
+                123
+            )
+
+        with self.assertRaises(ValueError):
+            field_value = _find_field(123, "name")
+
+    def test_filter_optimization(self):
+        """ Make sure that intersection optimizes return when no filter dictionary is provided """
+        test_docs = [{"A": {"X": 1, "Y": 2, "Z": 3}}, {"B": {"X": 9, "Y": 8, "Z": 7}}]
+        self.assertTrue(_filter_intersection(test_docs, None), test_docs)
+
+    def test_datetime_conversion(self):
+        """ tests json_date_to_datetime with different formats """
+        json_date = "2015-01-31"
+        self.assertTrue(json_date_to_datetime(json_date), datetime(2015, 1, 31))
+
+        json_datetime = "2015-01-31T07:30:28"
+        self.assertTrue(json_date_to_datetime(json_datetime), datetime(2015, 1, 31, 7, 30, 28))
+
+        json_datetime = "2015-01-31T07:30:28.65785"
+        self.assertTrue(json_date_to_datetime(json_datetime), datetime(2015, 1, 31, 7, 30, 28, 65785))
+
+        json_datetime = "2015-01-31T07:30:28Z"
+        self.assertTrue(json_date_to_datetime(json_datetime), datetime(2015, 1, 31, 7, 30, 28))
+
+        json_datetime = "2015-01-31T07:30:28.65785Z"
+        self.assertTrue(json_date_to_datetime(json_datetime), datetime(2015, 1, 31, 7, 30, 28, 65785))