dont consume arbitrary unused kwargs in TerrierIndexer, update overridden properties, addreses #499 (#500)

Author: cmacdonald

pyterrier/terrier/index.py

@@ -225,8 +225,9 @@ def __init__(self, index_path : str, *args,
             stemmer : Union[None, str, TerrierStemmer] = TerrierStemmer.porter,
             stopwords : Union[None, TerrierStopwords, List[str]] = TerrierStopwords.terrier,
             tokeniser : Union[str,TerrierTokeniser] = TerrierTokeniser.english,
-            type=IndexingType.CLASSIC, 
-            **kwargs):
+            type=IndexingType.CLASSIC,
+            properties : Dict[str,str] = {}
+            ):
         """
         Constructor called by all indexer subclasses. All arguments listed below are available in 
         IterDictIndexer, DFIndexer, TRECCollectionIndexer and FilesIndsexer. 
@@ -240,6 +241,7 @@ def __init__(self, index_path : str, *args,
             stopwords (TerrierStopwords): the stopwords list to apply. Default is ``TerrierStemmer.terrier``.
             tokeniser (TerrierTokeniser): the stemmer to apply. Default is ``TerrierTokeniser.english``.
             type (IndexingType): the specific indexing procedure to use. Default is ``IndexingType.CLASSIC``.
+            properties (dict): Terrier properties that you wish to overrride.
         """
         if type is IndexingType.MEMORY:
             self.path = None
@@ -256,6 +258,8 @@ def __init__(self, index_path : str, *args,
         self.tokeniser = TerrierTokeniser._to_obj(tokeniser)
         self.properties = pt.java.J.Properties()
         self.setProperties(**self.default_properties)
+        for k,v in properties.items():
+            self.properties[k] = v        
         self.overwrite = overwrite
         self.verbose = verbose
         self.meta_reverse = meta_reverse

tests/test_iterdictindex.py

@@ -150,6 +150,16 @@ def test_createindex3_single_pass_2fields(self):
         from pyterrier.terrier.index import IndexingType
         self._make_check_index(3, IndexingType.SINGLEPASS, fields=['text', 'title'])
 
+    def test_long_tokens(self):
+        longword = 'abc' * (1+int(60/3))
+        indexer = pt.IterDictIndexer(self.test_dir, tokeniser='identity', overwrite=True, properties={'max.term.length' : '80'})
+        ref = indexer.index([ {"docno" : "d1", 'text': longword}])
+        index = pt.IndexFactory.of(ref)
+        self.assertEqual(1, len(index))
+        self.assertEqual(1, index.getCollectionStatistics().getNumberOfUniqueTerms())
+        self.assertEqual('80', pt.java.cast("org.terrier.structures.PropertiesIndex", index).getIndexProperty("max.term.length", None))
+        self.assertTrue(longword in index.getLexicon())
+
     def test_meta_init(self):
         it = [
             {'docno': '11', 'url': 'url1', 'text': 'He ran out of money, so he had to stop playing', 'title': 'Woes of playing poker'},