hived off libraries into new gdtools project

Author: colinbatchelor

brown_gd_to_conll.py

@@ -7,8 +7,8 @@
 import re
 import sys
 from collections import namedtuple
-from acainn import Lemmatizer
-from acainn import Features
+from gdtools.acainn import Lemmatizer
+from gdtools.acainn import Features
 from pyconll.unit import Conll
 
 Split = namedtuple("split", "form1 upos1 xpos1 form2 upos2 xpos2")
@@ -166,19 +166,18 @@ def parse_brown_token(brown_token):
         return subtokens[0], "__MW" # special cases for multiword expressions like "ann an"
     return subtokens[0], subtokens[1].strip("*")
 
-def process_file(brown_file, filename):
+def process_file(brown_file, file_id):
     """Does the initial conversion to CoNLL-U format."""
     lemmatizer = Lemmatizer()
     result = []
-    file_id = filename.replace(".txt", "")
     subcorpus = re.findall("^[a-z]*", file_id)[0]
     if subcorpus in ["c", "p", "s"] or file_id in ["n06", "n07", "n08", "n09", "n10"]:
         genre = "oral"
     else:
         genre = "written"
     sent_id = 0
     for sentence in split_sentences(brown_file, genre):
-        conllu_tokens = [s for s in process_sentence(sentence, lemmatizer)]
+        conllu_tokens = process_sentence(sentence, lemmatizer)
         if len(conllu_tokens) > 0:
             result.append(f"# sent_id = {file_id}_{sent_id}")
             result.extend(conllu_tokens)
@@ -293,7 +292,7 @@ def add_text(corpus):
 for filename in files:
     if filename.startswith(sys.argv[2]):
         with open(os.path.join(sys.argv[1], filename)) as file:
-            lines = process_file(file, filename)
+            lines = process_file(file, filename.replace(".txt", ""))
 
             c = Conll(lines)
             with_text = add_text(c)

brown_gd_to_dot_ccg.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 import pickle
 import sys
-from acainn import Lemmatizer, Retagger, Subcat, Typer
+from gdtools.acainn import Lemmatizer, Retagger, Subcat, Typer
 
 def tidy_word(string):
     """outputs string suitable for XMLification further down the pipeline"""