SLT Translation

Author: jniehues-kit

scripts/monoTranslationData/AddPunctuation.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+
+
+sentenceEnd=[".","!","?"]
+
+def add(s_filename,t_filename,context):
+
+    s_f = open(s_filename)
+    t_f = open(t_filename)
+
+    s = s_f.readline()
+    t = t_f.readline()
+
+    stat = [({},{}) for i in range(context)]
+    
+    while(s and t):
+        w_s = s.strip().split();
+
+        w_t = t.strip().split();
+
+        for i in range(min(context,len(w_t))):
+            if(w_t[i][0] in stat[i][0]):
+                stat[i][0][w_t[i][0]] += 1;
+            else:
+                stat[i][0][w_t[i][0]] = 1
+            if(len(w_t[i]) > 1):
+                if(w_t[i][1:] in stat[i][1]):
+                    stat[i][1][w_t[i][1:]] += 1;
+                else:
+                    stat[i][1][w_t[i][1:]] = 1
+            else:
+                if(w_t[i][1:] in stat[i][1]):
+                    stat[i][1][""] += 1;
+                else:
+                    stat[i][1][""] = 1
+
+        if(not 'L' in stat[0][0]):
+            print w_s[0].title(),
+        elif(not 'U' in stat[0][0] or stat[0][0]['L'] > stat[0][0]['U']):
+            print w_s[0],
+        else:
+            print w_s[0].title(),
+
+        maxPunc = ""
+        maxCount = 0
+        for k in stat[0][1].keys():
+            if(stat[0][1][k] > maxCount):
+                maxPunc=k
+                maxCount = stat[0][1][k]
+        #print stat[0],maxPunc
+        if(maxPunc != ""):
+            print maxPunc,
+        if any(p in maxPunc for p in sentenceEnd):
+            print "";
+
+        s = s_f.readline()
+        t = t_f.readline()
+        stat = stat[1:]+[({},{})]        
+            
+
+
+def main():
+    args=sys.argv[1:];
+    if(len(args) == 3):
+        add(args[0],args[1],int(args[2]));
+    else:
+        raise SystemExit("Usage: python "+sys.argv[0]+" source U/LFile context")
+
+
+main();

scripts/monoTranslationData/Translate.sh

@@ -14,15 +14,18 @@ mkdir -p /tmp/$name
 mkdir -p /data/${name}/eval/
 
 cd /data/$input/eval/$set/IWSLT.$set/
-echo -n "" > /tmp/$name/$set.np
 i=0
+echo -n "" > /data/${name}/eval/$set.s
 while read -r line
 do
-    sed -e "s/([0-9]*)//g" $line | sed -e '/\$(.*)/d' | awk '{if($1 != "#") {printf("%s ",$5)}}END{print ""}' | sed -e 's/\,//g' | sed -e 's/\.//g' | sed -e 's/?//g' | sed -e 's/\!//g' | sed -e 's/\"//g' | sed -e 's/^\s*//g' | sed -e 's/\s\s*/ /g' | perl -nle 'print lc' >> /tmp/$name/$set.$i.np
+    sed -e "s/([0-9]*)//g" $line | sed -e '/\$(.*)/d' | awk '{if($1 != "#") {printf("%s ",$5)}}END{print ""}' | sed -e 's/\,//g' | sed -e 's/\.//g' | sed -e 's/?//g' | sed -e 's/\!//g' | sed -e 's/\"//g' | sed -e 's/^\s*//g' | sed -e 's/\s\s*/ /g' | perl -nle 'print lc' > /tmp/$name/$set.$i.np
     /opt/SLT.KIT/scripts/monoTranslationData/ConCat10.pl /tmp/$name/$set.$i.np 10 > /tmp/$name/$set.$i.np.concat
     cat /tmp/$name/$set.$i.np.concat | /opt/subword-nmt/apply_bpe.py -c /model/${name}/codec --vocabulary /model/${name}/voc --vocabulary-threshold 50 > /data/${name}/eval/$set.$i.s
     /opt/SLT.KIT/scripts/openNMT-py/Translate.sh $set.$i $name $translator 
+    python /opt/SLT.KIT/scripts/monoTranslationData/AddPunctuation.py /tmp/$name/$set.$i.np.concat /data/$translator/eval/$set.$i.t 10 >> /data/${name}/eval/$set.s
     ((i++))
 done < /data/$input/eval/$set/IWSLT.$set/CTM_LIST
 
+rm -r /tmp/$name
+
 

systems/smallTED/Translate.sh

@@ -11,17 +11,8 @@ if [ ! -e /data/orig/eval/$set ]; then
 
 fi
 
+#Add puncuation
 /opt/SLT.KIT/scripts/monoTranslationData/Translate.sh $set orig monoTransPrepro monTrans
 
-#Preprocess Data
-#/opt/SLT.KIT/scripts/defaultPreprocessor/Train.sh orig prepro
-
-
-#Train NMT
-#/opt/SLT.KIT/scripts/openNMT-py/Train.sh prepro mt
-
-
-#Preprocess for Puncutation
-
-#monTranslationSystem
-#/opt/SLT.KIT/scripts/openNMT-py/Train.sh monoTransPrepro monTrans
+#Translate
+/opt/SLT.KIT/scripts/openNMT-py/Translate.sh $set monoTransPrepro mt