Eval of manual Translation

Author: jniehues-kit

scripts/defaultPreprocessor/Translate.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+
+set=$1
+name=$2
+
+mkdir -p /data/${name}/eval
+mkdir -p /data/${name}/valid
+
+##TOKENIZE
+##SMARTCASE
+##BPE
+
+cat /data/orig/eval/$set/IWSLT.$set/IWSLT.TED.$set.$sl-$tl.$sl.xml | \
+    grep "<seg id" | sed -e "s/<[^>]*>//g" | \
+    perl /opt/mosesdecoder/scripts/tokenizer/tokenizer.perl -l ${sl} | \
+    /opt/mosesdecoder/scripts/recaser/truecase.perl --model /model/${name}/truecase-model.s | \
+    /opt/subword-nmt/apply_bpe.py -c /model/${name}/codec --vocabulary /model/${name}/voc.s --vocabulary-threshold 50 \
+				  > /data/${name}/eval/manualTranscript.$set.s
+
+

scripts/evaluate/Eval.manualTranscript.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set=$1
+output=$2
+
+
+source=/data/orig/eval/${set}/IWSLT.${set}/IWSLT.TED.${set}.${sl}-${tl}.${sl}.xml
+reference=/data/orig/eval/${set}/IWSLT.${set}/IWSLT.TED.${set}.${sl}-${tl}.${tl}.xml
+
+
+
+mkdir -p /tmp/eval.manual
+mkdir -p /results/$systemName/$set/
+
+
+grep "<seg id" $reference | sed -e "s/<[^>]*>//g" > /tmp/eval.manual/$set.reference
+
+sed -e "s/@@ //g" /data/$output/eval/manualTranscript.$set.t | sed -e "s/@@$//g" | sed -e "s/&apos;/'/g" -e 's/&#124;/|/g' -e "s/&amp;/&/g" -e 's/&lt;/>/g' -e 's/&gt;/>/g' -e 's/&quot;/"/g' -e 's/&#91;/[/g' -e 's/&#93;/]/g' | perl -nle 'print ucfirst' > /tmp/eval.manual/$set.hyp
+
+sed -e "s/^\s*$/_EMPTY_/g" /tmp/eval.manual/$set.hyp > /tmp/eval.manual/$set.no-empty.hyp
+cat /tmp/eval.manual/$set.hyp | perl /opt/SLT.KIT/scripts/evaluate/wrap-xml.perl de /data/orig/eval/dev2010/IWSLT.dev2010/IWSLT.TED.dev2010.en-de.en.xml $systemName > /tmp/eval.manual/$set.xml
+
+
+
+/opt/mosesdecoder/scripts/generic/mteval-v14.pl -c -s $source -r $reference -t /tmp/eval.manual/$set.xml > /results/$systemName/$set/manualTranscript.BLEU.case-sensitive
+/opt/mosesdecoder/scripts/generic/mteval-v14.pl  -s $source -r $reference -t /tmp/eval.manual/$set.xml > /results/$systemName/$set/manualTranscript.BLEU.case-insensitive
+
+java -Dfile.encoding=UTF8 -jar /opt/tercom-0.7.25/tercom.7.25.jar -N -s -r $reference -h /tmp/eval.manual/$set.xml > /results/$systemName/$set/manualTranscript.TER.case-sensitive
+java -Dfile.encoding=UTF8 -jar /opt/tercom-0.7.25/tercom.7.25.jar -N -r $reference -h /tmp/eval.manual/$set.xml > /results/$systemName/$set/manualTranscript.TER.case-insensitive
+
+/opt/beer_2.0/beer -s /tmp/eval.manual/$set.hyp -r /tmp/eval.manual/$set.reference > /results/$systemName/$set/manualTranscript.BEER.case-sensitive
+/opt/CharacTER/CharacTER.py -r /tmp/eval.manual/$set.reference -o /tmp/eval.manual/$set.no-empty.hyp > /results/$systemName/$set/manualTranscript.CharacTER.case-sensitive
+
+
+BLEU=`grep BLEU /results/$systemName/$set/manualTranscript.BLEU.case-sensitive | head -n 1 | awk '{print $8*100}'`
+ciBLEU=`grep BLEU /results/$systemName/$set/manualTranscript.BLEU.case-insensitive | head -n 1 | awk '{print $8*100}'`
+TER=`grep TER /results/$systemName/$set/manualTranscript.TER.case-sensitive | awk '{printf("%.2f\n",$3*100)}'`
+ciTER=`grep TER /results/$systemName/$set/manualTranscript.TER.case-insensitive | awk '{printf("%.2f\n",$3*100)}'`
+beer=`awk '{printf("%.2f\n",$3*100)}' /results/$systemName/$set/manualTranscript.BEER.case-sensitive`
+character=`awk '{printf("%.2f\n",$1*100)}' /results/$systemName/$set/manualTranscript.CharacTER.case-sensitive`
+
+echo "| $set (manual Transcript) | $BLEU | $TER | $beer | $character | $ciBLEU | $ciTER |" >> /results/$systemName/$set/Summary.md

scripts/evaluate/Eval.sh

@@ -35,15 +35,15 @@ java -Dfile.encoding=UTF8 -jar /opt/tercom-0.7.25/tercom.7.25.jar -N -s -r $refe
 java -Dfile.encoding=UTF8 -jar /opt/tercom-0.7.25/tercom.7.25.jar -N -r $reference -h /tmp/eval/$set.no-case.xml > /results/$systemName/$set/TER.case-insensitive
 
 /opt/beer_2.0/beer -s /tmp/eval/$set.hyp -r /tmp/eval/$set.reference > /results/$systemName/$set/BEER.case-sensitive
-/opt/CharacTER/CharacTER.py -r /tmp/eval/$set.reference -o /tmp/eval/$set.no-empty.hyp > /results/$systemName/$set/CharacTER.case-senstive
+/opt/CharacTER/CharacTER.py -r /tmp/eval/$set.reference -o /tmp/eval/$set.no-empty.hyp > /results/$systemName/$set/CharacTER.case-sensitive
 
 
 BLEU=`grep BLEU /results/$systemName/$set/BLEU.case-sensitive | head -n 1 | awk '{print $8*100}'`
 ciBLEU=`grep BLEU /results/$systemName/$set/BLEU.case-insensitive | head -n 1 | awk '{print $8*100}'`
 TER=`grep TER /results/$systemName/$set/TER.case-sensitive | awk '{printf("%.2f\n",$3*100)}'`
 ciTER=`grep TER /results/$systemName/$set/TER.case-insensitive | awk '{printf("%.2f\n",$3*100)}'`
 beer=`awk '{printf("%.2f\n",$3*100)}' /results/$systemName/$set/BEER.case-sensitive`
-character=`awk '{printf("%.1f\n",$3*100)}' /results/$systemName/$set/BEER.case-sensitive`
+character=`awk '{printf("%.2f\n",$1*100)}' /results/$systemName/$set/CharacTER.case-sensitive`
 
 echo "Results for $systemName" > /results/$systemName/$set/Summary.md
 echo "=======================" >> /results/$systemName/$set/Summary.md

systems/smallTED/Translate.sh

@@ -17,6 +17,19 @@ fi
 #Translate
 /opt/SLT.KIT/scripts/openNMT-py/Translate.sh $set monoTransPrepro mt
 
-
 #Eval
 /opt/SLT.KIT/scripts/evaluate/Eval.sh dev2010 mt
+
+#Prepro manual transcript
+/opt/SLT.KIT/scripts/defaultPreprocessor/Translate.sh $set prepro
+
+#Translate manual transcript
+/opt/SLT.KIT/scripts/openNMT-py/Translate.sh manualTranscript.$set prepro mt
+
+#Eval manual transcript
+/opt/SLT.KIT/scripts/evaluate/Eval.manualTranscript.sh dev2010 mt
+
+
+
+
+