Inital OpenNMT-py

Author: jniehues-kit

scripts/defaultPreprocessor/Train.sh

@@ -82,7 +82,7 @@ done
 
 for set in valid train
 do
-for f in /tmp/${name}/tok/$set/*\.s
+for f in /tmp/${name}/sc/$set/*\.s
 do
 echo $f
 /opt/subword-nmt/apply_bpe.py -c /model/${name}/codec --vocabulary /model/${name}/voc.s --vocabulary-threshold 50 < $f > /data/${name}/$set/${f##*/}
@@ -91,7 +91,7 @@ done
 
 for set in valid train
 do
-for f in /tmp/${name}/tok/$set/*\.t
+for f in /tmp/${name}/sc/$set/*\.t
 do
 echo $f
 /opt/subword-nmt/apply_bpe.py -c /model/${name}/codec --vocabulary /model/${name}/voc.t --vocabulary-threshold 50 < $f > /data/${name}/$set/${f##*/}

scripts/openNMT-py/Train.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+input=$1
+name=$2
+
+
+mkdir -p /tmp/${name}/
+mkdir -p /model/${name}/
+
+
+
+
+for l in s t
+do
+    for set in train valid
+    do
+	echo "" > /tmp/${name}/$set.$l
+	for f in /data/${input}/${set}/*\.${l}
+	do
+	    
+		 cat $f >> /tmp/${name}/$set.$l
+	done
+    done
+done
+
+
+
+python /opt/OpenNMT-py/preprocess.py \
+       -train_src /tmp/${name}/train.s \
+       -train_tgt /tmp/${name}/train.t \
+       -valid_src /tmp/${name}/valid.s \
+       -valid_tgt /tmp/${name}/vaild.t \
+       -save_data /tmp/${name}/train \
+       -src_langs s \
+       -tgt_langs t 
+
+
+python -u /opt/OpenNMT-py/train.py  -data /tmp/${name}/train.train.pt \
+       -save_model /model/${name}/model \
+       -brnn \
+       -rnn_size 512 \
+       -word_vec_size 512 \
+       -batch_size 128 \
+       -max_generator_batches 16 \
+       -optim adam \
+       -dropout 0.2 \
+       -epochs 10 \
+       -learning_rate 0.001 \
+       -gpus 0