Merging to resolve conflict

README.md

@@ -51,3 +51,14 @@ Development pattern for contributors
    You can use the [Google's cpplint.py]
    (https://raw.githubusercontent.com/google/styleguide/gh-pages/cpplint/cpplint.py)
    to verify that your code is free of basic mistakes.
+
+Platform specific notes
+-----------------------
+
+PowerPC 64bits little-endian (ppc64le):
+- Kaldi is expected to work out of the box in RHEL >= 7 and Ubuntu >= 16.04 with
+  OpenBLAS, ATLAS, or CUDA.
+- CUDA drivers for ppc64le can be found at [https://developer.nvidia.com/cuda-downloads]
+  (https://developer.nvidia.com/cuda-downloads).
+- An [IBM Redbook] (https://www.redbooks.ibm.com/abstracts/redp5169.html) is
+  available as a guide to install and configure CUDA.

egs/ami/s5b/RESULTS_ihm

@@ -51,9 +51,8 @@
 # local/nnet3/run_blstm.sh --mic ihm
 # nnet3 xent BLSTM with data cleaning
 # for d in exp/ihm/nnet3_cleaned/lstm_bidirectional_sp/decode_*; do grep Sum $d/*sc*/*ys | utils/best_wer.sh; done
-# Note: the results are with ClipGradientComponent, which may be different from with BackpropTruncationComponent
-%WER 22.3 | 13098 94494 | 80.9 11.7 7.4 3.2 22.3 55.7 | -0.618 | exp/ihm/nnet3_cleaned/lstm_bidirectional_sp/decode_dev/ascore_10/dev_hires.ctm.filt.sys
-%WER 22.5 | 12643 89962 | 80.2 12.7 7.1 2.7 22.5 53.4 | -0.476 | exp/ihm/nnet3_cleaned/lstm_bidirectional_sp/decode_eval/ascore_10/eval_hires.ctm.filt.sys
+%WER 22.4 | 13098 94483 | 80.8 11.6 7.6 3.2 22.4 55.4 | -0.620 | exp/ihm/nnet3_cleaned/lstm_bidirectional_sp/decode_dev/ascore_10/dev_hires.ctm.filt.sys
+%WER 22.4 | 12643 89977 | 80.3 12.5 7.2 2.7 22.4 53.6 | -0.503 | exp/ihm/nnet3_cleaned/lstm_bidirectional_sp/decode_eval/ascore_10/eval_hires.ctm.filt.sys
 
 ############################################
 

egs/ami/s5b/RESULTS_sdm

@@ -49,9 +49,8 @@
 # xent BLSTM system; cleaned data and IHM alignments.
 # local/nnet3/run_blstm.sh --mic sdm1 --use-ihm-ali true
 # for d in exp/sdm1/nnet3_cleaned/lstm_bidirectional_sp/decode_*; do grep Sum $d/*sc*/*ys | utils/best_wer.sh; done
-# Note: the results are with ClipGradientComponent, which may be different from with BackpropTruncationComponent
-%WER 37.8 | 14633 94518 | 67.1 22.3 10.7 4.9 37.8 64.2 | 0.745 | exp/sdm1/nnet3_cleaned/lstm_bidirectional_sp_ihmali/decode_dev/ascore_11/dev_hires_o4.ctm.filt.sys
-%WER 41.4 | 13809 89628 | 62.7 24.1 13.2 4.1 41.4 65.2 | 0.723 | exp/sdm1/nnet3_cleaned/lstm_bidirectional_sp_ihmali/decode_eval/ascore_11/eval_hires_o4.ctm.filt.sys
+%WER 37.9 | 15953 94512 | 66.7 22.0 11.3 4.7 37.9 58.9 | 0.734 | exp/sdm1/nnet3_cleaned/lstm_bidirectional_sp_ihmali/decode_dev/ascore_12/dev_hires_o4.ctm.filt.sys
+%WER 41.2 | 13271 89635 | 62.9 23.8 13.2 4.2 41.2 67.8 | 0.722 | exp/sdm1/nnet3_cleaned/lstm_bidirectional_sp_ihmali/decode_eval/ascore_11/eval_hires_o4.ctm.filt.sys
 
 # =========================
 

egs/ami/s5b/local/chain/multi_condition/run_tdnn.sh

@@ -260,7 +260,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_1a.sh

@@ -219,7 +219,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then
@@ -239,4 +239,4 @@ if [ $stage -le 18 ]; then
     exit 1
   fi
 fi
-exit 0
+exit 0

egs/ami/s5b/local/chain/tuning/run_tdnn_1b.sh

@@ -247,7 +247,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_1c.sh

@@ -234,7 +234,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1a.sh

@@ -259,7 +259,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1b.sh

@@ -263,7 +263,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1c.sh

@@ -262,7 +262,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1d.sh

@@ -264,7 +264,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1e.sh

@@ -264,7 +264,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1f.sh

@@ -263,7 +263,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1g.sh

@@ -264,7 +264,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1h.sh

@@ -265,7 +265,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/ami/s5b/local/chain/tuning/run_tdnn_lstm_1i.sh

@@ -263,7 +263,7 @@ if [ $stage -le 17 ]; then
   # Note: it might appear that this data/lang_chain directory is mismatched, and it is as
   # far as the 'topo' is concerned, but this script doesn't read the 'topo' from
   # the lang directory.
-  utils/mkgraph.sh --left-biphone --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
+  utils/mkgraph.sh --self-loop-scale 1.0 data/lang_${LM} $dir $graph_dir
 fi
 
 if [ $stage -le 18 ]; then

egs/an4/s5/run.sh

@@ -83,7 +83,7 @@ fi
 # train monophone system 
 if [ $stage -le 4 ]; then
     steps/train_mono.sh --nj $nj --cmd "$train_cmd" data/train data/lang exp/mono
-    utils/mkgraph.sh --mono data/lang exp/mono exp/mono/graph
+    utils/mkgraph.sh data/lang exp/mono exp/mono/graph
     steps/decode.sh --config conf/decode.config --nj $nj --cmd "$decode_cmd" \
         exp/mono/graph data/test exp/mono/decode
 fi

egs/aurora4/s5/run.sh

@@ -51,7 +51,7 @@ done
 steps/train_mono.sh --boost-silence 1.25 --nj 10  \
   data/train_si84_multi data/lang exp/mono0a_multi || exit 1;
 #(
-# utils/mkgraph.sh --mono data/lang_test_tgpr exp/mono0a exp/mono0a/graph_tgpr && \
+# utils/mkgraph.sh data/lang_test_tgpr exp/mono0a exp/mono0a/graph_tgpr && \
 # steps/decode.sh --nj 8  \
 #   exp/mono0a/graph_tgpr data/test_eval92 exp/mono0a/decode_tgpr_eval92 
 #) &

egs/chime2/s5/run.sh

@@ -114,7 +114,7 @@ steps/train_mono.sh --boost-silence 1.25 --nj 10 --cmd "$train_cmd" \
 
 
 
-utils/mkgraph.sh --mono data/lang_test_tgpr_5k exp/mono0a exp/mono0a/graph_tgpr_5k
+utils/mkgraph.sh data/lang_test_tgpr_5k exp/mono0a exp/mono0a/graph_tgpr_5k
 #steps/decode.sh --nj 8  \
 #  exp/mono0a/graph_tgpr_5k data/test_eval92_5k_clean exp/mono0a/decode_tgpr_eval92_5k_clean
 steps/decode.sh --nj 8  --cmd "$train_cmd" \

egs/farsdat/s5/run.sh

@@ -65,7 +65,7 @@ echo ===========================================================================
 
 steps/train_mono.sh  --nj "$train_nj" --cmd "$train_cmd" data/train data/lang exp/mono
 
-utils/mkgraph.sh --mono data/lang_test_bg exp/mono exp/mono/graph
+utils/mkgraph.sh data/lang_test_bg exp/mono exp/mono/graph
 
 steps/decode.sh --nj "$decode_nj" --cmd "$decode_cmd" \
  exp/mono/graph data/dev exp/mono/decode_dev

egs/gale_arabic/s5/local/split_wer_per_corpus.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Report WER for reports and conversational
+# Copyright 2014 QCRI (author: Ahmed Ali)
+# Apache 2.0
+
+if [ $# -ne 1 ]; then
+   echo "Arguments should be the gale folder, see ../run.sh for example."
+   exit 1;
+fi
+
+[ -f ./path.sh ] && . ./path.sh
+
+#set -o pipefail -e
+
+galeFolder=$(readlink -f $1)
+symtab=./data/lang/words.txt
+
+min_lmwt=7
+max_lmwt=20
+
+for dir in exp/*/*decode*; do
+ for type in $(ls -1 local/test_list local/test.* | xargs -n1 basename); do
+ #echo "Processing: $dir $type"
+  rm -fr $dir/scoring_$type
+  mkdir -p $dir/scoring_$type/log
+  for x in $dir/scoring/*.tra $dir/scoring/test_filt.txt; do
+    cat $x | grep -f local/$type > $dir/scoring_$type/$(basename $x)
+  done
+
+  utils/run.pl LMWT=$min_lmwt:$max_lmwt $dir/scoring_$type/log/score.LMWT.log \
+     cat $dir/scoring_${type}/LMWT.tra \| \
+      utils/int2sym.pl -f 2- $symtab \| sed 's:\<UNK\>::g' \| \
+      compute-wer --text --mode=present \
+       ark:$dir/scoring_${type}/test_filt.txt  ark,p:- ">&" $dir/wer_${type}_LMWT
+done
+done
+
+time=$(date +"%Y-%m-%d-%H-%M-%S")
+echo "#RESULTS splits generated by $USER at $time"
+
+for type in $(ls -1 local/test_list local/test.* | xargs -n1 basename); do
+ echo -e "\n# WER $type"
+ for x in exp/*/*decode*; do
+  grep WER $x/wer_${type}_* | utils/best_wer.sh;
+ done | sort -n -k2
+done
+
+
+
+