[src,scripts,egs] Updated TDNN-F recipes incorporating resnet skip connections.

egs/ami/s5b/local/nnet3/run_tdnn.sh

@@ -23,7 +23,6 @@
 # local/nnet3/run_tdnn.sh --mic sdm1 --use-ihm-ali true --affix _cleaned2 --gmm tri4a --train-set train_cleaned2 &
 
 
-
 set -e -o pipefail -u
 
 # First the options that are passed through to run_ivector_common.sh

egs/heroico/s5/RESULTS

@@ -1,27 +1,22 @@
-%WER 75.78 [ 6983 / 9215, 1377 ins, 507 del, 5099 sub ] exp/tri3b/decode_nonnative.si/wer_17_1.0
-%WER 74.25 [ 5680 / 7650, 1187 ins, 431 del, 4062 sub ] exp/tri3b/decode_devtest.si/wer_16_1.0
-%WER 73.85 [ 6805 / 9215, 517 ins, 1653 del, 4635 sub ] exp/chain/tdnn1c_sp/decode_nonnative/wer_7_1.0
-%WER 73.76 [ 12328 / 16713, 2541 ins, 804 del, 8983 sub ] exp/tri3b/decode_test.si/wer_17_1.0
-%WER 73.20 [ 6745 / 9215, 518 ins, 1566 del, 4661 sub ] exp/chain/tdnn1c_sp_online/decode_nonnative/wer_7_1.0
-%WER 71.86 [ 5497 / 7650, 530 ins, 959 del, 4008 sub ] exp/mono/decode_devtest/wer_7_1.0
-%WER 71.64 [ 6602 / 9215, 646 ins, 939 del, 5017 sub ] exp/mono/decode_nonnative/wer_7_1.0
-%WER 71.26 [ 5343 / 7498, 1159 ins, 293 del, 3891 sub ] exp/tri3b/decode_native.si/wer_17_1.0
-%WER 69.84 [ 11673 / 16713, 986 ins, 2523 del, 8164 sub ] exp/chain/tdnn1c_sp/decode_test/wer_7_1.0
-%WER 69.59 [ 11630 / 16713, 1153 ins, 1643 del, 8834 sub ] exp/mono/decode_test/wer_7_1.0
-%WER 69.10 [ 11548 / 16713, 976 ins, 2402 del, 8170 sub ] exp/chain/tdnn1c_sp_online/decode_test/wer_7_1.0
-%WER 67.09 [ 6182 / 9215, 907 ins, 626 del, 4649 sub ] exp/tri1/decode_nonnative/wer_14_1.0
-%WER 66.98 [ 5022 / 7498, 503 ins, 700 del, 3819 sub ] exp/mono/decode_native/wer_7_1.0
-%WER 66.78 [ 6154 / 9215, 1048 ins, 537 del, 4569 sub ] exp/tri2b/decode_nonnative/wer_15_1.0
-%WER 66.64 [ 6141 / 9215, 1226 ins, 425 del, 4490 sub ] exp/tri3b/decode_nonnative/wer_16_1.0
-%WER 66.33 [ 5074 / 7650, 921 ins, 481 del, 3672 sub ] exp/tri1/decode_devtest/wer_11_1.0
-%WER 66.30 [ 5072 / 7650, 1198 ins, 328 del, 3546 sub ] exp/tri3b/decode_devtest/wer_11_1.0
-%WER 65.88 [ 5040 / 7650, 985 ins, 450 del, 3605 sub ] exp/tri2b/decode_devtest/wer_13_1.0
-%WER 65.05 [ 10872 / 16713, 1725 ins, 959 del, 8188 sub ] exp/tri1/decode_test/wer_13_1.0
-%WER 64.76 [ 4856 / 7498, 461 ins, 862 del, 3533 sub ] exp/chain/tdnn1c_sp/decode_native/wer_7_1.0
-%WER 64.45 [ 10772 / 16713, 2261 ins, 698 del, 7813 sub ] exp/tri3b/decode_test/wer_16_1.0
-%WER 64.33 [ 10751 / 16713, 1955 ins, 845 del, 7951 sub ] exp/tri2b/decode_test/wer_14_1.0
-%WER 64.03 [ 4801 / 7498, 463 ins, 825 del, 3513 sub ] exp/chain/tdnn1c_sp_online/decode_native/wer_7_1.0
-%WER 62.54 [ 4689 / 7498, 781 ins, 379 del, 3529 sub ] exp/tri1/decode_native/wer_13_1.0
-%WER 61.66 [ 4623 / 7498, 1038 ins, 267 del, 3318 sub ] exp/tri3b/decode_native/wer_15_1.0
-%WER 61.28 [ 4595 / 7498, 899 ins, 309 del, 3387 sub ] exp/tri2b/decode_native/wer_13_1.0
-john@A-TEAM19054:~/work/kaldi/egs/heroico/s5$ 
+# for dir in $(echo exp/tri*/decode* | grep -v 'si/'); do grep WER $dir/wer* | utils/best_wer.sh; done
+
+%WER 67.01 [ 5126 / 7650, 837 ins, 575 del, 3714 sub ] exp/tri1/decode_devtest/wer_14_1.0
+%WER 62.39 [ 4678 / 7498, 768 ins, 397 del, 3513 sub ] exp/tri1/decode_native/wer_13_1.0
+%WER 67.05 [ 6179 / 9215, 895 ins, 606 del, 4678 sub ] exp/tri1/decode_nonnative/wer_13_1.0
+%WER 64.97 [ 10859 / 16713, 1678 ins, 999 del, 8182 sub ] exp/tri1/decode_test/wer_13_1.0
+%WER 65.90 [ 5041 / 7650, 1016 ins, 416 del, 3609 sub ] exp/tri2b/decode_devtest/wer_12_1.0
+%WER 61.26 [ 4593 / 7498, 908 ins, 300 del, 3385 sub ] exp/tri2b/decode_native/wer_14_1.0
+%WER 67.51 [ 6221 / 9215, 1085 ins, 524 del, 4612 sub ] exp/tri2b/decode_nonnative/wer_14_1.0
+%WER 64.87 [ 10842 / 16713, 2004 ins, 838 del, 8000 sub ] exp/tri2b/decode_test/wer_14_1.0
+%WER 66.09 [ 5056 / 7650, 1078 ins, 402 del, 3576 sub ] exp/tri3b/decode_devtest/wer_16_1.0
+%WER 74.88 [ 5728 / 7650, 1210 ins, 426 del, 4092 sub ] exp/tri3b/decode_devtest.si/wer_15_1.0
+%WER 61.19 [ 4588 / 7498, 1038 ins, 255 del, 3295 sub ] exp/tri3b/decode_native/wer_14_1.0
+%WER 70.99 [ 5323 / 7498, 1185 ins, 301 del, 3837 sub ] exp/tri3b/decode_native.si/wer_16_1.0
+%WER 66.35 [ 6114 / 9215, 1186 ins, 421 del, 4507 sub ] exp/tri3b/decode_nonnative/wer_17_1.0
+%WER 76.36 [ 7037 / 9215, 1420 ins, 467 del, 5150 sub ] exp/tri3b/decode_nonnative.si/wer_16_1.0
+%WER 64.06 [ 10706 / 16713, 2245 ins, 657 del, 7804 sub ] exp/tri3b/decode_test/wer_15_1.0
+%WER 73.97 [ 12362 / 16713, 2608 ins, 766 del, 8988 sub ] exp/tri3b/decode_test.si/wer_16_1.0
+%WER 53.07 [ 4060 / 7650, 744 ins, 376 del, 2940 sub ] exp/chain/tdnn1e_sp/decode_devtest/wer_7_1.0
+%WER 54.47 [ 4084 / 7498, 536 ins, 475 del, 3073 sub ] exp/chain/tdnn1e_sp/decode_native/wer_7_1.0
+%WER 63.01 [ 5806 / 9215, 685 ins, 784 del, 4337 sub ] exp/chain/tdnn1e_sp/decode_nonnative/wer_7_1.0
+%WER 59.25 [ 9903 / 16713, 1226 ins, 1259 del, 7418 sub ] exp/chain/tdnn1e_sp/decode_test/wer_7_1.0

egs/heroico/s5/cmd.sh

@@ -10,8 +10,8 @@
 # conf/queue.conf in http://kaldi-asr.org/doc/queue.html for more information,
 # or search for the string 'default_config' in utils/queue.pl or utils/slurm.pl.
 
-export train_cmd=queue.pl
-export decode_cmd="queue.pl --mem 2G"
+export train_cmd="retry.pl queue.pl"
+export decode_cmd="retry.pl queue.pl --mem 2G"
 
 #  queue_conf=$HOME/queue_conf/default.conf # see example /homes/kazi/iveselyk/queue_conf/default.conf,
 #  export train_cmd="queue.pl --config $queue_conf --mem 2G --matylda 0.2"

egs/heroico/s5/local/chain/compare_wer.sh

@@ -58,31 +58,25 @@ set_names() {
 
 
 
-echo -n "# System               "
+echo -n "# System                 "
 for x in $*; do   printf "% 10s" " $(basename $x)";   done
 echo
 
-strings=(
-    "#WER devtest  "
-    "#WER native  "
-    nonnative
-  "#WER test  ")
+test_sets=(devtest test native nonnative)
 
-for n in 0 1 2 3; do
-   echo -n "${strings[$n]}"
+for t in ${test_sets[@]}; do
+   printf '# %%WER % 14s  ' $t
    for x in $*; do
      set_names $x  # sets $dirname and $epoch_infix
-    decode_names=(devtest native nonnative test)
-
-     wer=$(cat $dirname/decode_${decode_names[$n]}/wer_* | utils/best_wer.sh | awk '{print $2}')
+     wer=$(cat $dirname/decode_$t/wer_* | utils/best_wer.sh | awk '{print $2}')
      printf "% 10s" $wer
    done
    echo
    if $include_looped; then
      echo -n "#             [looped:]    "
      for x in $*; do
        set_names $x  # sets $dirname and $epoch_infix
-       wer=$(cat $dirname/decode_looped_${decode_names[$n]}/wer_* | utils/best_wer.sh | awk '{print $2}')
+       wer=$(cat $dirname/decode_looped_$t/wer_* | utils/best_wer.sh | awk '{print $2}')
        printf "% 10s" $wer
      done
      echo
@@ -91,7 +85,7 @@ for n in 0 1 2 3; do
      echo -n "#             [online:]    "
      for x in $*; do
        set_names $x  # sets $dirname and $epoch_infix
-       wer=$(cat ${dirname}_online/decode_${decode_names[$n]}/wer_* | utils/best_wer.sh | awk '{print $2}')
+       wer=$(cat ${dirname}_online/decode_$t/wer_* | utils/best_wer.sh | awk '{print $2}')
        printf "% 10s" $wer
      done
      echo
@@ -104,14 +98,14 @@ if $used_epochs; then
 fi
 
 
-echo -n "# Final train prob     "
+echo -n "# Final train prob       "
 for x in $*; do
   prob=$(grep Overall $x/log/compute_prob_train.final.log | grep -v xent | awk '{printf("%.4f", $8)}')
   printf "% 10s" $prob
 done
 echo
 
-echo -n "# Final valid prob     "
+echo -n "# Final valid prob       "
 for x in $*; do
   prob=$(grep Overall $x/log/compute_prob_valid.final.log | grep -v xent | awk '{printf("%.4f", $8)}')
   printf "% 10s" $prob
@@ -131,3 +125,9 @@ for x in $*; do
   printf "% 10s" $prob
 done
 echo
+
+echo -n "# Num-params              "
+for x in $*; do
+  printf "% 10s" $(grep num-parameters $x/log/progress.1.log | awk '{print $2}')
+done
+echo

egs/heroico/s5/local/chain/run_tdnn.sh

@@ -1 +1 @@
-tuning/run_tdnn_1a.sh
+tuning/run_tdnn_1b.sh

egs/heroico/s5/local/chain/tuning/run_tdnn_1a.sh

@@ -1,42 +1,20 @@
 #!/bin/bash
-# 1e
-# lower number of epochs to 7 from 10 (avoid overfitting?)
-
-# compare with 1d
-# ./local/chain/compare_wer.sh exp/chain/tdnn1d_sp exp/chain/tdnn1e_sp
-# System                tdnn1d_sp tdnn1e_sp
-#WER devtest       52.78     52.21
-#WER native       55.32     53.43
-nonnative     64.35     61.03
-# test       60.28     57.70
-# Final train prob        -0.0229   -0.0250
-# Final valid prob        -0.0683   -0.0678
-# Final train prob (xent)   -0.7525   -0.7887
-# Final valid prob (xent)   -1.0296   -1.0419
-
-# info
-#exp/chain/tdnn1e_sp:
-# num-iters=105
-# nj=1..1
-# num-params=6.6M
-# dim=40+100->1392
-# combine=-0.036->-0.033
-# xent:train/valid[69,104,final]=(-1.20,-0.917,-0.789/-1.35,-1.16,-1.04)
-# logprob:train/valid[69,104,final]=(-0.049,-0.030,-0.025/-0.082,-0.075,-0.068)
-
-# Word Error Rates on folds
-%WER 61.03 [ 5624 / 9215, 630 ins, 727 del, 4267 sub ] exp/chain/tdnn1e_sp/decode_nonnative/wer_8_1.0
-%WER 57.70 [ 9644 / 16713, 1249 ins, 1040 del, 7355 sub ] exp/chain/tdnn1e_sp/decode_test/wer_7_1.0
-%WER 53.43 [ 4006 / 7498, 558 ins, 408 del, 3040 sub ] exp/chain/tdnn1e_sp/decode_native/wer_7_1.0
-%WER 52.21 [ 3994 / 7650, 585 ins, 456 del, 2953 sub ] exp/chain/tdnn1e_sp/decode_devtest/wer_9_1.0
-
-# | fold | 1a | 1b | 1c | 1d | 1e |
-#| devtest | 54.46 | 54.20 | 54.16 | 52.78 | 52.21 |
-#| native |  62.14 | 62.32 | 61.70 | 55.32 | 53.43 |
-#| nonnative | 70.58 | 71.20 | 71.68 | 64.35 | 61.03 |
-#| test | 66.85 | 67.21 | 67.25 | 60.28 | 57.70 |
-
-# this script came from the mini librispeech recipe
+
+# local/chain/compare_wer.sh exp/chain/tdnn1a_sp
+# System                  tdnn1a_sp
+# %WER        devtest       53.07
+# %WER           test       59.25
+# %WER         native       54.47
+# %WER      nonnative       63.01
+# Final train prob          -0.0253
+# Final valid prob          -0.0687
+# Final train prob (xent)   -0.7715
+# Final valid prob (xent)   -1.0719
+# Num-params                 6567648
+
+# steps/info/chain_dir_info.pl  exp/chain/tdnn1a_sp/
+#exp/chain/tdnn1a_sp/: num-iters=105 nj=1..1 num-params=6.6M dim=40+100->1392 combine=-0.040->-0.033 (over 7) xent:train/valid[69,104,final]=(-1.12,-0.880,-0.771/-1.33,-1.21,-1.07) logprob:train/valid[69,104,final]=(-0.050,-0.031,-0.025/-0.079,-0.080,-0.069)
+
 # Set -e here so that we catch if any executable fails immediately
 set -euo pipefail
 
@@ -51,7 +29,7 @@ nnet3_affix=
 
 # The rest are configs specific to this script.  Most of the parameters
 # are just hardcoded at this level, in the commands below.
-affix=1e   # affix for the TDNN directory name
+affix=1a   # affix for the TDNN directory name
 tree_affix=
 train_stage=-10
 get_egs_stage=-10
@@ -313,13 +291,9 @@ if $test_online_decoding && [ $stage -le 17 ]; then
     # note: we just give it "data/${data}" as it only uses the wav.scp, the
     # feature type does not matter.
     steps/online/nnet3/decode.sh \
-      --acwt 1.0 \
-      --post-decode-acwt 10.0 \
-      --nj $nspk \
-      --cmd "$decode_cmd" \
-      $tree_dir/graph \
-      data/${data} \
-      ${dir}_online/decode_${data} || exit 1
+      --acwt 1.0 --post-decode-acwt 10.0 \
+      --nj $nspk --cmd "$decode_cmd" \
+      $tree_dir/graph data/${data} ${dir}_online/decode_${data} || exit 1
     ) || touch $dir/.error &
   done
   wait