Svd draft normalize

.gitignore

@@ -83,6 +83,7 @@ GSYMS
 /tools/ATLAS/
 /tools/atlas3.8.3.tar.gz
 /tools/irstlm/
+/tools/mitlm/
 /tools/openfst
 /tools/openfst-1.3.2.tar.gz
 /tools/openfst-1.3.2/

.travis.yml

@@ -49,7 +49,7 @@ script:
   # for the explanation why extra switches needed for clang with ccache.
   - CXX="ccache clang++-3.8 -Qunused-arguments -fcolor-diagnostics -Wno-tautological-compare"
     CFLAGS=""
-    LDFLAGS="-llapack"
+    LDFLAGS="-llapack -Wl,-fuse-ld=gold"
     INCDIRS="$XROOT/usr/include"
     LIBDIRS="$XROOT/usr/lib"
       tools/extras/travis_script.sh

egs/ami/s5/local/ami_ihm_scoring_data_prep.sh

@@ -87,18 +87,15 @@ sort -k 2 $dir/utt2spk | utils/utt2spk_to_spk2utt.pl > $dir/spk2utt || exit 1;
 join $dir/utt2spk $dir/segments | \
    perl -ne '{BEGIN{$pu=""; $pt=0.0;} split;
            if ($pu eq $_[1] && $pt > $_[3]) {
-             print "$_[0] $_[2] $_[3] $_[4]>$_[0] $_[2] $pt $_[4]\n"
+             print "s/^$_[0] $_[2] $_[3] $_[4]\$/$_[0] $_[2] $pt $_[4]/;\n"
            }
-           $pu=$_[1]; $pt=$_[4]; 
+           $pu=$_[1]; $pt=$_[4];
          }' > $dir/segments_to_fix
-if [ `cat $dir/segments_to_fix | wc -l` -gt 0 ]; then
+
+if [ -s $dir/segments_to_fix ]; then
   echo "$0. Applying following fixes to segments"
   cat $dir/segments_to_fix
-  while read line; do
-     p1=`echo $line | awk -F'>' '{print $1}'`
-     p2=`echo $line | awk -F'>' '{print $2}'`
-     sed -ir "s!$p1!$p2!" $dir/segments
-  done < $dir/segments_to_fix
+  perl -i -pf $dir/segments_to_fix $dir/segments
 fi
 
 # Copy stuff into its final locations

egs/ami/s5/local/ami_mdm_scoring_data_prep.sh

@@ -94,19 +94,15 @@ awk '{print $1}' $tmpdir/segments | \
 join $tmpdir/utt2spk_stm $tmpdir/segments | \
   awk '{ utt=$1; spk=$2; wav=$3; t_beg=$4; t_end=$5;
          if(spk_prev == spk && t_end_prev > t_beg) {
-           print utt, wav, t_beg, t_end">"utt, wav, t_end_prev, t_end;
+           print "s/^"utt, wav, t_beg, t_end"$/"utt, wav, t_end_prev, t_end"/;";
          }
          spk_prev=spk; t_end_prev=t_end;
        }' > $tmpdir/segments_to_fix
 
-if [ `cat $tmpdir/segments_to_fix | wc -l` -gt 0 ]; then
+if [ -s $tmpdir/segments_to_fix ]; then
   echo "$0. Applying following fixes to segments"
   cat $tmpdir/segments_to_fix
-  while read line; do
-     p1=`echo $line | awk -F'>' '{print $1}'`
-     p2=`echo $line | awk -F'>' '{print $2}'`
-     sed -ir "s:$p1:$p2:" $tmpdir/segments
-  done < $tmpdir/segments_to_fix
+  perl -i -pf $tmpdir/segments_to_fix $tmpdir/segments
 fi
 
 # Copy stuff into its final locations [this has been moved from the format_data

egs/ami/s5/local/ami_sdm_scoring_data_prep.sh

@@ -101,19 +101,15 @@ awk '{print $1}' $tmpdir/segments | \
 join $tmpdir/utt2spk_stm $tmpdir/segments | \
   awk '{ utt=$1; spk=$2; wav=$3; t_beg=$4; t_end=$5;
          if(spk_prev == spk && t_end_prev > t_beg) {
-           print utt, wav, t_beg, t_end">"utt, wav, t_end_prev, t_end;
+           print "s/^"utt, wav, t_beg, t_end"$/"utt, wav, t_end_prev, t_end"/;";
          }
          spk_prev=spk; t_end_prev=t_end;
        }' > $tmpdir/segments_to_fix
 
-if [ `cat $tmpdir/segments_to_fix | wc -l` -gt 0 ]; then
+if [ -s $tmpdir/segments_to_fix ]; then
   echo "$0. Applying following fixes to segments"
   cat $tmpdir/segments_to_fix
-  while read line; do
-     p1=`echo $line | awk -F'>' '{print $1}'`
-     p2=`echo $line | awk -F'>' '{print $2}'`
-     sed -ir "s:$p1:$p2:" $tmpdir/segments
-  done < $tmpdir/segments_to_fix
+  perl -i -pf $tmpdir/segments_to_fix $tmpdir/segments
 fi
 
 # Copy stuff into its final locations [this has been moved from the format_data

egs/ami/s5b/local/ami_ihm_scoring_data_prep.sh

@@ -93,18 +93,15 @@ sort -k 2 $dir/utt2spk | utils/utt2spk_to_spk2utt.pl > $dir/spk2utt || exit 1;
 join $dir/utt2spk $dir/segments | \
    perl -ne '{BEGIN{$pu=""; $pt=0.0;} split;
            if ($pu eq $_[1] && $pt > $_[3]) {
-             print "$_[0] $_[2] $_[3] $_[4]>$_[0] $_[2] $pt $_[4]\n"
+             print "s/^$_[0] $_[2] $_[3] $_[4]\$/$_[0] $_[2] $pt $_[4]/;\n"
            }
            $pu=$_[1]; $pt=$_[4];
          }' > $dir/segments_to_fix
-if [ `cat $dir/segments_to_fix | wc -l` -gt 0 ]; then
+
+if [ -s $dir/segments_to_fix ]; then
   echo "$0. Applying following fixes to segments"
   cat $dir/segments_to_fix
-  while read line; do
-     p1=`echo $line | awk -F'>' '{print $1}'`
-     p2=`echo $line | awk -F'>' '{print $2}'`
-     sed -ir "s!$p1!$p2!" $dir/segments
-  done < $dir/segments_to_fix
+  perl -i -pf $dir/segments_to_fix $dir/segments
 fi
 
 # Copy stuff into its final locations

egs/ami/s5b/local/ami_mdm_scoring_data_prep.sh

@@ -99,19 +99,15 @@ awk '{print $1}' $tmpdir/segments | \
 join $tmpdir/utt2spk_stm $tmpdir/segments | \
   awk '{ utt=$1; spk=$2; wav=$3; t_beg=$4; t_end=$5;
          if(spk_prev == spk && t_end_prev > t_beg) {
-           print utt, wav, t_beg, t_end">"utt, wav, t_end_prev, t_end;
+           print "s/^"utt, wav, t_beg, t_end"$/"utt, wav, t_end_prev, t_end"/;";
          }
          spk_prev=spk; t_end_prev=t_end;
        }' > $tmpdir/segments_to_fix
 
-if [ `cat $tmpdir/segments_to_fix | wc -l` -gt 0 ]; then
+if [ -s $tmpdir/segments_to_fix ]; then
   echo "$0. Applying following fixes to segments"
   cat $tmpdir/segments_to_fix
-  while read line; do
-     p1=`echo $line | awk -F'>' '{print $1}'`
-     p2=`echo $line | awk -F'>' '{print $2}'`
-     sed -ir "s:$p1:$p2:" $tmpdir/segments
-  done < $tmpdir/segments_to_fix
+  perl -i -pf $tmpdir/segments_to_fix $tmpdir/segments
 fi
 
 # Copy stuff into its final locations [this has been moved from the format_data

egs/ami/s5b/local/ami_sdm_scoring_data_prep.sh

@@ -111,25 +111,21 @@ awk '{print $1}' $tmpdir/segments | \
 join $tmpdir/utt2spk_stm $tmpdir/segments | \
   awk '{ utt=$1; spk=$2; wav=$3; t_beg=$4; t_end=$5;
          if(spk_prev == spk && t_end_prev > t_beg) {
-           print utt, wav, t_beg, t_end">"utt, wav, t_end_prev, t_end;
+           print "s/^"utt, wav, t_beg, t_end"$/"utt, wav, t_end_prev, t_end"/;";
          }
          spk_prev=spk; t_end_prev=t_end;
        }' > $tmpdir/segments_to_fix
 
-if [ `cat $tmpdir/segments_to_fix | wc -l` -gt 0 ]; then
+if [ -s $tmpdir/segments_to_fix ]; then
   echo "$0. Applying following fixes to segments"
   cat $tmpdir/segments_to_fix
-  while read line; do
-     p1=`echo $line | awk -F'>' '{print $1}'`
-     p2=`echo $line | awk -F'>' '{print $2}'`
-     sed -ir "s:$p1:$p2:" $tmpdir/segments
-  done < $tmpdir/segments_to_fix
+  perl -i -pf $tmpdir/segments_to_fix $tmpdir/segments
 fi
 
 # Copy stuff into its final locations [this has been moved from the format_data
 # script]
 mkdir -p $dir
-for f in spk2utt utt2spk utt2spk_stm wav.scp text segments reco2file_and_channel; do
+for f in segments_to_fix spk2utt utt2spk utt2spk_stm wav.scp text segments reco2file_and_channel; do
   cp $tmpdir/$f $dir/$f || exit 1;
 done
 

egs/aspire/s5/local/multi_condition/prepare_impulses_noises.sh

@@ -114,7 +114,7 @@ cp ${output_dir}_non_normalized/info/* $output_dir/info
 
 # rename file location in the noise-rir pairing files 
 for file in `ls $output_dir/info/noise_impulse*`; do
-  sed -i "s/_non_normalized//g" $file
+  perl -i -pe "s/_non_normalized//g" $file
 done
 
 # generating the rir-list with probabilities alloted for each rir

egs/babel/s5c/local/syllab/generate_syllable_lang.sh

@@ -118,8 +118,7 @@ ln -s lex.syllabs2phones.disambig.fst $out/L_disambig.fst
 echo "Validating the output lang dir"
 utils/validate_lang.pl $out || exit 1
 
-sed -i'' 's/#1$//g' $lout/lexicon.txt
-sed -i'' 's/#1$//g' $lout/lexiconp.txt
+perl -i -pe 's/#1$//g' $lout/lexicon.txt $lout/lexiconp.txt
 
 echo "Done OK."
 exit 0

egs/babel/s5d/conf/lang/404-georgian.FLP.official.conf

@@ -75,8 +75,8 @@ unsup_data_list=./conf/lists/404-georgian/untranscribed-training.list
 unsup_nj=32
 
 
-lexicon_file=
-lexiconFlags="--romanized --oov <unk>"
+lexicon_file=/export/corpora/LDC/LDC2016S12/IARPA_BABEL_OP3_404/conversational/reference_materials/lexicon.txt
+lexiconFlags=" --romanized --oov <unk>"
 
 
 

egs/babel/s5d/local/make_L_align.sh

@@ -34,18 +34,24 @@ tmpdir=$1
 dir=$2
 outdir=$3
 
+for f in  $dir/phones/optional_silence.txt $dir/phones.txt $dir/words.txt ; do
+  [ ! -f $f ] &&  echo "$0: The file $f must exist!" exit 1
+fi
+
 silphone=`cat $dir/phones/optional_silence.txt` || exit 1;
 
+if [ ! -f $tmpdir/lexicon.txt ] && [ ! -f $tmpdir/lexiconp.txt ] ; then
+  echo "$0: At least one of the files $tmpdir/lexicon.txt or $tmpdir/lexiconp.txt must exist" >&2
+  exit 1
+fi
+
 # Create lexicon with alignment info
 if  [ -f $tmpdir/lexicon.txt ] ; then
   cat $tmpdir/lexicon.txt | \
     awk '{printf("%s #1 ", $1); for (n=2; n <= NF; n++) { printf("%s ", $n); } print "#2"; }'
-elif [ -f $tmpdir/lexiconp.txt ] ;  then
+else
   cat $tmpdir/lexiconp.txt | \
     awk '{printf("%s #1 ", $1); for (n=3; n <= NF; n++) { printf("%s ", $n); } print "#2"; }'
-else
-  echo "Neither $tmpdir/lexicon.txt nor $tmpdir/lexiconp.txt does not exist"
-  exit 1
 fi | utils/make_lexicon_fst.pl - 0.5 $silphone | \
 fstcompile --isymbols=$dir/phones.txt --osymbols=$dir/words.txt \
   --keep_isymbols=false --keep_osymbols=false | \

egs/babel/s5d/local/syllab/generate_phone_lang.sh

@@ -122,8 +122,7 @@ ln -s lex.syllabs2phones.disambig.fst $out/L_disambig.fst
 echo "Validating the output lang dir"
 utils/validate_lang.pl $out || exit 1
 
-sed -i'' 's/#1$//g' $lout/lexicon.txt
-sed -i'' 's/#1$//g' $lout/lexiconp.txt
+perl -i -pe 's/#1$//g' $lout/lexicon.txt $lout/lexiconp.txt
 
 echo "Done OK."
 exit 0

egs/babel/s5d/local/syllab/generate_syllable_lang.sh

@@ -122,8 +122,7 @@ ln -s lex.syllabs2phones.disambig.fst $out/L_disambig.fst
 echo "Validating the output lang dir"
 utils/validate_lang.pl $out || exit 1
 
-sed -i'' 's/#1$//g' $lout/lexicon.txt
-sed -i'' 's/#1$//g' $lout/lexiconp.txt
+perl -i -pe 's/#1$//g' $lout/lexicon.txt $lout/lexiconp.txt
 
 echo "Done OK."
 exit 0

egs/bentham/v1/local/create_splits.sh

@@ -27,10 +27,8 @@ function split {
         echo $name $lines_dir"/"$name".png" >> $split_dir/images.scp
         echo $name $spkid >> $split_dir/utt2spk 
 	done < "$line_file"
-
-    sed -i '/^\s*$/d' $split_dir/images.scp
-    sed -i '/^\s*$/d' $split_dir/text
-    sed -i '/^\s*$/d' $split_dir/utt2spk
+
+    perl -i -ne 'print if /\S/' $split_dir/images.scp $split_dir/text $split_dir/utt2spk
     utils/utt2spk_to_spk2utt.pl $split_dir/utt2spk > $split_dir/spk2utt
 }
 

egs/bn_music_speech/v1/local/make_musan.py

@@ -45,7 +45,7 @@ def prepare_music(root_dir, use_vocals):
     else:
       print("Missing file {}".format(utt))
       num_bad_files += 1
-  print(("In music directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
+  print("In music directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
   return utt2spk_str, utt2wav_str
 
 def prepare_speech(root_dir):
@@ -71,7 +71,7 @@ def prepare_speech(root_dir):
     else:
       print("Missing file {}".format(utt))
       num_bad_files += 1
-  print(("In speech directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
+  print("In speech directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
   return utt2spk_str, utt2wav_str
 
 def prepare_noise(root_dir):
@@ -97,7 +97,7 @@ def prepare_noise(root_dir):
     else:
       print("Missing file {}".format(utt))
       num_bad_files += 1
-  print(("In noise directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
+  print("In noise directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
   return utt2spk_str, utt2wav_str
 
 def main():

egs/callhome_diarization/v1/diarization/nnet3/xvector/extract_xvectors.sh

@@ -102,7 +102,7 @@ if [ $stage -le 0 ]; then
   fi
   utils/data/get_uniform_subsegments.py \
       --max-segment-duration=$window \
-      --overlap-duration=$(echo "$window-$period" | bc) \
+      --overlap-duration=$(perl -e "print ($window-$period);") \
       --max-remaining-duration=$min_segment \
       --constant-duration=True \
       $segments > $dir/subsegments

egs/callhome_diarization/v1/local/make_musan.py

@@ -45,7 +45,7 @@ def prepare_music(root_dir, use_vocals):
     else:
       print("Missing file: {}".format(utt))
       num_bad_files += 1
-  print("In music directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files)
+  print("In music directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
   return utt2spk_str, utt2wav_str
 
 def prepare_speech(root_dir):
@@ -71,7 +71,7 @@ def prepare_speech(root_dir):
     else:
       print("Missing file: {}".format(utt))
       num_bad_files += 1
-  print("In speech directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files)
+  print("In speech directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
   return utt2spk_str, utt2wav_str
 
 def prepare_noise(root_dir):
@@ -97,7 +97,7 @@ def prepare_noise(root_dir):
     else:
       print("Missing file: {}".format(utt))
       num_bad_files += 1
-  print("In noise directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files)
+  print("In noise directory, processed {} files: {} had missing wav data".format(num_good_files, num_bad_files))
   return utt2spk_str, utt2wav_str
 
 def main():

egs/callhome_diarization/v1/run.sh

@@ -188,7 +188,7 @@ if [ $stage -le 6 ]; then
 
       der=$(grep -oP 'DIARIZATION\ ERROR\ =\ \K[0-9]+([.][0-9]+)?' \
         exp/tuning/${dataset}_t${threshold})
-      if [ $(echo $der'<'$best_der | bc -l) -eq 1 ]; then
+      if [ $(perl -e "print ($der < $best_der ? 1 : 0);") -eq 1 ]; then
         best_der=$der
         best_threshold=$threshold
       fi

egs/callhome_diarization/v2/run.sh

@@ -115,7 +115,7 @@ if [ $stage -le 2 ]; then
 
   # Make a reverberated version of the SWBD+SRE list.  Note that we don't add any
   # additive noise here.
-  python steps/data/reverberate_data_dir.py \
+  steps/data/reverberate_data_dir.py \
     "${rvb_opts[@]}" \
     --speech-rvb-probability 1 \
     --pointsource-noise-addition-probability 0 \
@@ -140,11 +140,11 @@ if [ $stage -le 2 ]; then
   done
 
   # Augment with musan_noise
-  python steps/data/augment_data_dir.py --utt-suffix "noise" --fg-interval 1 --fg-snrs "15:10:5:0" --fg-noise-dir "data/musan_noise" data/train data/train_noise
+  steps/data/augment_data_dir.py --utt-suffix "noise" --fg-interval 1 --fg-snrs "15:10:5:0" --fg-noise-dir "data/musan_noise" data/train data/train_noise
   # Augment with musan_music
-  python steps/data/augment_data_dir.py --utt-suffix "music" --bg-snrs "15:10:8:5" --num-bg-noises "1" --bg-noise-dir "data/musan_music" data/train data/train_music
+  steps/data/augment_data_dir.py --utt-suffix "music" --bg-snrs "15:10:8:5" --num-bg-noises "1" --bg-noise-dir "data/musan_music" data/train data/train_music
   # Augment with musan_speech
-  python steps/data/augment_data_dir.py --utt-suffix "babble" --bg-snrs "20:17:15:13" --num-bg-noises "3:4:5:6:7" --bg-noise-dir "data/musan_speech" data/train data/train_babble
+  steps/data/augment_data_dir.py --utt-suffix "babble" --bg-snrs "20:17:15:13" --num-bg-noises "3:4:5:6:7" --bg-noise-dir "data/musan_speech" data/train data/train_babble
 
   # Combine reverb, noise, music, and babble into one directory.
   utils/combine_data.sh data/train_aug data/train_reverb data/train_noise data/train_music data/train_babble
@@ -297,7 +297,7 @@ if [ $stage -le 10 ]; then
 
       der=$(grep -oP 'DIARIZATION\ ERROR\ =\ \K[0-9]+([.][0-9]+)?' \
         $nnet_dir/tuning/${dataset}_t${threshold})
-      if [ $(echo $der'<'$best_der | bc -l) -eq 1 ]; then
+      if [ $(perl -e "print ($der < $best_der ? 1 : 0);") -eq 1 ]; then
         best_der=$der
         best_threshold=$threshold
       fi

egs/callhome_egyptian/s5/local/callhome_prepare_dict.sh

@@ -54,9 +54,8 @@ cat $dir/silence_phones.txt| awk '{printf("%s ", $1);} END{printf "\n";}' > \
 $dir/extra_questions.txt || exit 1;
 
 # Add prons for laughter, noise, oov
-for w in `grep -v sil $dir/silence_phones.txt`; do
-sed -i "/\[$w\]/d" $tmpdir/lexicon.3
-done
+w=$(grep -v sil $dir/silence_phones.txt | tr '\n' '|')
+perl -i -ne "print unless /\[(${w%?})\]/"  $tmpdir/lexicon.3
 
 for w in `grep -v sil $dir/silence_phones.txt`; do
 echo "[$w] $w"

egs/callhome_egyptian/s5/local/ctm.sh

@@ -18,9 +18,9 @@ fi
 steps/get_ctm.sh $data_dir $lang_dir $decode_dir
 
 # Make sure that channel markers match
-#sed -i "s:\s.*_fsp-([AB]): \1:g" data/dev/stm
-#ls exp/tri5a/decode_dev/score_*/dev.ctm | xargs -I {} sed -i -r 's:fsp\s1\s:fsp A :g' {}
-#ls exp/tri5a/decode_dev/score_*/dev.ctm | xargs -I {} sed -i -r 's:fsp\s2\s:fsp B :g' {}
+#perl -i -pe "s:\s.*_fsp-([AB]): \1:g" data/dev/stm
+#ls exp/tri5a/decode_dev/score_*/dev.ctm | xargs -I {} perl -i -pe 's:fsp\s1\s:fsp A :g' {}
+#ls exp/tri5a/decode_dev/score_*/dev.ctm | xargs -I {} perl -i -pe 's:fsp\s2\s:fsp B :g' {}
 
 # Get the environment variables
 . /export/babel/data/software/env.sh

egs/csj/s5/local/csj_make_trans/csj_autorun.sh

@@ -61,7 +61,7 @@ if [ ! -e $outd/.done_make_trans ];then
                 mkdir -p $outd/$vol/$id
 
                 case "$csjv" in
-                    "usb" ) TPATH="$resource/${SDB}$vol" ; WPATH="$resource/$WAV" ;;
+                    "usb" ) TPATH="$resource/${SDB}$vol" ; WPATH="$resource/${WAV}$vol" ;;
                     "dvd" ) TPATH="$resource/$vol/$id"   ; WPATH="$resource/$vol/$id" ;;
                     "merl" ) TPATH="$resource/$vol/$SDB" ; WPATH="$resource/$vol/$WAV" ;;
                 esac