diff --git a/egs/swbd/s5c/local/swbd1_prepare_dict.sh b/egs/swbd/s5c/local/swbd1_prepare_dict.sh index f9059b7df41..dadd33e7954 100755 --- a/egs/swbd/s5c/local/swbd1_prepare_dict.sh +++ b/egs/swbd/s5c/local/swbd1_prepare_dict.sh @@ -19,6 +19,7 @@ srcdict=$srcdir/swb_ms98_transcriptions/sw-ms98-dict.text [ ! -f "$srcdict" ] && echo "$0: No such file $srcdict" && exit 1; cp $srcdict $dir/lexicon0.txt || exit 1; +chmod +r $dir/lexicon0.txt # fix a strange permission in the source. patch kaldi.sourceforge.net is only a backup - location. kaldi-asr.org/doc is the - definitive location of this documentation. Kaldi's code repository is - now located at http://github.com/kaldi-asr/kaldi + Kaldi is a toolkit for speech recognition, intended for use by speech recognition researchers and + professionals. + + Find the code repository at http://github.com/kaldi-asr/kaldi. - See also the top level of kaldi-asr.org, where - you can download pre-built models.

- \subpage about diff --git a/src/ivectorbin/select-voiced-frames.cc b/src/ivectorbin/select-voiced-frames.cc index beba1a4f068..18e6009de72 100644 --- a/src/ivectorbin/select-voiced-frames.cc +++ b/src/ivectorbin/select-voiced-frames.cc @@ -33,11 +33,12 @@ int main(int argc, char *argv[]) { const char *usage = "Select a subset of frames of the input files, based on the output of\n" "compute-vad or a similar program (a vector of length num-frames,\n" - "containing 1.0 for voiced, 0.0 for unvoiced).\n" + "containing 1.0 for voiced, 0.0 for unvoiced). Caution: this is\n" + "mostly useful only in speaker identification applications.\n" "Usage: select-voiced-frames [options] " " \n" "E.g.: select-voiced-frames [options] scp:feats.scp scp:vad.scp ark:-\n"; - + ParseOptions po(usage); po.Read(argc, argv); @@ -45,17 +46,17 @@ int main(int argc, char *argv[]) { po.PrintUsage(); exit(1); } - + std::string feat_rspecifier = po.GetArg(1), vad_rspecifier = po.GetArg(2), feat_wspecifier = po.GetArg(3); - + SequentialBaseFloatMatrixReader feat_reader(feat_rspecifier); RandomAccessBaseFloatVectorReader vad_reader(vad_rspecifier); BaseFloatMatrixWriter feat_writer(feat_wspecifier); int32 num_done = 0, num_err = 0; - + for (;!feat_reader.Done(); feat_reader.Next()) { std::string utt = feat_reader.Key(); const Matrix &feat = feat_reader.Value(); @@ -72,8 +73,8 @@ int main(int argc, char *argv[]) { const Vector &voiced = vad_reader.Value(utt); if (feat.NumRows() != voiced.Dim()) { - KALDI_WARN << "Mismatch in number for frames " << feat.NumRows() - << " for features and VAD " << voiced.Dim() + KALDI_WARN << "Mismatch in number for frames " << feat.NumRows() + << " for features and VAD " << voiced.Dim() << ", for utterance " << utt; num_err++; continue; @@ -111,5 +112,3 @@ int main(int argc, char *argv[]) { return -1; } } - - diff --git a/src/lat/compose-lattice-pruned.cc b/src/lat/compose-lattice-pruned.cc index f37e72ba1af..46c44d76595 100644 --- a/src/lat/compose-lattice-pruned.cc +++ b/src/lat/compose-lattice-pruned.cc @@ -126,7 +126,7 @@ class PrunedCompactLatticeComposer { // 'arc_delta_costs' is an array, one for each arc (and the final-prob, if // present), showing how much the cost to the final-state for the best path - // starting in this state and exiting through each arc (or final-state), + // starting in this state and exiting through each arc (or final-prob), // differs from 'backward_cost'. Specifically, it contains pairs // (delta_cost, arc_index), where delta_cost >= 0 and arc_index is // either the index into this state's array of arcs (for arcs), or -1 @@ -201,8 +201,8 @@ class PrunedCompactLatticeComposer { // - If backward_cost is finite (this state in the composed result can // reach the final state via currently expanded states), then // delta_backward_cost is this->backward_cost minus - // lat_state_info_[lat_state].backward_cost. (It will mostly, - // not always, be >= 0, reflecting that the new LM is better than + // lat_state_info_[lat_state].backward_cost. (It will mostly, but + // not always, be <= 0, reflecting that the new LM is better than // the old LM). // - On the other hand, if backward_cost is infinite: delta_backward_cost // is set to the delta_backward_cost of the previous state on the best @@ -216,7 +216,10 @@ class PrunedCompactLatticeComposer { // - For other states, delta_backward_cost will be unchanged since // RecomputePruningInfo() was last called. // The above rules may make the delta_backward_cost a less accurate, but - // still probably reasonable, heuristic. + // still probably reasonable, heuristic. What it is a heuristic for, + // is: if we were to successfully reach an end-state of the composed output + // from this state, what would be the resulting backward_cost + // minus lat_state_info_[lat_state].backward_cost. BaseFloat delta_backward_cost; // 'prev_composed_state' is the previous state on the best path from @@ -226,14 +229,14 @@ class PrunedCompactLatticeComposer { // used. int32 prev_composed_state; - // 'sorted_arc_index' is an index into the 'arc_delta_costs' array of the - // LatticeStateInfo corresponding to 'lat_state'. It corresponds to the - // next arc (or final-prob) out of the input lattice that we have yet to - // expand in the composition; or -1 if we have expanded all of them. When - // we first reach a composed state, 'sorted_arc_index' will be zero; then it - // will increase one at a time as we expand arcs until either the - // composition terminates or we have expanded all the arcs and it becomes - // -1. + // 'sorted_arc_index' is an index into the 'arc_delta_costs' array which is + // a member of the LatticeStateInfo object corresponding to the lattice + // state 'lat_state'. It corresponds to the next arc (or final-prob) out of + // the input lattice that we have yet to expand in the composition; or -1 if + // we have expanded all of them. When we first reach a composed state, + // 'sorted_arc_index' will be zero; then it will increase one at a time as + // we expand arcs until either the composition terminates or we have + // expanded all the arcs and it becomes -1. int32 sorted_arc_index; // 'arc_delta_cost' is a derived quantity that we store here for easier @@ -241,6 +244,11 @@ class PrunedCompactLatticeComposer { // if sorted_arc_index >= 0, then: // arc_delta_cost == this_lat_info.arc_delta_costs[sorted_arc_index].first // else: arc_delta_cost == +infinity. + // + // what 'arc_delta_cost' represents (or is a heuristic for), is the expected + // cost of a path to the final-state leaving through the arc we're about to + // expand, minus the expected cost of any path to the final-state starting + // from this state. BaseFloat arc_delta_cost; // view 'expected_cost_offset' a phantom field of this struct, that has @@ -249,8 +257,11 @@ class PrunedCompactLatticeComposer { // // 'expected_cost_offset' is a derived quantity that reflects the expected // cost (according to our heuristic) of the best path we might encounter - // when expanding the next previously unseen arc (or final-prob), corresponding - // to 'sorted_arc_index'. + // when expanding the next previously unseen arc (or final-prob), + // corresponding to 'sorted_arc_index'. (This is the expected cost of a + // successful path, from the beginning to the end of the lattice, but + // constrained to be a path that contains the arc we're about to expand). + // // The 'offset' part is about subtracting the best cost of the lattice, so we // can cast to float without too much loss of accuracy: // expected_cost_offset = expected_cost - lat_best_cost_. @@ -258,7 +269,8 @@ class PrunedCompactLatticeComposer { // We define expected_cost_offset by defining the 'expected_cost' part; // for clarity: // First, let lat_backward_cost equal the backward_cost of the LatticeStateInfo - // corresponding to 'lat_state'. Then: + // corresponding to 'lat_state', i.e. + // lat_backward_cost = lat_state_info_[lat_state].backward_cost. Then: // expected_cost = forward_cost + lat_backward_cost + // delta_backward_cost + arc_delta_cost. // expected_cost_offset will always equal the above minus lat_best_cost_. @@ -298,7 +310,7 @@ class PrunedCompactLatticeComposer { // current_cutoff_ is a value used in deciding which composed states // need to be included in the queue. Each time RecomputePruningInfo() - // called, it is set to + // called, current_cutoff_ is set to // (output_best_cost_ - lat_best_cost_ + opts_.lattice_compose_beam). // It will be +infinity if the output lattice doesn't yet have any // successful paths. It decreases with time. You can compare the @@ -755,8 +767,9 @@ void PrunedCompactLatticeComposer::ProcessTransition(int32 src_composed_state, std::pair ret = pair_to_state_.insert(value); if (ret.second) { - // Successfully inserted. Most of the rest of this block deals with the - // consequences of adding a new state. + // Successfully inserted: this dest-state did not already exist. Most of + // the rest of this block deals with the consequences of adding a new + // state. int32 ans = clat_out_->AddState(); KALDI_ASSERT(ans == new_composed_state); dest_composed_state = new_composed_state; @@ -777,7 +790,8 @@ void PrunedCompactLatticeComposer::ProcessTransition(int32 src_composed_state, dest_info->delta_backward_cost = src_info->delta_backward_cost; // The 'prev_composed_state' field will not be read again until after it's - // overwritten; we set it as below only for debugging purposes. + // overwritten; we set it as below only for debugging purposes (the + // negation is also for debugging purposes). dest_info->prev_composed_state = -src_composed_state; dest_info->sorted_arc_index = 0; dest_info->arc_delta_cost = 0.0; @@ -791,8 +805,11 @@ void PrunedCompactLatticeComposer::ProcessTransition(int32 src_composed_state, dest_info->delta_backward_cost - lat_best_cost_); if (expected_cost_offset < current_cutoff_) { - composed_state_queue_.push(std::pair( - expected_cost_offset, dest_composed_state)); + // the following call should be equivalent to + // composed_state_queue_.push(std::pair(...)) with + // the same pair of args. + composed_state_queue_.emplace(expected_cost_offset, + dest_composed_state); } } else { // the destination composed state already existed. dest_composed_state = ret.first->second; diff --git a/tools/extras/install_sequitur.sh b/tools/extras/install_sequitur.sh index c6d06592408..65f82a14134 100755 --- a/tools/extras/install_sequitur.sh +++ b/tools/extras/install_sequitur.sh @@ -62,10 +62,19 @@ if [ ! -d ./sequitur-g2p ] ; then echo >&2 " manually and re-run the script" fi } +else + echo >&2 "$0: Updating the repository -- we will try to merge with local changes (if you have any)" + ( + cd sequitur-g2p/ + git pull + # this would work also, but would drop all local modifications + #git fetch + #git reset --hard origin/master + ) || { + echo >&2 "Failed to do git pull, delete the sequitur dir and run again"; + exit 1 + } fi -#just to retain backward compatibility for a while. Can be removed -#in a couple of months. -ln -sf sequitur-g2p sequitur ( cd sequitur-g2p @@ -75,15 +84,21 @@ cd sequitur-g2p #the primary issue is that real GNU GCC does not accept that switch #in addition, Apple fake g++ based on LLVM version 8.1 prints warning about #the libstdc++ should no longer be used. -if (g++ --version 2>/dev/null | grep -s "LLVM version 8.0" >/dev/null) ; then +if (g++ --version 2>/dev/null | grep -s "LLVM version 8.0" >/dev/null) ; then #Apple fake-g++ make CXX=g++ CC=gcc CPPFLAGS="-stdlib=libstdc++" else make CXX=g++ CC=gcc fi -python setup.py install --prefix `pwd` +# the next two lines deal with the issue that the new setup tools +# expect the directory in which we will be installing to be visible +# as module directory to python +site_packages_dir=$(python -m site --user-site | grep -oE "lib.*") +SEQUITUR=$(pwd)/$site_packages_dir +PYTHONPATH=${PYTHONPATH:-}:$SEQUITUR python setup.py install --prefix `pwd` ) + site_packages_dir=$(cd sequitur-g2p; find ./lib{,64} -type d -name site-packages | head -n 1) ( set +u