Update README.md

Author: jimbozhang

.gitignore

@@ -1,4 +1,13 @@
+#common
+.vscode
 build
+*.zip
+
+# kaldi
+depends/kaldi-5.*
+depends/kaldi
+
+# egs
 egs/gop-compute/steps
 egs/gop-compute/utils
 egs/gop-compute/data

README.md

@@ -1,17 +1,40 @@
-
 # kaldi-gop
-This project computes GOP (Goodness of Pronunciation) bases on Kaldi.
+This project computes GMM-based GOP (Goodness of Pronunciation) using Kaldi.
 
 ## How to build
-1. Download and complile [Kaldi](http://www.kaldi-asr.org). Note that you need to check out the branch 5.1 instead of master, and do not use the "--shared" option.
-1. Edit src/CMakeLists.txt to set the variable $KALDI_ROOT.
-1. Compile the binary:
 ```
-cd src/
-mkdir build && cd build
-cmake .. && make
+./build.sh
 ```
 ## Run the example
 ```
 cd egs/gop-compute
 ./run.sh
+```
+
+## Theory
+
+In the conventional GMM-HMM based system, GOP was first proposed in (Witt et al., 2000). It was defined as the duration normalised log of the posterior:
+
+$$
+GOP(p)=\frac{1}{t_e-t_s+1} \log p(p|\mathbf o)
+$$
+
+where $\mathbf o$ is the input observations, $p$ is the canonical phone, $t_s, t_e$ are the start and end frame indexes.
+
+Assuming $p(q_i)\approx p(q_j)$ for any $q_i, q_j$, we have:
+
+$$
+\log p(p|\mathbf o)=\frac{p(\mathbf o|p)p(p)}{\sum_{q\in Q} p(\mathbf o|q)p(q)}
+                   \approx\frac{p(\mathbf o|p)}{\sum_{q\in Q} p(\mathbf o|q)}
+$$
+
+where $Q$ is the whole phone set.
+
+The numerator of the equation is calculated from forced alignment result and the denominator is calculated from a Viterbi decoding with an unconstrained phone loop.
+
+## DNN-based implementation
+
+This implementation is GMM-based. For DNN-based implementation, please check Kaldi's official repository:
+> https://github.com/kaldi-asr/kaldi/tree/master/egs/gop
+
+The performance of GOP-DNN should be much better than GOP-GMM.

build.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+utils/check_dependencies.sh || exit
+cd src/
+[ -d build ] || mkdir build
+cd build
+cmake .. && make

depends/kaldi-makefile.patch

@@ -0,0 +1,16 @@
+--- "Makefile"	2018-02-15 02:58:49.000000000 +0800
++++ Makefile	2018-09-21 13:51:28.577621316 +0800
+@@ -6,10 +6,9 @@
+ 
+ 
+ SUBDIRS = base matrix util feat tree gmm transform \
+-          fstext hmm lm decoder lat kws cudamatrix nnet \
+-          bin fstbin gmmbin fgmmbin featbin \
+-          nnetbin latbin sgmm2 sgmm2bin nnet2 nnet3 rnnlm chain nnet3bin nnet2bin kwsbin \
+-          ivector ivectorbin online2 online2bin lmbin chainbin rnnlmbin
++          fstext hmm lm decoder lat cudamatrix \
++          nnet3 chain \
++          ivector online2
+ 
+ MEMTESTDIRS = base matrix util feat tree gmm transform \
+           fstext hmm lm decoder lat nnet kws chain \

egs/gop-compute/local/compute-gmm-gop.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Copyright 2016-2017  Author: Junbo Zhang
 

egs/gop-compute/path.sh

@@ -1,4 +1,4 @@
-export KALDI_ROOT=~/src/kaldi
+export KALDI_ROOT=../../depends/kaldi
 [ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh
 export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:../../src/build:$PWD:$PATH
 [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1

egs/gop-compute/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Copyright 2016-2017   Author: Junbo Zhang  <[email protected]>
 

src/CMakeLists.txt

@@ -1,25 +1,40 @@
 cmake_minimum_required (VERSION 2.6)
 project (kaldi-gop)
 
-# BLAS
-find_package(BLAS REQUIRED)
+######
+# Check BLAS
+find_package(BLAS)
 if(NOT BLAS_FOUND)
   message(FATAL_ERROR "Can't find BLAS. On Ubunbu, you may want to "
                 "install it by:\n  sudo apt-get install libatlas-dev")
 endif()
 
+######
 # Build Kaldi
-SET(KALDI_ROOT "${CMAKE_SOURCE_DIR}/kaldi-5.1")
-SET(BUILD_JOBS 8)
-if(NOT EXISTS ${KALDI_ROOT})
-  execute_process(COMMAND git clone -b 5.1 https://github.com/kaldi-asr/kaldi.git ${KALDI_ROOT})
-  execute_process(COMMAND extras/check_dependencies.sh WORKING_DIRECTORY ${KALDI_ROOT}/tools)
+set(KALDI_VERSION 5.3)
+set(BUILD_JOBS 8)
+set(DEPENDS_DIR "${CMAKE_SOURCE_DIR}/../depends")
+set(KALDI_ROOT "${DEPENDS_DIR}/kaldi-${KALDI_VERSION}")
+if(NOT EXISTS "${KALDI_ROOT}/tools/openfst/bin/fstcompile")
+  if (NOT EXISTS "${DEPENDS_DIR}/${KALDI_VERSION}.zip")
+    execute_process(COMMAND wget https://github.com/kaldi-asr/kaldi/archive/${KALDI_VERSION}.zip
+                    WORKING_DIRECTORY ${DEPENDS_DIR})
+  endif()
+  execute_process(COMMAND unzip -o ${KALDI_VERSION}.zip -d ${DEPENDS_DIR}
+                   WORKING_DIRECTORY ${DEPENDS_DIR})
+  execute_process(COMMAND ln -s kaldi-${KALDI_VERSION} kaldi
+                   WORKING_DIRECTORY ${DEPENDS_DIR})
   execute_process(COMMAND make -j ${BUILD_JOBS} WORKING_DIRECTORY ${KALDI_ROOT}/tools)
+  execute_process(COMMAND patch kaldi/src/Makefile kaldi-makefile.patch WORKING_DIRECTORY ${DEPENDS_DIR})
+endif()
+
+if(NOT EXISTS "${KALDI_ROOT}/src/decoder/kaldi-decoder.a")
   execute_process(COMMAND ./configure WORKING_DIRECTORY ${KALDI_ROOT}/src)
   execute_process(COMMAND make depend -j ${BUILD_JOBS} WORKING_DIRECTORY ${KALDI_ROOT}/src)
   execute_process(COMMAND make -j ${BUILD_JOBS} WORKING_DIRECTORY ${KALDI_ROOT}/src)
 endif()
 
+######
 # Settings
 add_compile_options(-std=c++11)
 add_definitions(-DHAVE_ATLAS)
@@ -28,7 +43,8 @@ include_directories("${KALDI_ROOT}/src")
 include_directories("${KALDI_ROOT}/tools/openfst/include")
 include_directories("${KALDI_ROOT}/tools/ATLAS/include")
 
-# gop
+######
+# Build libgop.a
 include_directories(".")
 file(GLOB GOP "gop/*.cc")
 add_library(gop ${GOP})
@@ -42,12 +58,13 @@ target_link_libraries(gop ${KALDI_ROOT}/src/matrix/kaldi-matrix.a)
 target_link_libraries(gop ${KALDI_ROOT}/src/base/kaldi-base.a)
 target_link_libraries(gop ${KALDI_ROOT}/tools/openfst/lib/libfst.a)
 target_link_libraries(gop dl.so)
-target_link_libraries(gop /usr/lib/libatlas.so.3)
-target_link_libraries(gop /usr/lib/libf77blas.so.3)
-target_link_libraries(gop /usr/lib/libcblas.so.3)
-target_link_libraries(gop /usr/lib/liblapack_atlas.so.3)
+find_library (LIBCBLAS libcblas.so.3)
+find_library (LIBLAPACK liblapack_atlas.so.3)
+target_link_libraries(gop ${LIBCBLAS})
+target_link_libraries(gop ${LIBLAPACK})
 
-# compute-gmm-gop
+######
+# Build compute-gmm-gop
 add_executable(compute-gmm-gop gopbin/compute-gmm-gop.cc)
 target_link_libraries(compute-gmm-gop gop)
 find_package(Threads)

src/gop/gmm-gop.cc

@@ -19,14 +19,14 @@
 #include <string>
 #include <vector>
 #include "base/kaldi-common.h"
-#include "util/common-utils.h"
-#include "gmm/am-diag-gmm.h"
-#include "hmm/transition-model.h"
-#include "fstext/fstext-utils.h"
 #include "decoder/decoder-wrappers.h"
+#include "fstext/fstext-utils.h"
+#include "gmm/am-diag-gmm.h"
 #include "gmm/decodable-am-diag-gmm.h"
-#include "lat/kaldi-lattice.h"
 #include "hmm/hmm-utils.h"
+#include "hmm/transition-model.h"
+#include "lat/kaldi-lattice.h"
+#include "util/common-utils.h"
 #include "gop/gmm-gop.h"
 
 namespace kaldi {
@@ -35,21 +35,23 @@ typedef typename fst::StdArc Arc;
 typedef typename Arc::StateId StateId;
 typedef typename Arc::Weight Weight;
 
-void GmmGop::Init(std::string &tree_in_filename,
-            std::string &model_in_filename,
-            std::string &lex_in_filename) {
+void GmmGop::Init(std::string &tree_in_filename, std::string &model_in_filename,
+                  std::string &lex_in_filename) {
   bool binary;
   Input ki(model_in_filename, &binary);
   tm_.Read(ki.Stream(), binary);
   am_.Read(ki.Stream(), binary);
   ReadKaldiObject(tree_in_filename, &ctx_dep_);
 
   fst::VectorFst<fst::StdArc> *lex_fst = fst::ReadFstKaldi(lex_in_filename);
-  std::vector<int32> disambig_syms;  
+  std::vector<int32> disambig_syms;
   TrainingGraphCompilerOptions gopts;
   gc_ = new TrainingGraphCompiler(tm_, ctx_dep_, lex_fst, disambig_syms, gopts);
 
   for (size_t i = 0; i < tm_.NumTransitionIds(); i++) {
+    // The transition-ids are only for building the denominator graph. Although
+    // one pdf-id may have multiple transition-ids, all those transitions-ids
+    //  share the same HMM state (of course).
     pdfid_to_tid[tm_.TransitionIdToPdf(i)] = i;
   }
 }
@@ -61,23 +63,22 @@ BaseFloat GmmGop::Decode(fst::VectorFst<fst::StdArc> &fst,
   decode_opts.beam = 500;
   FasterDecoder decoder(fst, decode_opts);
   decoder.Decode(&decodable);
-  if (! decoder.ReachedFinal()) {
+  if (!decoder.ReachedFinal()) {
     KALDI_WARN << "Did not successfully decode.";
   }
   fst::VectorFst<LatticeArc> decoded;
   decoder.GetBestPath(&decoded);
   std::vector<int32> osymbols;
   LatticeWeight weight;
   GetLinearSymbolSequence(decoded, align, &osymbols, &weight);
-  BaseFloat likelihood = -(weight.Value1()+weight.Value2());
+  BaseFloat likelihood = -(weight.Value1() + weight.Value2());
 
   return likelihood;
 }
 
 BaseFloat GmmGop::ComputeGopNumera(DecodableAmDiagGmmScaled &decodable,
                                    std::vector<int32> &align,
-                                   MatrixIndexT start_frame,
-                                   int32 size) {
+                                   MatrixIndexT start_frame, int32 size) {
   KALDI_ASSERT(start_frame + size <= align.size());
   BaseFloat likelihood = 0;
   for (MatrixIndexT frame = start_frame; frame < start_frame + size; frame++) {
@@ -88,7 +89,8 @@ BaseFloat GmmGop::ComputeGopNumera(DecodableAmDiagGmmScaled &decodable,
 }
 
 BaseFloat GmmGop::ComputeGopNumeraViterbi(DecodableAmDiagGmmScaled &decodable,
-                                          int32 phone_l, int32 phone, int32 phone_r) {
+                                          int32 phone_l, int32 phone,
+                                          int32 phone_r) {
   KALDI_ASSERT(ctx_dep_.ContextWidth() == 3);
   KALDI_ASSERT(ctx_dep_.CentralPosition() == 1);
   std::vector<int32> phoneseq(3);
@@ -101,7 +103,8 @@ BaseFloat GmmGop::ComputeGopNumeraViterbi(DecodableAmDiagGmmScaled &decodable,
   fst.SetStart(cur_state);
   for (size_t c = 0; c < tm_.GetTopo().NumPdfClasses(phone); c++) {
     int32 pdf_id;
-    KALDI_ASSERT(ctx_dep_.Compute(phoneseq, c, &pdf_id));
+    if (!ctx_dep_.Compute(phoneseq, c, &pdf_id))
+      KALDI_ERR << "Failed to obtain pdf_id.";
     int32 tid = pdfid_to_tid[pdf_id];
 
     StateId next_state = fst.AddState();
@@ -137,7 +140,8 @@ BaseFloat GmmGop::ComputeGopDenomin(DecodableAmDiagGmmScaled &decodable,
     StateId cur_state = start_state;
     for (size_t c = 0; c < pdfclass_num; c++) {
       int32 pdf_id;
-      KALDI_ASSERT(ctx_dep_.Compute(phoneseq, c, &pdf_id));
+      if (!ctx_dep_.Compute(phoneseq, c, &pdf_id))
+        KALDI_ERR << "Failed to obtain pdf_id.";
       int32 tid = pdfid_to_tid[pdf_id];
 
       StateId next_state = fst.AddState();
@@ -157,11 +161,14 @@ BaseFloat GmmGop::ComputeGopDenomin(DecodableAmDiagGmmScaled &decodable,
 }
 
 void GmmGop::GetContextFromSplit(std::vector<std::vector<int32> > split,
-                                 int32 index, int32 &phone_l, int32 &phone, int32 &phone_r) {
+                                 int32 index, int32 &phone_l, int32 &phone,
+                                 int32 &phone_r) {
   KALDI_ASSERT(index < split.size());
-  phone_l = (index > 0) ? tm_.TransitionIdToPhone(split[index-1][0]) : 1;
+  phone_l = (index > 0) ? tm_.TransitionIdToPhone(split[index - 1][0]) : 1;
   phone = tm_.TransitionIdToPhone(split[index][0]);
-  phone_r = (index < split.size() - 1) ? tm_.TransitionIdToPhone(split[index+1][0]): 1;
+  phone_r = (index < split.size() - 1)
+                ? tm_.TransitionIdToPhone(split[index + 1][0])
+                : 1;
 }
 
 void GmmGop::Compute(const Matrix<BaseFloat> &feats,
@@ -181,33 +188,31 @@ void GmmGop::Compute(const Matrix<BaseFloat> &feats,
   phones_.resize(split.size());
   int32 frame_start_idx = 0;
   for (MatrixIndexT i = 0; i < split.size(); i++) {
-    SubMatrix<BaseFloat> feats_in_phone = feats.Range(frame_start_idx, split[i].size(),
-                                                      0, feats.NumCols());
+    SubMatrix<BaseFloat> feats_in_phone =
+        feats.Range(frame_start_idx, split[i].size(), 0, feats.NumCols());
     const Matrix<BaseFloat> features(feats_in_phone);
     DecodableAmDiagGmmScaled split_decodable(am_, tm_, features, 1.0);
 
     int32 phone, phone_l, phone_r;
     GetContextFromSplit(split, i, phone_l, phone, phone_r);
 
     bool use_viterbi_numera = true;
-    BaseFloat gop_numerator = use_viterbi_numera ?
-                                ComputeGopNumeraViterbi(split_decodable, phone_l, phone, phone_r):
-                                ComputeGopNumera(ali_decodable, align,
-                                                 frame_start_idx, split[i].size());
-    BaseFloat gop_denominator = ComputeGopDenomin(split_decodable, phone_l, phone_r);
+    BaseFloat gop_numerator =
+        use_viterbi_numera
+            ? ComputeGopNumeraViterbi(split_decodable, phone_l, phone, phone_r)
+            : ComputeGopNumera(ali_decodable, align, frame_start_idx,
+                               split[i].size());
+    BaseFloat gop_denominator =
+        ComputeGopDenomin(split_decodable, phone_l, phone_r);
     gop_result_(i) = (gop_numerator - gop_denominator) / split[i].size();
     phones_[i] = phone;
 
     frame_start_idx += split[i].size();
   }
 }
 
-Vector<BaseFloat>& GmmGop::Result() {
-  return gop_result_;
-}
+Vector<BaseFloat> &GmmGop::Result() { return gop_result_; }
 
-std::vector<int32>& GmmGop::Phonemes() {
-  return phones_;
-}
+std::vector<int32> &GmmGop::Phonemes() { return phones_; }
 
 }  // End namespace kaldi