Update lip reading example

example/gluon/lipnet/.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+utils/*.dat
+

example/gluon/lipnet/BeamSearch.py

@@ -0,0 +1,168 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Module : this module to decode using beam search
+https://github.com/ThomasDelteil/HandwrittenTextRecognition_MXNet/blob/master/utils/CTCDecoder/BeamSearch.py 
+"""
+
+from __future__ import division
+from __future__ import print_function
+import numpy as np
+
+class BeamEntry:
+    """
+    information about one single beam at specific time-step
+    """
+    def __init__(self):
+        self.prTotal = 0 # blank and non-blank
+        self.prNonBlank = 0 # non-blank
+        self.prBlank = 0 # blank
+        self.prText = 1 # LM score
+        self.lmApplied = False # flag if LM was already applied to this beam
+        self.labeling = () # beam-labeling
+
+class BeamState:
+    """
+    information about the beams at specific time-step
+    """
+    def __init__(self):
+        self.entries = {}
+
+    def norm(self):
+        """
+        length-normalise LM score
+        """
+        for (k, _) in self.entries.items():
+            labelingLen = len(self.entries[k].labeling)
+            self.entries[k].prText = self.entries[k].prText ** (1.0 / (labelingLen if labelingLen else 1.0))
+
+    def sort(self):
+        """
+        return beam-labelings, sorted by probability
+        """
+        beams = [v for (_, v) in self.entries.items()]
+        sortedBeams = sorted(beams, reverse=True, key=lambda x: x.prTotal*x.prText)
+        return [x.labeling for x in sortedBeams]
+
+def applyLM(parentBeam, childBeam, classes, lm):
+    """
+    calculate LM score of child beam by taking score from parent beam and bigram probability of last two chars
+    """
+    if lm and not childBeam.lmApplied:
+        c1 = classes[parentBeam.labeling[-1] if parentBeam.labeling else classes.index(' ')] # first char
+        c2 = classes[childBeam.labeling[-1]] # second char
+        lmFactor = 0.01 # influence of language model
+        bigramProb = lm.getCharBigram(c1, c2) ** lmFactor # probability of seeing first and second char next to each other
+        childBeam.prText = parentBeam.prText * bigramProb # probability of char sequence
+        childBeam.lmApplied = True # only apply LM once per beam entry
+
+def addBeam(beamState, labeling):
+    """
+    add beam if it does not yet exist
+    """
+    if labeling not in beamState.entries:
+        beamState.entries[labeling] = BeamEntry()
+
+def ctcBeamSearch(mat, classes, lm, k, beamWidth):
+    """
+    beam search as described by the paper of Hwang et al. and the paper of Graves et al.
+    """
+
+    blankIdx = len(classes)
+    maxT, maxC = mat.shape
+
+    # initialise beam state
+    last = BeamState()
+    labeling = ()
+    last.entries[labeling] = BeamEntry()
+    last.entries[labeling].prBlank = 1
+    last.entries[labeling].prTotal = 1
+
+    # go over all time-steps
+    for t in range(maxT):
+        curr = BeamState()
+
+        # get beam-labelings of best beams
+        bestLabelings = last.sort()[0:beamWidth]
+
+	# go over best beams
+        for labeling in bestLabelings:
+
+	    # probability of paths ending with a non-blank
+            prNonBlank = 0
+	    # in case of non-empty beam
+            if labeling:
+		# probability of paths with repeated last char at the end
+                try: 
+                    prNonBlank = last.entries[labeling].prNonBlank * mat[t, labeling[-1]]
+                except FloatingPointError:
+                    prNonBlank = 0
+
+	    # probability of paths ending with a blank
+            prBlank = (last.entries[labeling].prTotal) * mat[t, blankIdx]
+
+	    # add beam at current time-step if needed
+            addBeam(curr, labeling)
+
+            # fill in data
+            curr.entries[labeling].labeling = labeling
+            curr.entries[labeling].prNonBlank += prNonBlank
+            curr.entries[labeling].prBlank += prBlank
+            curr.entries[labeling].prTotal += prBlank + prNonBlank
+            curr.entries[labeling].prText = last.entries[labeling].prText # beam-labeling not changed, therefore also LM score unchanged from
+            curr.entries[labeling].lmApplied = True # LM already applied at previous time-step for this beam-labeling
+
+            # extend current beam-labeling
+            for c in range(maxC - 1):
+                # add new char to current beam-labeling
+                newLabeling = labeling + (c,)
+
+                # if new labeling contains duplicate char at the end, only consider paths ending with a blank
+                if labeling and labeling[-1] == c:
+                    prNonBlank = mat[t, c] * last.entries[labeling].prBlank
+                else:
+                    prNonBlank = mat[t, c] * last.entries[labeling].prTotal
+
+		# add beam at current time-step if needed
+                addBeam(curr, newLabeling)
+
+		# fill in data
+                curr.entries[newLabeling].labeling = newLabeling
+                curr.entries[newLabeling].prNonBlank += prNonBlank
+                curr.entries[newLabeling].prTotal += prNonBlank
+
+		# apply LM
+                applyLM(curr.entries[labeling], curr.entries[newLabeling], classes, lm)
+
+        # set new beam state
+        last = curr
+
+    # normalise LM scores according to beam-labeling-length
+    last.norm()
+
+    # sort by probability
+    bestLabelings = last.sort()[:k] # get most probable labeling
+
+    output = []
+    for bestLabeling in bestLabelings:
+        # map labels to chars
+        res = ''
+        for l in bestLabeling:
+            res += classes[l]
+        output.append(res)
+    return output

example/gluon/lipnet/README.md

@@ -0,0 +1,127 @@
+# LipNet: End-to-End Sentence-level Lipreading
+
+---
+
+Gluon implementation of [LipNet: End-to-End Sentence-level Lipreading](https://arxiv.org/abs/1611.01599)
+
+![net_structure](asset/network_structure.png)
+
+## Requirements
+- Python 3.6.4
+- MXnet 1.3.0
+- The Required Disk Space: 35Gb
+```
+pip install -r requirements.txt
+```
+
+## Test Environment
+- 4 CPU cores
+- 1 GPU (Tesla K80 12GB)
+
+
+## The Data
+- The GRID audiovisual sentence corpus (http://spandh.dcs.shef.ac.uk/gridcorpus/)
+  - GRID is a large multitalker audiovisual sentence corpus to support joint computational-behavioral studies in speech perception. In brief, the corpus consists of high-quality audio and video (facial) recordings of 1000 sentences spoken by each of 34 talkers (18 male, 16 female). Sentences are of the form "put red at G9 now". The corpus, together with transcriptions, is freely available for research use.
+- Video: (normal)(480 M each)
+  - Each movie has one sentence consist of 6 words.
+- Align: word alignments(190 K each) 
+  - One align has 6 words. Each word has start time and end time. But this tutorial needs just sentence because of using ctc-loss.
+
+## Prepare the Data
+### Download the data
+- Outputs
+  - The Total Moives(mp4): 16GB
+  - The Total Aligns(text): 134MB
+- Arguments
+  - src_path : Path for videos (default='./data/mp4s/')
+  - align_path : Path for aligns (default='./data/')
+  - n_process : num of process (default=1)
+
+```
+cd ./utils && python download_data.py --n_process $(nproc)
+```
+
+### Preprocess the Data: Extracting the mouth images from a video and save it.
+- Outputs
+  - The Total Images(png): 19GB
+- Elapsed time
+  - About 54 Hours using 1 process
+  - If you use the multi-processes, you can finish the number of processes faster.
+    - e.g) 9 hours using 6 processes
+- Arguments
+  - src_path : Path for videos (default='./data/mp4s/')
+  - tgt_path : Path for preprocessed images (default='./data/datasets/')
+  - n_process : num of process (default=1)
+
+You can run the preprocessing with just one processor, but this will take a long time (>48 hours). To use all of the available processors, use the following command: 
+
+```
+cd ./utils && python preprocess_data.py --n_process $(nproc)
+```
+
+## Data Structure
+
+```
+The training data folder should look like : 
+<train_data_root>
+                |--datasets
+                        |--s1
+                           |--bbir7s
+                               |--mouth_000.png
+                               |--mouth_001.png
+                                   ...
+                           |--bgaa8p
+                               |--mouth_000.png
+                               |--mouth_001.png
+                                  ...
+                        |--s2
+                            ...
+                 |--align
+                         |--bw1d8a.align
+                         |--bggzzs.align
+                             ...
+
+```
+
+
+## Training
+
+- arguments
+  - batch_size : Define batch size (default=64)
+  - epochs : Define total epochs (default=100)
+  - image_path : Path for lip image files (default='./data/datasets/')
+  - align_path : Path for align files (default='./data/align/')
+  - dr_rate : Dropout rate(default=0.5)
+  - num_gpus : Num of gpus (if num_gpus is 0, then use cpu) (default=1)
+  - num_workers : Num of workers when generating data (default=0)
+
+```
+python main.py
+```
+
+## Results
+```
+[Target]
+['lay green with a zero again',
+ 'bin blue with r nine please',
+ 'set blue with e five again',
+ 'bin green by t seven soon',
+ 'lay red at d five now',
+ 'bin green in x eight now',
+ 'bin blue with e one now',
+ 'lay red at j nine now']
+ ```
+
+ ```
+[Pred]
+['lay green with s zero again',
+ 'bin blue with r nine please',
+ 'set blue with e five again',
+ 'bin green by t seven soon',
+ 'lay red at c five now',
+ 'bin green in x eight now',
+ 'bin blue with m one now',
+ 'lay red at j nine now']
+ ```
+
+

example/gluon/lipnet/checkpoint/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

example/gluon/lipnet/data_loader.py

@@ -0,0 +1,73 @@
+"""
+Description : Set DataSet module for lip images
+"""
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import glob
+from mxnet import nd
+import mxnet.gluon.data.dataset as dataset
+from mxnet.gluon.data.vision.datasets import image
+from utils.align import Align
+
+class LipsDataset(dataset.Dataset):
+    """
+    Description : DataSet class for lip images
+    """
+    def __init__(self, root, align_root, flag=1, transform=None):
+        self._root = os.path.expanduser(root)
+        self._align_root = align_root
+        self._flag = flag
+        self._transform = transform
+        self._exts = ['.jpg', '.jpeg', '.png']
+        self._list_images(self._root)
+
+    def _list_images(self, root):
+        """
+        Description : generate list for lip images
+        """
+        self.labels = []
+        self.items = []
+        folder_path = glob.glob(os.path.join(root, "*", "*"))
+        for folder in folder_path:
+            filename = glob.glob(os.path.join(folder, "*"))
+            if len(filename) != 75:
+                continue
+            filename.sort()
+            label = os.path.split(folder)[-1]
+            self.items.append((filename, label))
+    def align_generation(self, file_nm, padding=75):
+        """
+        Description : Align to lip position
+        """
+        align = Align(self._align_root + '/' + file_nm + '.align')
+        return nd.array(align.sentence(padding))
+    def __getitem__(self, idx):
+        img = list()
+        for image_name in self.items[idx][0]:
+            tmp_img = image.imread(image_name, self._flag)
+            if self._transform is not None:
+                tmp_img = self._transform(tmp_img)
+            img.append(tmp_img)
+        img = nd.stack(*img)
+        #print(self.items[idx][0][0])
+        label = self.align_generation(self.items[idx][1])
+        return img, label
+
+    def __len__(self):
+        return len(self.items)