Merge pull request apache#6 from sergeykolychev/visualization

Support for Visualization

.gitignore

@@ -142,3 +142,6 @@ tools/pip_package/mxnet
 
 # temporary path for building dependencies when building wheel
 deps/
+
+*.jar
+target

.travis.yml

@@ -71,6 +71,7 @@ addons:
       - pdl
       - cpanminus
       - swig
+      - libgraphviz-perl
 
 before_install:
   - export NVCC_PREFIX=${HOME}

docs/how_to/perf.md

@@ -37,12 +37,12 @@ with MXNet commit `0a03417`
 
 | Batch | Alexnet | VGG | Inception-BN | Inception-v3 | Resnet 50 | Resnet 152 |
 | --- | --- | --- | --- | --- | --- | --- |
-|   1 |  98.44 | 24.66 |  27.43 |  11.91 |  38.74 | 14.95 |
-|   2 | 121.67 | 27.07 |  24.78 |  11.61 |  44.27 | 15.62 |
-|   4 | 254.39 | 38.51 |  28.76 |  12.86 |  40.88 | 19.01 |
-|   8 | 237.73 | 36.97 |  25.57 |  12.68 |  43.00 | 16.11 |
-|  16 | 280.82 | 40.00 |  20.85 |  11.77 |  55.00 | 16.93 |
-|  32 | 285.41 | 44.40 |  31.03 |  12.45 |  55.70 | 17.02 |
+|   1 |  122.21 | 34.23 |  99.24 |  52.16 |  46.03 | 20.11 |
+|   2 | 224.83 | 51.02 |  138.88 |  66.76 |  52.27 | 24.82 |
+|   4 | 295.87 | 65.88 |  185.46 |  76.70 |  67.45 | 28.16 |
+|   8 | 389.08 | 77.78 |  212.96 |  84.00 |  69.26 | 29.70 |
+|  16 | 519.87 | 85.08 |  222.81 |  85.10 |  68.94 | 29.11 |
+|  32 | 626.25 | 87.63 |  221.66 |  84.36 |  67.69 | 28.70 |
 
 ## Other CPU
 

example/speech-demo/README.md

@@ -145,3 +145,16 @@ Here are the results on TIMIT and AMI test set (using all default setup, 3 layer
 |AMI     | 51.7 (42.2) |
 
 Note that for AMI 42.2 was evaluated non-overlapped speech. Kaldi-HMM baseline was 67.2% and DNN was 57.5%.
+
+### update Feb 07
+
+We had updated this demo on Feb 07 (kaldi c747ed5, mxnet 912a7eb). We had also added timit demo script in this folder. 
+
+To run the timit demo:
+
+1. cd path/to/kaldi/egs/timit/s5/
+2. ./run.sh (setup the kaild timit demo and run it) 
+3. ln -s path/to/mxnet/example/speech-demo/* path/to/kaldi/egs/timit/s5/
+4. set **ali_src, graph_src** and so on in the run_timit.sh and default_timit.cfg to the generated folder in kaldi/egs/timit/s5/exp. In the demo script, we use tri3_ali as the alignment dir
+5. set ydim (in default_timit.cfg) to kaldi/egs/timit/s5/exp/tri3/graph/num_pdfs + 1
+6. ./run_timit.sh

example/speech-demo/decode_mxnet.sh

@@ -70,7 +70,7 @@ feats="scp:$sdata/JOB/feats.scp"
 inputfeats="$sdata/JOB/mxnetInput.scp"
 
 
-if [ ! -f $sdata/1/feats.scp ]; then
+if [ -f $sdata/1/feats.scp ]; then
     $cmd JOB=1:$nj $dir/log/make_input.JOB.log \
         echo NO_FEATURE_TRANSFORM scp:$sdata/JOB/feats.scp \> $inputfeats
 fi

example/speech-demo/default_timit.cfg

@@ -0,0 +1,52 @@
+[data]
+kaldi_root =
+train = /home/sooda/speech/kaldi/egs/timit/s5/data/train/train.feats
+dev = /home/sooda/speech/kaldi/egs/timit/s5/data/dev/dev.feats
+test =
+out_file = |
+format = kaldi
+xdim = 13
+ydim = 1939
+#ydim = 1909
+label_mean = label_mean.feats
+[arch]
+num_hidden = 1024
+# set it to zero if you want a regular LSTM
+num_hidden_proj = 512
+num_lstm_layer = 3
+
+[train]
+batch_size = 40
+buckets = 100, 200, 300, 400, 500, 600, 700, 800
+num_epoch = 12
+
+# used only if method is truncated-bptt
+truncate_len = 20
+
+# gpu0, gpu1
+context = gpu0
+
+# bucketing, truncated-bptt
+method = truncated-bptt
+#method = bucketing
+
+# checkpoint prefix
+prefix = timit
+
+learning_rate = 1
+decay_factor = 2
+decay_lower_bound = 1e-6
+
+optimizer = speechSGD
+momentum = 0.9
+
+# set to 0 to disable gradient clipping
+clip_gradient = 0
+
+# uniform, normal, xavier
+initializer = Uniform
+init_scale = 0.05
+weight_decay = 0.008
+
+# show progress every how many batches
+show_every = 1000

example/speech-demo/io_util.py

@@ -300,7 +300,6 @@ def __init__(self, train_sets, batch_size, init_states, truncate_len=20, delay=5
         self.pad_zeros = pad_zeros
 
         self.time_major = time_major
-        self.layout_mapper = mx.io.DefaultLayoutMapper(1 if time_major else 0)
 
         self.label = None
         if self.time_major:

example/speech-demo/make_stats.py

@@ -8,7 +8,7 @@
 import mxnet as mx
 import numpy as np
 
-from lstm import lstm_unroll
+from lstm_proj import lstm_unroll
 from io_util import BucketSentenceIter, TruncatedSentenceIter, SimpleIter, DataReadStream
 from config_util import parse_args, get_checkpoint_path, parse_contexts
 

example/speech-demo/python_wrap/ctypes.cc

@@ -203,10 +203,10 @@ extern "C" {
     }
 
     if (dropout_retention > 0.0) {
-      nnet->nnet_transf.SetDropoutRetention(dropout_retention);
+      nnet->nnet_transf.SetDropoutRate(dropout_retention);
     }
     if (crossvalidate) {
-      nnet->nnet_transf.SetDropoutRetention(1.0);
+      nnet->nnet_transf.SetDropoutRate(1.0);
     }
 
     return nnet;

example/speech-demo/run_ami.sh

@@ -81,7 +81,7 @@ if [ $stage -le 0 ] ; then
             ark:- \| ali-to-post ark:- ark,scp:$dir/rawpost/post.JOB.ark,$dir/rawpost/post.JOB.scp || exit 1;
 
 
-    for n in $(seq $njdec); do
+    for n in $(seq $num); do
         cat $dir/rawpost/post.${n}.scp || exit 1;
     done > $dir/post.scp
 fi

example/speech-demo/run_timit.sh

@@ -0,0 +1,123 @@
+#!/bin/bash
+
+# This script trains and evaluate LSTM models. There is no
+# discriminative training yet.
+# In this recipe, MXNet directly read Kaldi features and labels,
+# which makes the whole pipline much simpler.
+
+set -e           #Exit on non-zero return code from any command
+set -o pipefail  #Exit if any of the commands in the pipeline will
+                 #return non-zero return code
+set -u           #Fail on an undefined variable
+
+. ./cmd.sh
+. ./path.sh
+
+cmd=run.pl
+# root folder,
+expdir=exp_timit
+
+##################################################
+# Kaldi generated folder
+##################################################
+
+# alignment folder
+ali_src=/home/sooda/speech/kaldi/egs/timit/s5/exp/tri3_ali
+
+# decoding graph
+graph_src=/home/sooda/speech/kaldi/egs/timit/s5/exp/tri3/graph
+
+# features
+train_src=/home/sooda/speech/kaldi/egs/timit/s5/data/train
+dev_src=/home/sooda/speech/kaldi/egs/timit/s5/data/dev
+
+# config file
+config=default_timit.cfg
+# optional settings,
+njdec=8
+scoring="--min-lmwt 5 --max-lmwt 19"
+
+# The device number to run the training
+# change to AUTO to select the card automatically
+deviceNumber=gpu0
+
+# decoding method
+method=simple
+modelName=
+# model
+prefix=timit
+num_epoch=12
+acwt=0.1
+#smbr training variables
+num_utts_per_iter=40
+smooth_factor=0.1
+use_one_sil=true
+
+stage=4
+. utils/parse_options.sh || exit 1;
+
+
+###############################################
+# Training
+###############################################
+
+mkdir -p $expdir
+dir=$expdir/data-for-mxnet
+
+# prepare listing data
+if [ $stage -le 0 ] ; then
+    mkdir -p $dir
+    mkdir -p $dir/log
+    mkdir -p $dir/rawpost
+
+    # for compressed ali
+    num=`cat $ali_src/num_jobs`
+    $cmd JOB=1:$num $dir/log/gen_post.JOB.log \
+        ali-to-pdf $ali_src/final.mdl "ark:gunzip -c $ali_src/ali.JOB.gz |" \
+            ark:- \| ali-to-post ark:- ark,scp:$dir/rawpost/post.JOB.ark,$dir/rawpost/post.JOB.scp || exit 1;
+    #num=`cat $ali_src/num_jobs`
+    #$cmd JOB=1:$num $dir/log/gen_post.JOB.log \
+    #    ali-to-pdf $ali_src/final.mdl ark:$ali_src/ali.JOB.ark \
+    #        ark:- \| ali-to-post ark:- ark,scp:$dir/rawpost/post.JOB.ark,$dir/rawpost/post.JOB.scp || exit 1;
+
+
+    for n in $(seq $num); do
+        cat $dir/rawpost/post.${n}.scp || exit 1;
+    done > $dir/post.scp
+fi
+
+if [ $stage -le 1 ] ; then
+    # split the data : 90% train and 10% held-out
+    [ ! -e ${train_src}_tr90 ] && utils/subset_data_dir_tr_cv.sh $train_src ${train_src}_tr90 ${train_src}_cv10
+
+    # generate dataset list
+    echo NO_FEATURE_TRANSFORM scp:${train_src}_tr90/feats.scp > $dir/train.feats
+    echo scp:$dir/post.scp >> $dir/train.feats
+
+    echo NO_FEATURE_TRANSFORM scp:${train_src}_cv10/feats.scp > $dir/dev.feats
+    echo scp:$dir/post.scp >> $dir/dev.feats
+
+    echo NO_FEATURE_TRANSFORM scp:${dev_src}/feats.scp > $dir/test.feats
+fi
+
+# generate label counts
+if [ $stage -le 2 ] ; then
+    $cmd JOB=1:1 $dir/log/gen_label_mean.JOB.log \
+        python make_stats.py --configfile $config --data_train $dir/train.feats \| copy-feats ark:- ark:$dir/label_mean.ark
+    echo NO_FEATURE_TRANSFORM ark:$dir/label_mean.ark > $dir/label_mean.feats
+fi
+
+
+# training, note that weight decay is for the whole batch (0.00001 * 20 (minibatch) * 40 (batch_size))
+if [ $stage -le 3 ] ; then
+    python train_lstm_proj.py --configfile $config --data_train $dir/train.feats --data_dev $dir/dev.feats --train_prefix $PWD/$expdir/$prefix --train_optimizer speechSGD --train_learning_rate 1 --train_context $deviceNumber --train_weight_decay 0.008 --train_show_every 1000
+fi
+
+# decoding
+if [ $stage -le 4 ] ; then
+  cp $ali_src/final.mdl $expdir
+  mxnet_string="OMP_NUM_THREADS=1 python decode_mxnet.py --config $config --data_test $dir/test.feats --data_label_mean $dir/label_mean.feats --train_method $method --train_prefix $PWD/$expdir/$prefix --train_num_epoch $num_epoch --train_context cpu0 --train_batch_size 1"
+  ./decode_mxnet.sh --nj $njdec --cmd $cmd --acwt $acwt --scoring-opts "$scoring" \
+    $graph_src $dev_src $expdir/decode_${prefix}_$(basename $dev_src) "$mxnet_string" || exit 1;
+
+fi

include/mxnet/tensor_blob.h

@@ -136,6 +136,17 @@ class TBlob {
   inline bool CheckContiguous(void) const {
     return shape_[shape_.ndim() - 1] == stride_;
   }
+  /*!
+   * \brief reshape to shape
+   * \param shape desired shape
+   * \return reshaped blob
+   */
+  inline TBlob reshape(const TShape& shape) const {
+    CHECK_EQ(this->shape_.Size(), shape.Size()) << "Shape size mismatch "
+    << this->shape_.Size() << " v.s. "  << shape.Size();
+    TBlob ret(this->dptr_, shape, this->dev_mask_, this->type_flag_);
+    return ret;
+  }
   /*!
    * \brief flatten the tensor to 2 dimension, collapse the higher dimensions together
    * \param stream the possible stream target tensor should reside on

perl-package/AI-MXNet/MANIFEST

@@ -1,5 +1,4 @@
 MANIFEST
-META.yml
 Makefile.PL
 README
 Changes
@@ -36,9 +35,11 @@ lib/AI/MXNet/Symbol/NameManager.pm
 lib/AI/MXNet/TestUtils.pm
 lib/AI/MXNet/Types.pm
 lib/AI/MXNet/Util/Printable.pm
+lib/AI/MXNet/Visualization.pm
 t/AI-MXNet.t
 t/initializers.t
 t/io.t
 t/models.t
 t/optimizers.t
 t/test_conv.t
+t/test_viz.t

perl-package/AI-MXNet/Makefile.PL

@@ -16,14 +16,15 @@ my %WriteMakefileArgs = (
   },
   "DISTNAME" => "AI-MXNet",
   "EXE_FILES" => [],
-  "LICENSE" => "perl",
+  "LICENSE" => "apache_2_0",
   "NAME" => "AI::MXNet",
   "PREREQ_PM" => {
     "AI::MXNetCAPI" => "0.02",
     "AI::NNVMCAPI" => "0.02",
     "Function::Parameters" => "1.0705",
-    "Mouse" => "v2.1.0",
-    "PDL" => "2.007"
+    "Mouse" => "2.1.0",
+    "PDL" => "2.007",
+    "GraphViz" => "2.14"
   },
   "TEST_REQUIRES" => {},
   "VERSION" => "0.02",
@@ -37,8 +38,9 @@ my %FallbackPrereqs = (
   "AI::MXNetCAPI" => "0.02",
   "AI::NNVMCAPI" => "0.02",
   "Function::Parameters" => "1.0705",
-  "Mouse" => "v2.1.0",
-  "PDL" => "2.007"
+  "Mouse" => "2.1.0",
+  "PDL" => "2.007",
+  "GraphViz" => "2.14"
 );
 
 

perl-package/AI-MXNet/lib/AI/MXNet.pm

@@ -21,6 +21,7 @@ use AI::MXNet::Monitor;
 use AI::MXNet::Profiler;
 use AI::MXNet::Module::Base;
 use AI::MXNet::Module;
+use AI::MXNet::Visualization;
 our $VERSION = '0.02';
 
 sub import
@@ -48,6 +49,7 @@ sub import
             sub io { 'AI::MXNet::IO' }
             sub metric { 'AI::MXNet::Metric' }
             sub mod { 'AI::MXNet::Module' }
+            sub viz { 'AI::MXNet::Visualization' }
             1;
 EOP
             eval $short_name_package;

perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm

@@ -78,7 +78,7 @@ sub _compose
 func _make_atomic_symbol_function($handle, $name)
 {
     my ($real_name, $desc, $arg_names, 
-        $arg_types, $arg_descs, $key_var_num_args, 
+        $arg_types, $arg_descs, $key_var_num_args,
         $ret_type) = @{ check_call(AI::MXNetCAPI::SymbolGetAtomicSymbolInfo($handle)) };
     $ret_type //= '';
     my $func_name = $name;
@@ -105,16 +105,26 @@ func _make_atomic_symbol_function($handle, $name)
         symbol: Symbol
             the resulting symbol
 =cut
-
     my $creator = sub {
-
         my $class = shift;
         my (@args, %kwargs);
         if(@_ and ref $_[-1] eq 'HASH')
         {
             %kwargs = %{ pop(@_) };
+            @args = @_;
+        }
+        elsif(blessed $_[0] and $_[0]->isa(__PACKAGE__))
+        {
+            while(blessed $_[0] and $_[0]->isa(__PACKAGE__))
+            {
+                push @args, shift(@_);
+            }
+            %kwargs = @_;
+        }
+        else
+        {
+            %kwargs = @_;
         }
-        @args = @_;
         my $params = {};
         my $symbol_kwargs = {};
         my $attr = delete $kwargs{ 'attr' };