Skip to content

xconfig : Added delay option for FixedAffineLayer. This will be used … #1540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danpovey merged 2 commits into from
Apr 12, 2017
Merged

xconfig : Added delay option for FixedAffineLayer. This will be used … #1540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
78aaa79
xconfig : Added delay option for FixedAffineLayer. This will be used …
vijayaditya Apr 11, 2017
64c9826
minor changes
vijayaditya Apr 12, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
304 changes: 304 additions & 0 deletions egs/swbd/s5c/local/chain/tuning/run_lstm_6k.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,304 @@
#!/bin/bash

# Copyright 2015 Johns Hopkins University (Author: Daniel Povey).
# 2015 Vijayaditya Peddinti
# 2015 Xingyu Na
# 2015 Pegah Ghahrmani
# 2017 Google Inc. (vpeddinti@google.com)
# Apache 2.0.



# run_lstm_6k.sh is like run_lstm_6j.sh but making
# various kaldi-5.1-related upgrades to the script.
# For the list of changes compare tuning/run_tdnn_lstm_1{c,d}.sh

set -e

# configs for 'chain'
stage=12
train_stage=-10
get_egs_stage=-10
speed_perturb=true
dir=exp/chain/lstm_6k # Note: _sp will get added to this if $speed_perturb == true.
decode_iter=
decode_nj=50

# training options
xent_regularize=0.01
self_repair_scale=0.00001
label_delay=5

chunk_left_context=40
chunk_right_context=0
# we'll put chunk-left-context-initial=0 and chunk-right-context-final=0
# directly without variables.
frames_per_chunk=140,100,160

# (non-looped) decoding options
frames_per_chunk_primary=$(echo $frames_per_chunk | cut -d, -f1)
extra_left_context=50
extra_right_context=0
# we'll put extra-left-context-initial=0 and extra-right-context-final=0
# directly without variables.


remove_egs=false
common_egs_dir=

test_online_decoding=false # if true, it will run the last decoding stage.

# End configuration section.
echo "$0 $@" # Print the command line for logging

. ./cmd.sh
. ./path.sh
. ./utils/parse_options.sh

if ! cuda-compiled; then
cat <<EOF && exit 1
This script is intended to be used with GPUs but you have not compiled Kaldi with CUDA
If you want to use GPUs (and have them), go to src/, and configure and make on a machine
where "nvcc" is installed.
EOF
fi

# The iVector-extraction and feature-dumping parts are the same as the standard
# nnet3 setup, and you can skip them by setting "--stage 8" if you have already
# run those things.

suffix=
if [ "$speed_perturb" == "true" ]; then
suffix=_sp
fi

dir=${dir}$suffix
train_set=train_nodup$suffix
ali_dir=exp/tri4_ali_nodup$suffix
treedir=exp/chain/tri5_7d_tree$suffix
lang=data/lang_chain_2y


# if we are using the speed-perturbed data we need to generate
# alignments for it.
local/nnet3/run_ivector_common.sh --stage $stage \
--speed-perturb $speed_perturb \
--generate-alignments $speed_perturb || exit 1;


if [ $stage -le 9 ]; then
# Get the alignments as lattices (gives the CTC training more freedom).
# use the same num-jobs as the alignments
nj=$(cat exp/tri4_ali_nodup$suffix/num_jobs) || exit 1;
steps/align_fmllr_lats.sh --nj $nj --cmd "$train_cmd" data/$train_set \
data/lang exp/tri4 exp/tri4_lats_nodup$suffix
rm exp/tri4_lats_nodup$suffix/fsts.*.gz # save space
fi


if [ $stage -le 10 ]; then
# Create a version of the lang/ directory that has one state per phone in the
# topo file. [note, it really has two states.. the first one is only repeated
# once, the second one has zero or more repeats.]
rm -rf $lang
cp -r data/lang $lang
silphonelist=$(cat $lang/phones/silence.csl) || exit 1;
nonsilphonelist=$(cat $lang/phones/nonsilence.csl) || exit 1;
# Use our special topology... note that later on may have to tune this
# topology.
steps/nnet3/chain/gen_topo.py $nonsilphonelist $silphonelist >$lang/topo
fi

if [ $stage -le 11 ]; then
# Build a tree using our new topology.
steps/nnet3/chain/build_tree.sh --frame-subsampling-factor 3 \
--context-opts "--context-width=2 --central-position=1" \
--cmd "$train_cmd" 7000 data/$train_set $lang $ali_dir $treedir
fi

if [ $stage -le 12 ]; then
echo "$0: creating neural net configs using the xconfig parser";

num_targets=$(tree-info $treedir/tree |grep num-pdfs|awk '{print $2}')
[ -z $num_targets ] && { echo "$0: error getting num-targets"; exit 1; }
learning_rate_factor=$(echo "print 0.5/$xent_regularize" | python)

lstm_opts="decay-time=20"

mkdir -p $dir/configs
cat <<EOF > $dir/configs/network.xconfig
input dim=100 name=ivector
input dim=40 name=input

# please note that it is important to have input layer with the name=input
# as the layer immediately preceding the fixed-affine-layer to enable
# the use of short notation for the descriptor
# Note : The delay variable will be used just in the init.config.
fixed-affine-layer name=lda input=Append(-2,-1,0,1,2,ReplaceIndex(ivector, t, 0)) affine-transform-file=$dir/configs/lda.mat delay=$label_delay

# check steps/libs/nnet3/xconfig/lstm.py for the other options and defaults
fast-lstmp-layer name=fastlstm1 cell-dim=1024 recurrent-projection-dim=256 non-recurrent-projection-dim=256 delay=-3 $lstm_opts
fast-lstmp-layer name=fastlstm2 cell-dim=1024 recurrent-projection-dim=256 non-recurrent-projection-dim=256 delay=-3 $lstm_opts
fast-lstmp-layer name=fastlstm3 cell-dim=1024 recurrent-projection-dim=256 non-recurrent-projection-dim=256 delay=-3 $lstm_opts

## adding the layers for chain branch
output-layer name=output input=fastlstm3 output-delay=$label_delay include-log-softmax=false dim=$num_targets max-change=1.5

# adding the layers for xent branch
# This block prints the configs for a separate output that will be
# trained with a cross-entropy objective in the 'chain' models... this
# has the effect of regularizing the hidden parts of the model. we use
# 0.5 / args.xent_regularize as the learning rate factor- the factor of
# 0.5 / args.xent_regularize is suitable as it means the xent
# final-layer learns at a rate independent of the regularization
# constant; and the 0.5 was tuned so as to make the relative progress
# similar in the xent and regular final layers.
output-layer name=output-xent input=fastlstm3 output-delay=$label_delay dim=$num_targets learning-rate-factor=$learning_rate_factor max-change=1.5

EOF
steps/nnet3/xconfig_to_configs.py --xconfig-file $dir/configs/network.xconfig --config-dir $dir/configs/
fi

if [ $stage -le 13 ]; then
if [[ $(hostname -f) == *.clsp.jhu.edu ]] && [ ! -d $dir/egs/storage ]; then
utils/create_split_dir.pl \
/export/b0{5,6,7,8}/$USER/kaldi-data/egs/swbd-$(date +'%m_%d_%H_%M')/s5c/$dir/egs/storage $dir/egs/storage
fi

steps/nnet3/chain/train.py --stage $train_stage \
--cmd "$decode_cmd" \
--feat.online-ivector-dir exp/nnet3/ivectors_${train_set} \
--feat.cmvn-opts "--norm-means=false --norm-vars=false" \
--chain.xent-regularize $xent_regularize \
--chain.leaky-hmm-coefficient 0.1 \
--chain.l2-regularize 0.00005 \
--chain.apply-deriv-weights false \
--chain.lm-opts="--num-extra-lm-states=2000" \
--trainer.num-chunk-per-minibatch 64,32 \
--trainer.frames-per-iter 1500000 \
--trainer.max-param-change 2.0 \
--trainer.num-epochs 4 \
--trainer.optimization.shrink-value 0.99 \
--trainer.optimization.num-jobs-initial 3 \
--trainer.optimization.num-jobs-final 16 \
--trainer.optimization.initial-effective-lrate 0.001 \
--trainer.optimization.final-effective-lrate 0.0001 \
--trainer.optimization.momentum 0.0 \
--trainer.deriv-truncate-margin 8 \
--egs.stage $get_egs_stage \
--egs.opts "--frames-overlap-per-eg 0" \
--egs.chunk-width $frames_per_chunk \
--egs.chunk-left-context $chunk_left_context \
--egs.chunk-right-context $chunk_right_context \
--egs.chunk-left-context-initial 0 \
--egs.chunk-right-context-final 0 \
--egs.dir "$common_egs_dir" \
--cleanup.remove-egs $remove_egs \
--feat-dir data/${train_set}_hires \
--tree-dir $treedir \
--lat-dir exp/tri4_lats_nodup$suffix \
--dir $dir || exit 1;
fi

if [ $stage -le 14 ]; then
# Note: it might appear that this $lang directory is mismatched, and it is as
# far as the 'topo' is concerned, but this script doesn't read the 'topo' from
# the lang directory.
utils/mkgraph.sh --self-loop-scale 1.0 data/lang_sw1_tg $dir $dir/graph_sw1_tg
fi


graph_dir=$dir/graph_sw1_tg
iter_opts=
if [ ! -z $decode_iter ]; then
iter_opts=" --iter $decode_iter "
fi

if [ $stage -le 15 ]; then
rm $dir/.error 2>/dev/null || true
for decode_set in train_dev eval2000; do
(
steps/nnet3/decode.sh --num-threads 4 \
--acwt 1.0 --post-decode-acwt 10.0 \
--nj 25 --cmd "$decode_cmd" $iter_opts \
--extra-left-context $extra_left_context \
--extra-right-context $extra_right_context \
--extra-left-context-initial 0 \
--extra-right-context-final 0 \
--frames-per-chunk "$frames_per_chunk_primary" \
--online-ivector-dir exp/nnet3/ivectors_${decode_set} \
$graph_dir data/${decode_set}_hires \
$dir/decode_${decode_set}${decode_iter:+_$decode_iter}_sw1_tg || exit 1;
if $has_fisher; then
steps/lmrescore_const_arpa.sh --cmd "$decode_cmd" \
data/lang_sw1_{tg,fsh_fg} data/${decode_set}_hires \
$dir/decode_${decode_set}${decode_iter:+_$decode_iter}_sw1_{tg,fsh_fg} || exit 1;
fi
) &
done
wait
if [ -f $dir/.error ]; then
echo "$0: something went wrong in decoding"
exit 1
fi
fi

if [ $stage -le 16 ]; then
# looped decoding. Note: this does not make sense for BLSTMs or other
# backward-recurrent setups, and for TDNNs and other non-recurrent there is no
# point doing it because it would give identical results to regular decoding.
rm $dir/.error 2>/dev/null || true
for decode_set in train_dev eval2000; do
(
steps/nnet3/decode_looped.sh \
--acwt 1.0 --post-decode-acwt 10.0 \
--nj $decode_nj --cmd "$decode_cmd" $iter_opts \
--online-ivector-dir exp/nnet3/ivectors_${decode_set} \
$graph_dir data/${decode_set}_hires \
$dir/decode_${decode_set}${decode_iter:+_$decode_iter}_sw1_tg_looped || exit 1;
if $has_fisher; then
steps/lmrescore_const_arpa.sh --cmd "$decode_cmd" \
data/lang_sw1_{tg,fsh_fg} data/${decode_set}_hires \
$dir/decode_${decode_set}${decode_iter:+_$decode_iter}_sw1_{tg,fsh_fg}_looped || exit 1;
fi
) &
done
wait
if [ -f $dir/.error ]; then
echo "$0: something went wrong in looped decoding"
exit 1
fi
fi

if $test_online_decoding && [ $stage -le 17 ]; then
# note: if the features change (e.g. you add pitch features), you will have to
# change the options of the following command line.
steps/online/nnet3/prepare_online_decoding.sh \
--mfcc-config conf/mfcc_hires.conf \
$lang exp/nnet3/extractor $dir ${dir}_online

rm $dir/.error 2>/dev/null || true
for decode_set in train_dev eval2000; do
(
# note: we just give it "$decode_set" as it only uses the wav.scp, the
# feature type does not matter.

steps/online/nnet3/decode.sh --nj $decode_nj --cmd "$decode_cmd" $iter_opts \
--acwt 1.0 --post-decode-acwt 10.0 \
$graph_dir data/${decode_set}_hires \
${dir}_online/decode_${decode_set}${decode_iter:+_$decode_iter}_sw1_tg || exit 1;
if $has_fisher; then
steps/lmrescore_const_arpa.sh --cmd "$decode_cmd" \
data/lang_sw1_{tg,fsh_fg} data/${decode_set}_hires \
${dir}_online/decode_${decode_set}${decode_iter:+_$decode_iter}_sw1_{tg,fsh_fg} || exit 1;
fi
) || touch $dir/.error &
done
wait
if [ -f $dir/.error ]; then
echo "$0: something went wrong in online decoding"
exit 1
fi
fi

exit 0;
22 changes: 16 additions & 6 deletions egs/wsj/s5/steps/libs/nnet3/xconfig/basic_layers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Copyright 2016 Johns Hopkins University (Dan Povey)
# 2016 Vijayaditya Peddinti
# 2017 Google Inc. (vpeddinti@google.com)
# Apache 2.0.

""" This module contains the parent class from which all layers are inherited
Expand Down Expand Up @@ -775,7 +776,7 @@ def _add_components(self, input_desc, input_dim, nonlinearities):
# input='[-1]' [Descriptor giving the input of the layer.]
# dim=None [Output dimension of layer; defaults to the same as the input dim.]
# affine-transform-file='' [Must be specified.]
#
# delay=0 [Optional delay for the output-node in init.config]
class XconfigFixedAffineLayer(XconfigLayerBase):
def __init__(self, first_token, key_to_value, prev_names = None):
assert first_token == 'fixed-affine-layer'
Expand All @@ -787,6 +788,7 @@ def set_default_configs(self):
self.config = { 'input':'[-1]',
'dim':-1,
'affine-transform-file':'',
'delay':0,
'write-init-config':True}

def check_configs(self):
Expand Down Expand Up @@ -819,11 +821,19 @@ def get_full_config(self):
transform_file = self.config['affine-transform-file']

if self.config['write-init-config']:
# to init.config we write an output-node with the name 'output' and
# with a Descriptor equal to the descriptor that's the input to this
# layer. This will be used to accumulate stats to learn the LDA transform.
line = 'output-node name=output input={0}'.format(descriptor_final_string)
ans.append(('init', line))
if self.config['delay'] != 0:
line = 'component name={0}.delayed type=NoOpComponent dim={1}'.format(self.name, input_dim)
ans.append(('init', line))
line = 'component-node name={0}.delayed component={0}.delayed input={1}'.format(self.name, descriptor_final_string)
ans.append(('init', line))
line = 'output-node name=output input=Offset({0}.delayed, {1})'.format(self.name, self.config['delay'])
ans.append(('init', line))
else:
# to init.config we write an output-node with the name 'output' and
# with a Descriptor equal to the descriptor that's the input to this
# layer. This will be used to accumulate stats to learn the LDA transform.
line = 'output-node name=output input={0}'.format(descriptor_final_string)
ans.append(('init', line))

# write the 'real' component to final.config
line = 'component name={0} type=FixedAffineComponent matrix={1}'.format(
Expand Down
Loading