Skip to content

Add dropout to Xconfig #1589

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danpovey merged 1 commit into from
Apr 28, 2017
Merged

Add dropout to Xconfig #1589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 113 additions & 0 deletions egs/cifar/v1/local/nnet3/run_cnn_1c.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/bin/bash


# 1c uses dropout with fewer but larger layers

#teps/info/nnet3_dir_info.pl exp/cnn1c_cifar10
#exp/cnn1c_cifar10: num-iters=60 nj=1..2 num-params=4.3M dim=96->10 combine=-0.00->-0.00 loglike:train/valid[39,59,final]=(-0.08,-0.01,-0.00/-0.71,-0.79,-2.09) accuracy:train/valid[39,59,final]=(0.98,1.00,1.00/0.78,0.78,0.78)


# Set -e here so that we catch if any executable fails immediately
set -euo pipefail



# training options
stage=0
train_stage=-10
dataset=cifar10
srand=0
reporting_email=
affix=1c


# End configuration section.
echo "$0 $@" # Print the command line for logging

. ./cmd.sh
. ./path.sh
. ./utils/parse_options.sh

if ! cuda-compiled; then
cat <<EOF && exit 1
This script is intended to be used with GPUs but you have not compiled Kaldi with CUDA
If you want to use GPUs (and have them), go to src/, and configure and make on a machine
where "nvcc" is installed.
EOF
fi



dir=exp/cnn${affix}_${dataset}

egs=exp/${dataset}_egs

if [ ! -d $egs ]; then
echo "$0: expected directory $egs to exist. Run the get_egs.sh commands in the"
echo " run.sh before this script."
exit 1
fi

# check that the expected files are in the egs directory.

for f in $egs/egs.1.ark $egs/train_diagnostic.egs $egs/valid_diagnostic.egs $egs/combine.egs \
$egs/info/feat_dim $egs/info/left_context $egs/info/right_context \
$egs/info/output_dim; do
if [ ! -e $f ]; then
echo "$0: expected file $f to exist."
exit 1;
fi
done


mkdir -p $dir/log


if [ $stage -le 1 ]; then
mkdir -p $dir
echo "$0: creating neural net configs using the xconfig parser";

num_targets=$(cat $egs/info/output_dim)

# Note: we hardcode in the CNN config that we are dealing with 32x3x color
# images.

common1="required-time-offsets=0 height-offsets=-1,0,1 num-filters-out=32"
common2="required-time-offsets=0 height-offsets=-1,0,1 num-filters-out=64"

mkdir -p $dir/configs
cat <<EOF > $dir/configs/network.xconfig
input dim=96 name=input
conv-relu-layer name=cnn1 height-in=32 height-out=32 time-offsets=-1,0,1 $common1
conv-relu-dropout-layer name=cnn2 height-in=32 height-out=16 time-offsets=-1,0,1 dropout-proportion=0.25 $common1 height-subsample-out=2
conv-relu-layer name=cnn3 height-in=16 height-out=16 time-offsets=-1,0,1 $common2
conv-relu-dropout-layer name=cnn4 height-in=16 height-out=8 time-offsets=-1,0,1 dropout-proportion=0.25 $common2 height-subsample-out=2
relu-dropout-layer name=fully_connected1 input=Append(0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30) dropout-proportion=0.5 dim=512
output-layer name=output dim=$num_targets
EOF
steps/nnet3/xconfig_to_configs.py --xconfig-file $dir/configs/network.xconfig --config-dir $dir/configs/
fi


if [ $stage -le 2 ]; then

steps/nnet3/train_raw_dnn.py --stage=$train_stage \
--cmd="$train_cmd" \
--trainer.srand=$srand \
--trainer.max-param-change=2.0 \
--trainer.num-epochs=30 \
--egs.frames-per-eg=1 \
--trainer.optimization.num-jobs-initial=1 \
--trainer.optimization.num-jobs-final=2 \
--trainer.optimization.initial-effective-lrate=0.003 \
--trainer.optimization.final-effective-lrate=0.0003 \
--trainer.optimization.minibatch-size=256,128,64 \
--trainer.shuffle-buffer-size=2000 \
--egs.dir="$egs" \
--use-gpu=true \
--reporting.email="$reporting_email" \
--dir=$dir || exit 1;
fi


exit 0;
11 changes: 9 additions & 2 deletions egs/wsj/s5/steps/libs/nnet3/xconfig/basic_layers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,7 @@ def __init__(self, first_token, key_to_value, prev_names = None):
# Here we just list some likely combinations.. you can just add any
# combinations you want to use, to this list.
assert first_token in [ 'relu-layer', 'relu-renorm-layer', 'sigmoid-layer',
'tanh-layer', 'relu-batchnorm-layer' ]
'tanh-layer', 'relu-batchnorm-layer', 'relu-dropout-layer' ]
XconfigLayerBase.__init__(self, first_token, key_to_value, prev_names)

def set_default_configs(self):
Expand All @@ -641,7 +641,8 @@ def set_default_configs(self):
'self-repair-scale' : 1.0e-05,
'target-rms' : 1.0,
'learning-rate-factor' : 1.0,
'ng-affine-options' : ''}
'ng-affine-options' : '',
'dropout-proportion': 0.5}

def check_configs(self):
if self.config['dim'] < 0:
Expand Down Expand Up @@ -767,6 +768,12 @@ def _add_components(self, input_desc, input_dim, nonlinearities):
''.format(self.name, nonlinearity, output_dim,
target_rms))

elif nonlinearity == 'dropout':
line = ('component name={0}.{1} type=DropoutComponent '
'dim={2} dropout-proportion={3}'.format(
self.name, nonlinearity, output_dim,
self.config['dropout-proportion']))

else:
raise RuntimeError("Unknown nonlinearity type: {0}"
.format(nonlinearity))
Expand Down
14 changes: 11 additions & 3 deletions egs/wsj/s5/steps/libs/nnet3/xconfig/convolution.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@
class XconfigConvLayer(XconfigLayerBase):
def __init__(self, first_token, key_to_value, prev_names = None):
for operation in first_token.split('-')[:-1]:
assert operation in ['conv', 'renorm', 'batchnorm', 'relu']
assert operation in ['conv', 'renorm', 'batchnorm', 'relu', 'dropout']
XconfigLayerBase.__init__(self, first_token, key_to_value, prev_names)

def set_default_configs(self):
Expand All @@ -132,7 +132,8 @@ def set_default_configs(self):
'max-change': 0.75, 'learning-rate-factor':'',
'use-natural-gradient':'',
'rank-in':'', 'rank-out':'', 'num-minibatches-history':'',
'alpha-in':'', 'alpha-out':''}
'alpha-in':'', 'alpha-out':'',
'dropout-proportion': 0.5}

def set_derived_configs(self):
# sets 'num-filters-in'.
Expand Down Expand Up @@ -211,7 +212,7 @@ def output_name(self, auxiliary_output = None):
assert len(operations) > 1
last_operation = operations[-1]
assert last_operation in ['relu', 'conv',
'renorm', 'batchnorm']
'renorm', 'batchnorm', 'dropout']
# we'll return something like 'layer1.batchnorm'.
return '{0}.{1}'.format(self.name, last_operation)

Expand Down Expand Up @@ -291,6 +292,13 @@ def generate_cnn_config(self):
self.config['self-repair-scale']))
configs.append('component-node name={0}.relu component={0}.relu '
'input={1}'.format(name, cur_descriptor))
elif operation == 'dropout':
configs.append('component name={0}.dropout type=DropoutComponent '
'dim={1} dropout-proportion={2}'.format(
name, cur_num_filters * cur_height,
self.config['dropout-proportion']))
configs.append('component-node name={0}.dropout component={0}.dropout '
'input={1}'.format(name, cur_descriptor))
else:
raise RuntimeError("Un-handled operation type: " + operation)

Expand Down
5 changes: 4 additions & 1 deletion egs/wsj/s5/steps/libs/nnet3/xconfig/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,10 @@
'relu-conv-renorm-layer': xlayers.XconfigConvLayer,
'conv-relu-renorm-layer': xlayers.XconfigConvLayer,
'relu-conv-batchnorm-layer': xlayers.XconfigConvLayer,
'conv-relu-batchnorm-layer': xlayers.XconfigConvLayer
'conv-relu-batchnorm-layer': xlayers.XconfigConvLayer,
'conv-relu-dropout-layer': xlayers.XconfigConvLayer,
'relu-dropout-layer': xlayers.XconfigBasicLayer

}

# Turn a config line and a list of previous layers into
Expand Down