-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathresnet50_pruned_70_schedule.yaml
178 lines (162 loc) · 15.9 KB
/
resnet50_pruned_70_schedule.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
# This schedule demonstrates high-rate element-wise pruning (70.66% sparsity) of Resnet 50.
# ************************************************************
# Accuracy Performance
# ************************************************************
# +----------------------+------------------+----------------+
# | Accuracy Metric | Pruned Model | Base Model |
# +----------------------+------------------+----------------+
# | Top1 | 75.944 | 76.15 |
# | Top5 | 92.96 | 92.87 |
# +----------------------+------------------+----------------+
# ************************************************************
# Distiller Command Line
# ************************************************************
# Training : python compress_classifier.py -a=resnet50 --pretrained -p=100 PATH_TO_IMAGENET_DATA -j=8 -b=32 --epochs=96 --lr=0.001 --compress=PATH_TO_THIS_FILE
# Evaluation : python compress_classifier.py -a resnet50 --resume PATH_TO_CHEKCPOINT PATH_TO_IMAGENET_VAL_DATA --evaluate
# Quantization : python compress_classifier.py -a resnet50 --resume PATH_TO_CHEKCPOINT PATH_TO_IMAGENET_VAL_DATA --evaluate --quantize-eval --qe-mode asym_u
# ************************************************************
# Best epoch (90)
# ************************************************************
# ************************************************************
# Sparsity Profile
# ************************************************************
# The first layers are left unpruned, because the weights tensors are very small. The arithmetic-intensity is
# especially low, and the weight tensors are large, in module.layer4.*, so it's important to prune those.
# The Linear (fully-connected) layer is pruned to 87% because we have empirical evidence that the classifier layers
# are prune-friendly.
#
# Parameters:
# +----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------+
# | | Name | Shape | NNZ (dense) | NNZ (sparse) | Cols (%) | Rows (%) | Ch (%) | 2D (%) | 3D (%) | Fine (%) | Std | Mean | Abs-Mean |
# |----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------|
# | 0 | module.conv1.weight | (64, 3, 7, 7) | 9408 | 9408 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.11248 | -0.00037 | 0.07019 |
# | 1 | module.layer1.0.conv1.weight | (64, 64, 1, 1) | 4096 | 4096 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.06613 | -0.00432 | 0.03783 |
# | 2 | module.layer1.0.conv2.weight | (64, 64, 3, 3) | 36864 | 36864 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02664 | 0.00076 | 0.01601 |
# | 3 | module.layer1.0.conv3.weight | (256, 64, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03260 | 0.00044 | 0.01968 |
# | 4 | module.layer1.0.downsample.0.weight | (256, 64, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.05307 | -0.00313 | 0.02958 |
# | 5 | module.layer1.1.conv1.weight | (64, 256, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02803 | 0.00103 | 0.01868 |
# | 6 | module.layer1.1.conv2.weight | (64, 64, 3, 3) | 36864 | 36864 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02638 | 0.00006 | 0.01796 |
# | 7 | module.layer1.1.conv3.weight | (256, 64, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03028 | 0.00008 | 0.01902 |
# | 8 | module.layer1.2.conv1.weight | (64, 256, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02786 | -0.00017 | 0.02013 |
# | 9 | module.layer1.2.conv2.weight | (64, 64, 3, 3) | 36864 | 36864 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02887 | -0.00078 | 0.02168 |
# | 10 | module.layer1.2.conv3.weight | (256, 64, 1, 1) | 16384 | 16384 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.02831 | -0.00189 | 0.01717 |
# | 11 | module.layer2.0.conv1.weight | (128, 256, 1, 1) | 32768 | 32768 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.03205 | -0.00119 | 0.02269 |
# | 12 | module.layer2.0.conv2.weight | (128, 128, 3, 3) | 147456 | 44237 | 0.00000 | 0.00000 | 0.00000 | 15.66162 | 0.00000 | 69.99986 | 0.01620 | -0.00023 | 0.00756 |
# | 13 | module.layer2.0.conv3.weight | (512, 128, 1, 1) | 65536 | 19661 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 12.50000 | 69.99969 | 0.02147 | 0.00010 | 0.00930 |
# | 14 | module.layer2.0.downsample.0.weight | (512, 256, 1, 1) | 131072 | 39322 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 12.30469 | 69.99969 | 0.01695 | -0.00024 | 0.00688 |
# | 15 | module.layer2.1.conv1.weight | (128, 512, 1, 1) | 65536 | 19661 | 0.00000 | 0.00000 | 12.30469 | 69.99969 | 0.00000 | 69.99969 | 0.01358 | 0.00007 | 0.00575 |
# | 16 | module.layer2.1.conv2.weight | (128, 128, 3, 3) | 147456 | 44237 | 0.00000 | 0.00000 | 0.00000 | 20.31250 | 0.00000 | 69.99986 | 0.01606 | 0.00013 | 0.00707 |
# | 17 | module.layer2.1.conv3.weight | (512, 128, 1, 1) | 65536 | 19661 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 1.36719 | 69.99969 | 0.01822 | -0.00089 | 0.00755 |
# | 18 | module.layer2.2.conv1.weight | (128, 512, 1, 1) | 65536 | 19661 | 0.00000 | 0.00000 | 1.56250 | 69.99969 | 0.00000 | 69.99969 | 0.01784 | -0.00051 | 0.00798 |
# | 19 | module.layer2.2.conv2.weight | (128, 128, 3, 3) | 147456 | 44237 | 0.00000 | 0.00000 | 0.00000 | 12.85400 | 0.00000 | 69.99986 | 0.01648 | -0.00014 | 0.00737 |
# | 20 | module.layer2.2.conv3.weight | (512, 128, 1, 1) | 65536 | 19661 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.00000 | 69.99969 | 0.01998 | -0.00028 | 0.00901 |
# | 21 | module.layer2.3.conv1.weight | (128, 512, 1, 1) | 65536 | 19661 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.00000 | 69.99969 | 0.01814 | -0.00055 | 0.00841 |
# | 22 | module.layer2.3.conv2.weight | (128, 128, 3, 3) | 147456 | 44237 | 0.00000 | 0.00000 | 0.00000 | 10.16846 | 0.00000 | 69.99986 | 0.01668 | -0.00040 | 0.00781 |
# | 23 | module.layer2.3.conv3.weight | (512, 128, 1, 1) | 65536 | 19661 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.00000 | 69.99969 | 0.01881 | -0.00054 | 0.00841 |
# | 24 | module.layer3.0.conv1.weight | (256, 512, 1, 1) | 131072 | 39322 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.00000 | 69.99969 | 0.02244 | -0.00071 | 0.01041 |
# | 25 | module.layer3.0.conv2.weight | (256, 256, 3, 3) | 589824 | 176948 | 0.00000 | 0.00000 | 0.00000 | 19.08569 | 0.00000 | 69.99986 | 0.01255 | -0.00014 | 0.00568 |
# | 26 | module.layer3.0.conv3.weight | (1024, 256, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 4.00391 | 69.99969 | 0.01755 | -0.00009 | 0.00805 |
# | 27 | module.layer3.0.downsample.0.weight | (1024, 512, 1, 1) | 524288 | 157287 | 0.00000 | 0.00000 | 0.00000 | 69.99989 | 3.90625 | 69.99989 | 0.01184 | 0.00005 | 0.00512 |
# | 28 | module.layer3.1.conv1.weight | (256, 1024, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 4.39453 | 69.99969 | 0.00000 | 69.99969 | 0.01273 | -0.00025 | 0.00567 |
# | 29 | module.layer3.1.conv2.weight | (256, 256, 3, 3) | 589824 | 176948 | 0.00000 | 0.00000 | 0.00000 | 14.56604 | 0.00000 | 69.99986 | 0.01192 | -0.00013 | 0.00539 |
# | 30 | module.layer3.1.conv3.weight | (1024, 256, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.48828 | 69.99969 | 0.01600 | -0.00083 | 0.00723 |
# | 31 | module.layer3.2.conv1.weight | (256, 1024, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.58594 | 69.99969 | 0.00000 | 69.99969 | 0.01260 | -0.00029 | 0.00555 |
# | 32 | module.layer3.2.conv2.weight | (256, 256, 3, 3) | 589824 | 176948 | 0.00000 | 0.00000 | 0.00000 | 10.07385 | 0.00000 | 69.99986 | 0.01158 | -0.00043 | 0.00530 |
# | 33 | module.layer3.2.conv3.weight | (1024, 256, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.09766 | 69.99969 | 0.01460 | -0.00043 | 0.00663 |
# | 34 | module.layer3.3.conv1.weight | (256, 1024, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.09766 | 69.99969 | 0.00000 | 69.99969 | 0.01373 | -0.00038 | 0.00618 |
# | 35 | module.layer3.3.conv2.weight | (256, 256, 3, 3) | 589824 | 176948 | 0.00000 | 0.00000 | 0.00000 | 9.75800 | 0.00000 | 69.99986 | 0.01161 | -0.00041 | 0.00535 |
# | 36 | module.layer3.3.conv3.weight | (1024, 256, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.09766 | 69.99969 | 0.01428 | -0.00070 | 0.00654 |
# | 37 | module.layer3.4.conv1.weight | (256, 1024, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.09766 | 69.99969 | 0.00000 | 69.99969 | 0.01420 | -0.00051 | 0.00648 |
# | 38 | module.layer3.4.conv2.weight | (256, 256, 3, 3) | 589824 | 176948 | 0.00000 | 0.00000 | 0.00000 | 10.54840 | 0.00000 | 69.99986 | 0.01159 | -0.00051 | 0.00537 |
# | 39 | module.layer3.4.conv3.weight | (1024, 256, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.00000 | 69.99969 | 0.01442 | -0.00098 | 0.00663 |
# | 40 | module.layer3.5.conv1.weight | (256, 1024, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.00000 | 69.99969 | 0.01516 | -0.00045 | 0.00704 |
# | 41 | module.layer3.5.conv2.weight | (256, 256, 3, 3) | 589824 | 176948 | 0.00000 | 0.00000 | 0.00000 | 12.30621 | 0.00000 | 69.99986 | 0.01186 | -0.00051 | 0.00554 |
# | 42 | module.layer3.5.conv3.weight | (1024, 256, 1, 1) | 262144 | 78644 | 0.00000 | 0.00000 | 0.00000 | 69.99969 | 0.00000 | 69.99969 | 0.01562 | -0.00151 | 0.00733 |
# | 43 | module.layer4.0.conv1.weight | (512, 1024, 1, 1) | 524288 | 157287 | 0.00000 | 0.00000 | 0.00000 | 69.99989 | 0.00000 | 69.99989 | 0.01767 | -0.00075 | 0.00852 |
# | 44 | module.layer4.0.conv2.weight | (512, 512, 3, 3) | 2359296 | 707789 | 0.00000 | 0.00000 | 0.00000 | 20.89386 | 0.00000 | 69.99999 | 0.00950 | -0.00021 | 0.00448 |
# | 45 | module.layer4.0.conv3.weight | (2048, 512, 1, 1) | 1048576 | 314573 | 0.00000 | 0.00000 | 0.00000 | 69.99998 | 0.00000 | 69.99998 | 0.01291 | -0.00035 | 0.00612 |
# | 46 | module.layer4.0.downsample.0.weight | (2048, 1024, 1, 1) | 2097152 | 629146 | 0.00000 | 0.00000 | 0.00000 | 69.99998 | 0.00000 | 69.99998 | 0.00867 | -0.00011 | 0.00391 |
# | 47 | module.layer4.1.conv1.weight | (512, 2048, 1, 1) | 1048576 | 314573 | 0.00000 | 0.00000 | 0.00000 | 69.99998 | 0.00000 | 69.99998 | 0.01320 | -0.00042 | 0.00620 |
# | 48 | module.layer4.1.conv2.weight | (512, 512, 3, 3) | 2359296 | 707789 | 0.00000 | 0.00000 | 0.00000 | 18.00766 | 0.00000 | 69.99999 | 0.00989 | -0.00043 | 0.00472 |
# | 49 | module.layer4.1.conv3.weight | (2048, 512, 1, 1) | 1048576 | 314573 | 0.00000 | 0.00000 | 0.00000 | 69.99998 | 0.00000 | 69.99998 | 0.01329 | 0.00004 | 0.00633 |
# | 50 | module.layer4.2.conv1.weight | (512, 2048, 1, 1) | 1048576 | 314573 | 0.00000 | 0.00000 | 0.00000 | 69.99998 | 0.00000 | 69.99998 | 0.01561 | -0.00012 | 0.00746 |
# | 51 | module.layer4.2.conv2.weight | (512, 512, 3, 3) | 2359296 | 707789 | 0.00000 | 0.00000 | 0.00000 | 36.40137 | 0.00000 | 69.99999 | 0.00822 | -0.00030 | 0.00407 |
# | 52 | module.layer4.2.conv3.weight | (2048, 512, 1, 1) | 1048576 | 314573 | 0.00000 | 0.00000 | 0.00000 | 69.99998 | 0.00000 | 69.99998 | 0.01085 | 0.00013 | 0.00512 |
# | 53 | module.fc.weight | (1000, 2048) | 2048000 | 266240 | 0.00000 | 0.24414 | 0.00000 | 0.00000 | 0.00000 | 87.00000 | 0.03213 | 0.00547 | 0.01052 |
# | 54 | Total sparsity: | - | 25502912 | 7481351 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 0.00000 | 70.66472 | 0.00000 | 0.00000 | 0.00000 |
# +----+-------------------------------------+--------------------+---------------+----------------+------------+------------+----------+----------+----------+------------+---------+----------+------------+
# Total sparsity: 70.66
# Ideal Number of MACS: 1.846 Billion
version: 1
pruners:
fc_pruner:
class: AutomatedGradualPruner
initial_sparsity : 0.05
final_sparsity: 0.87
weights: module.fc.weight
mid_pruner:
class: AutomatedGradualPruner
initial_sparsity : 0.05
final_sparsity: 0.70
weights: [
module.layer2.0.conv2.weight,
module.layer2.0.conv3.weight,
module.layer2.0.downsample.0.weight,
module.layer2.1.conv1.weight,
module.layer2.1.conv2.weight,
module.layer2.1.conv3.weight,
module.layer2.2.conv1.weight,
module.layer2.2.conv2.weight,
module.layer2.2.conv3.weight,
module.layer2.3.conv1.weight,
module.layer2.3.conv2.weight,
module.layer2.3.conv3.weight,
module.layer3.0.conv1.weight,
module.layer3.0.conv2.weight,
module.layer3.0.conv3.weight,
module.layer3.0.downsample.0.weight,
module.layer3.1.conv1.weight,
module.layer3.1.conv2.weight,
module.layer3.1.conv3.weight,
module.layer3.2.conv1.weight,
module.layer3.2.conv2.weight,
module.layer3.2.conv3.weight,
module.layer3.3.conv1.weight,
module.layer3.3.conv2.weight,
module.layer3.3.conv3.weight,
module.layer3.4.conv1.weight,
module.layer3.4.conv2.weight,
module.layer3.4.conv3.weight,
module.layer3.5.conv1.weight,
module.layer3.5.conv2.weight,
module.layer3.5.conv3.weight,
module.layer4.0.conv1.weight,
module.layer4.0.conv2.weight,
module.layer4.0.conv3.weight,
module.layer4.0.downsample.0.weight,
module.layer4.1.conv1.weight,
module.layer4.1.conv2.weight,
module.layer4.1.conv3.weight,
module.layer4.2.conv1.weight,
module.layer4.2.conv2.weight,
module.layer4.2.conv3.weight]
lr_schedulers:
pruning_lr:
class: ExponentialLR
gamma: 0.95
policies:
- pruner:
instance_name : mid_pruner
starting_epoch: 0
ending_epoch: 30
frequency: 2
- pruner:
instance_name : fc_pruner
starting_epoch: 1
ending_epoch: 29
frequency: 2
- lr_scheduler:
instance_name: pruning_lr
starting_epoch: 40
ending_epoch: 100
frequency: 1