Skip to content

Commit b14d6fb

Browse files
nutszebranutszebra
committed
optimizer
1 parent 811654e commit b14d6fb

File tree

1 file changed

+190
-0
lines changed

1 file changed

+190
-0
lines changed

‎nutszebra_optimizer.py

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
import chainer
2+
from chainer import optimizers
3+
import nutszebra_basic_print
4+
5+
6+
class Optimizer(object):
7+
8+
def __init__(self, model=None):
9+
self.model = model
10+
self.optimizer = None
11+
12+
def __call__(self, i):
13+
pass
14+
15+
def update(self):
16+
self.optimizer.update()
17+
18+
19+
class OptimizerResnet(Optimizer):
20+
21+
def __init__(self, model=None, schedule=(int(32000. / (50000. / 128)), int(48000. / (50000. / 128))), lr=0.1, momentum=0.9, weight_decay=1.0e-4, warm_up_lr=0.01):
22+
super(OptimizerResnet, self).__init__(model)
23+
optimizer = optimizers.MomentumSGD(warm_up_lr, momentum)
24+
weight_decay = chainer.optimizer.WeightDecay(weight_decay)
25+
optimizer.setup(self.model)
26+
optimizer.add_hook(weight_decay)
27+
self.optimizer = optimizer
28+
self.schedule = schedule
29+
self.lr = lr
30+
self.warmup_lr = warm_up_lr
31+
self.momentum = momentum
32+
self.weight_decay = weight_decay
33+
34+
def __call__(self, i):
35+
if i == 1:
36+
lr = self.lr
37+
print('finishded warming up')
38+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
39+
self.optimizer.lr = lr
40+
if i in self.schedule:
41+
lr = self.optimizer.lr / 10
42+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
43+
self.optimizer.lr = lr
44+
45+
46+
class OptimizerDense(Optimizer):
47+
48+
def __init__(self, model=None, schedule=(150, 225), lr=0.1, momentum=0.9, weight_decay=1.0e-4):
49+
super(OptimizerDense, self).__init__(model)
50+
optimizer = optimizers.MomentumSGD(lr, momentum)
51+
weight_decay = chainer.optimizer.WeightDecay(weight_decay)
52+
optimizer.setup(self.model)
53+
optimizer.add_hook(weight_decay)
54+
self.optimizer = optimizer
55+
self.schedule = schedule
56+
self.lr = lr
57+
self.momentum = momentum
58+
self.weight_decay = weight_decay
59+
60+
def __call__(self, i):
61+
if i in self.schedule:
62+
lr = self.optimizer.lr / 10
63+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
64+
self.optimizer.lr = lr
65+
66+
67+
class OptimizerWideRes(Optimizer):
68+
69+
def __init__(self, model=None, schedule=(60, 120, 160), lr=0.1, momentum=0.9, weight_decay=5.0e-4):
70+
super(OptimizerWideRes, self).__init__(model)
71+
optimizer = optimizers.MomentumSGD(lr, momentum)
72+
weight_decay = chainer.optimizer.WeightDecay(weight_decay)
73+
optimizer.setup(self.model)
74+
optimizer.add_hook(weight_decay)
75+
self.optimizer = optimizer
76+
self.schedule = schedule
77+
self.lr = lr
78+
self.momentum = momentum
79+
self.weight_decay = weight_decay
80+
81+
def __call__(self, i):
82+
if i in self.schedule:
83+
lr = self.optimizer.lr * 0.2
84+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
85+
self.optimizer.lr = lr
86+
87+
88+
class OptimizerSwapout(Optimizer):
89+
90+
def __init__(self, model=None, schedule=(196, 224), lr=0.1, momentum=0.9, weight_decay=1.0e-4):
91+
super(OptimizerSwapout, self).__init__(model)
92+
optimizer = optimizers.MomentumSGD(lr, momentum)
93+
weight_decay = chainer.optimizer.WeightDecay(weight_decay)
94+
optimizer.setup(self.model)
95+
optimizer.add_hook(weight_decay)
96+
self.optimizer = optimizer
97+
self.schedule = schedule
98+
self.lr = lr
99+
self.momentum = momentum
100+
self.weight_decay = weight_decay
101+
102+
def __call__(self, i):
103+
if i in self.schedule:
104+
lr = self.optimizer.lr / 10
105+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
106+
self.optimizer.lr = lr
107+
108+
109+
class OptimizerXception(Optimizer):
110+
111+
def __init__(self, model=None, lr=0.045, momentum=0.9, weight_decay=1.0e-5, period=2):
112+
super(OptimizerXception, self).__init__(model)
113+
optimizer = optimizers.MomentumSGD(lr, momentum)
114+
weight_decay = chainer.optimizer.WeightDecay(weight_decay)
115+
optimizer.setup(self.model)
116+
optimizer.add_hook(weight_decay)
117+
self.optimizer = optimizer
118+
self.lr = lr
119+
self.momentum = momentum
120+
self.weight_decay = weight_decay
121+
self.period = int(period)
122+
123+
def __call__(self, i):
124+
if i % self.period == 0:
125+
lr = self.optimizer.lr * 0.94
126+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
127+
self.optimizer.lr = lr
128+
129+
130+
class OptimizerVGG(Optimizer):
131+
132+
def __init__(self, model=None, lr=0.01, momentum=0.9, weight_decay=5.0e-4):
133+
super(OptimizerVGG, self).__init__(model)
134+
optimizer = optimizers.MomentumSGD(lr, momentum)
135+
weight_decay = chainer.optimizer.WeightDecay(weight_decay)
136+
optimizer.setup(self.model)
137+
optimizer.add_hook(weight_decay)
138+
self.optimizer = optimizer
139+
self.lr = lr
140+
self.momentum = momentum
141+
self.weight_decay = weight_decay
142+
143+
def __call__(self, i):
144+
# 150 epoch means (0.94 ** 75) * lr
145+
# if lr is 0.01, then (0.94 ** 75) * 0.01 is 0.0001 at the end
146+
if i % 2 == 0:
147+
lr = self.optimizer.lr * 0.94
148+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
149+
self.optimizer.lr = lr
150+
151+
152+
class OptimizerGooglenet(Optimizer):
153+
154+
def __init__(self, model=None, lr=0.0015, momentum=0.9, weight_decay=2.0e-4):
155+
super(OptimizerGooglenet, self).__init__(model)
156+
optimizer = optimizers.MomentumSGD(lr, momentum)
157+
weight_decay = chainer.optimizer.WeightDecay(weight_decay)
158+
optimizer.setup(self.model)
159+
optimizer.add_hook(weight_decay)
160+
self.optimizer = optimizer
161+
self.lr = lr
162+
self.momentum = momentum
163+
self.weight_decay = weight_decay
164+
165+
def __call__(self, i):
166+
if i % 8 == 0:
167+
lr = self.optimizer.lr * 0.96
168+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
169+
self.optimizer.lr = lr
170+
171+
172+
class OptimizerNetworkInNetwork(Optimizer):
173+
174+
def __init__(self, model=None, lr=0.1, momentum=0.9, weight_decay=1.0e-4, schedule=(int(1.0e5 / (50000. / 128)), )):
175+
super(OptimizerNetworkInNetwork, self).__init__(model)
176+
optimizer = optimizers.MomentumSGD(lr, momentum)
177+
weight_decay = chainer.optimizer.WeightDecay(weight_decay)
178+
optimizer.setup(self.model)
179+
optimizer.add_hook(weight_decay)
180+
self.optimizer = optimizer
181+
self.lr = lr
182+
self.momentum = momentum
183+
self.weight_decay = weight_decay
184+
self.schedule = schedule
185+
186+
def __call__(self, i):
187+
if i in self.schedule:
188+
lr = self.optimizer.lr / 10
189+
print('lr is changed: {} -> {}'.format(self.optimizer.lr, lr))
190+
self.optimizer.lr = lr

0 commit comments

Comments
 (0)