Skip to content

Commit 662553c

Browse files
nickottonickotto
authored
Merge pull request #103 from perib/hyperparameter_demo
Hyperparameter mutation probabilities and gradual changes
2 parents d7ff57f + 3ea1729 commit 662553c

File tree

4 files changed

+250
-85
lines changed

4 files changed

+250
-85
lines changed

‎tpot2/config/hyperparametersuggestor.py

+173-72
Original file line numberDiff line numberDiff line change
@@ -2,88 +2,189 @@
22
from scipy.stats import loguniform, logser #TODO: remove this dependency?
33
import numpy as np #TODO: remove this dependency and use scipy instead?
44

5+
#function that selects selects items from a list with each having independent probability p of being selected
6+
def select(items, p):
7+
selected = [item for item in items if random.random() < p]
8+
#if selected is empty, select one item at random
9+
if not selected:
10+
return [random.choice(items)]
11+
return selected
512

613

14+
class Trial():
715

8-
#Replicating the API found in optuna: https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html
9-
#copy-pasted some code
10-
def suggest_categorical(name, choices):
11-
return random.choice(choices)
12-
13-
def suggest_float(
14-
name: str,
15-
low: float,
16-
high: float,
17-
*,
18-
step = None,
19-
log = False,
20-
):
21-
22-
if log and step is not None:
23-
raise ValueError("The parameter `step` is not supported when `log` is true.")
24-
25-
if low > high:
26-
raise ValueError(
27-
"The `low` value must be smaller than or equal to the `high` value "
28-
"(low={}, high={}).".format(low, high)
29-
)
30-
31-
if log and low <= 0.0:
32-
raise ValueError(
33-
"The `low` value must be larger than 0 for a log distribution "
34-
"(low={}, high={}).".format(low, high)
35-
)
36-
37-
if step is not None and step <= 0:
38-
raise ValueError(
39-
"The `step` value must be non-zero positive value, " "but step={}.".format(step)
40-
)
41-
42-
#TODO check this produces correct output
43-
if log:
44-
value = np.random.uniform(np.log(low),np.log(high))
45-
return np.e**value
46-
47-
else:
48-
if step is not None:
49-
return np.random.choice(np.arange(low,high,step))
16+
def __init__(self, old_params=None, alpha=1, hyperparameter_probability=1):
17+
self._params = dict()
18+
19+
self.old_params = old_params
20+
self.alpha = alpha
21+
self.hyperparameter_probability = hyperparameter_probability
22+
23+
if old_params is not None and len(old_params) > 0:
24+
self.params_to_update = select(list(old_params.keys()), self.hyperparameter_probability)
5025
else:
51-
return np.random.uniform(low,high)
26+
self.params_to_update = None
27+
28+
29+
#Replicating the API found in optuna: https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html
30+
#copy-pasted some code
31+
def suggest_categorical(self, name, choices):
32+
if self.params_to_update == None or name in self.params_to_update or name not in self.old_params: #If this parameter is selected to be changed
33+
choice = self.suggest_categorical_(name, choices)
34+
else: #if this parameter is not selected to be changed
35+
choice = self.old_params[name]
36+
if choice not in choices: #if the old value is not in the choices, then we need to choose a value for it
37+
choice = self.suggest_categorical_(name, choices)
38+
39+
self._params[name] = choice
40+
return choice
41+
42+
def suggest_float(self,
43+
name: str,
44+
low: float,
45+
high: float,
46+
*,
47+
step = None,
48+
log = False,
49+
):
50+
if self.params_to_update == None or name in self.params_to_update or name not in self.old_params: #If this parameter is selected to be changed
51+
choice = self.suggest_float_(name, low=low, high=high, step=step, log=log)
52+
if self.old_params is not None and name in self.old_params:
53+
choice = self.alpha*choice + (1-self.alpha)*self.old_params[name]
54+
else: #if this parameter is not selected to be changed
55+
choice = self.old_params[name]
56+
57+
self._params[name] = choice
58+
return choice
59+
60+
61+
62+
def suggest_discrete_uniform(self, name, low, high, q):
63+
if self.params_to_update == None or name in self.params_to_update or name not in self.old_params:
64+
choice = self.suggest_discrete_uniform_(name, low=low, high=high, q=q)
65+
if self.old_params is not None and name in self.old_params:
66+
choice = self.alpha*choice + (1-self.alpha)*self.old_params[name]
67+
else:
68+
choice = self.old_params[name]
5269

70+
self._params[name] = choice
71+
return choice
5372

54-
def suggest_discrete_uniform(name, low, high, q):
55-
return suggest_float(name, low, high, step=q)
5673

5774

58-
def suggest_int(name, low, high, step=1, log=False):
59-
if low == high: #TODO check that this matches optuna's behaviour
60-
return low
61-
62-
if log and step >1:
63-
raise ValueError("The parameter `step`>1 is not supported when `log` is true.")
75+
def suggest_int(self, name, low, high, step=1, log=False):
76+
if self.params_to_update == None or name in self.params_to_update or name not in self.old_params:
77+
choice = self.suggest_int_(name, low=low, high=high, step=step, log=log)
78+
if self.old_params is not None and name in self.old_params:
79+
choice = int(self.alpha*choice + (1-self.alpha)*self.old_params[name])
80+
else:
81+
choice = self.old_params[name]
6482

65-
if low > high:
66-
raise ValueError(
67-
"The `low` value must be smaller than or equal to the `high` value "
68-
"(low={}, high={}).".format(low, high)
69-
)
83+
self._params[name] = choice
84+
return choice
7085

71-
if log and low <= 0.0:
72-
raise ValueError(
73-
"The `low` value must be larger than 0 for a log distribution "
74-
"(low={}, high={}).".format(low, high)
75-
)
7686

77-
if step is not None and step <= 0:
78-
raise ValueError(
79-
"The `step` value must be non-zero positive value, " "but step={}.".format(step)
80-
)
87+
def suggest_uniform(self, name, low, high):
88+
if self.params_to_update == None or name in self.params_to_update or name not in self.old_params:
89+
choice = self.suggest_uniform_(name, low=low, high=high)
90+
if self.old_params is not None and name in self.old_params:
91+
choice = self.alpha*choice + (1-self.alpha)*self.old_params[name]
92+
else:
93+
choice = self.old_params[name]
94+
95+
self._params[name] = choice
96+
return choice
97+
98+
99+
100+
####################################
101+
#Replicating the API found in optuna: https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html
102+
#copy-pasted some code
103+
def suggest_categorical_(self, name, choices):
104+
105+
choice = random.choice(choices)
106+
return choice
107+
108+
def suggest_float_(self,
109+
name: str,
110+
low: float,
111+
high: float,
112+
*,
113+
step = None,
114+
log = False,
115+
):
116+
117+
if log and step is not None:
118+
raise ValueError("The parameter `step` is not supported when `log` is true.")
119+
120+
if low > high:
121+
raise ValueError(
122+
"The `low` value must be smaller than or equal to the `high` value "
123+
"(low={}, high={}).".format(low, high)
124+
)
125+
126+
if log and low <= 0.0:
127+
raise ValueError(
128+
"The `low` value must be larger than 0 for a log distribution "
129+
"(low={}, high={}).".format(low, high)
130+
)
131+
132+
if step is not None and step <= 0:
133+
raise ValueError(
134+
"The `step` value must be non-zero positive value, " "but step={}.".format(step)
135+
)
136+
137+
#TODO check this produces correct output
138+
if log:
139+
value = np.random.uniform(np.log(low),np.log(high))
140+
choice = np.e**value
141+
return choice
81142

82-
if log:
83-
value = np.random.uniform(np.log(low),np.log(high))
84-
return int(np.e**value)
85-
else:
86-
return np.random.choice(list(range(low,high,step)))
143+
else:
144+
if step is not None:
145+
choice = np.random.choice(np.arange(low,high,step))
146+
return choice
147+
else:
148+
choice = np.random.uniform(low,high)
149+
return choice
150+
151+
152+
def suggest_discrete_uniform_(self, name, low, high, q):
153+
choice = self.suggest_float(name, low, high, step=q)
154+
return choice
155+
156+
157+
def suggest_int_(self, name, low, high, step=1, log=False):
158+
if low == high: #TODO check that this matches optuna's behaviour
159+
return low
160+
161+
if log and step >1:
162+
raise ValueError("The parameter `step`>1 is not supported when `log` is true.")
163+
164+
if low > high:
165+
raise ValueError(
166+
"The `low` value must be smaller than or equal to the `high` value "
167+
"(low={}, high={}).".format(low, high)
168+
)
169+
170+
if log and low <= 0.0:
171+
raise ValueError(
172+
"The `low` value must be larger than 0 for a log distribution "
173+
"(low={}, high={}).".format(low, high)
174+
)
175+
176+
if step is not None and step <= 0:
177+
raise ValueError(
178+
"The `step` value must be non-zero positive value, " "but step={}.".format(step)
179+
)
180+
181+
if log:
182+
value = np.random.uniform(np.log(low),np.log(high))
183+
choice = int(np.e**value)
184+
return choice
185+
else:
186+
choice = np.random.choice(list(range(low,high,step)))
187+
return choice
87188

88-
def suggest_uniform(name, low, high):
89-
return suggest_float(name, low, high)
189+
def suggest_uniform_(self, name, low, high):
190+
return self.suggest_float(name, low, high)

0 commit comments

Comments
 (0)