From 02063cd4642376bdebcd697e926f766c268c8603 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 28 Nov 2016 15:55:31 -0500 Subject: [PATCH 001/154] point mutation operator and codes clean --- test_point_mut.py | 9 ++++++ tpot/base.py | 73 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 test_point_mut.py diff --git a/test_point_mut.py b/test_point_mut.py new file mode 100644 index 00000000..1c3a2824 --- /dev/null +++ b/test_point_mut.py @@ -0,0 +1,9 @@ +from sklearn.datasets import make_classification +from tpot import TPOTClassifier + +X, y = make_classification(n_samples=200, n_features=80, + n_informative=2, n_redundant=10, + random_state=42) + +tpot = TPOTClassifier(generations=5, crossover_rate= 0.5, population_size=20, verbosity = 2) +tpot.fit(X, y) diff --git a/tpot/base.py b/tpot/base.py index 60d92ed5..a64b167c 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -25,6 +25,7 @@ import sys from functools import partial from datetime import datetime +from inspect import isclass import numpy as np import deap @@ -561,7 +562,7 @@ def _evaluate_individual(self, individual, features, classes): # Count the number of pipeline operators as a measure of pipeline complexity operator_count = 0 - + # add time limit for evaluation of pipeline for i in range(len(individual)): node = individual[i] @@ -613,6 +614,65 @@ def _combined_selection_operator(self, individuals, k): """ return tools.selNSGA2(individuals, int(k / 5.)) * 5 + # point mutation function + def _mutNodeReplacement(self, individual, pset): + """Replaces a randomly chosen primitive from *individual* by a randomly + chosen primitive no matter if it has the same number of arguments from the :attr:`pset` + attribute of the individual. + Parameters + ---------- + individual: DEAP individual + A list of pipeline operators and model parameters that can be + compiled by DEAP into a callable function + + Returns + ------- + individual: DEAP individual + Returns the individual with one of point mutation applied to it + + """ + + index = random.randrange(len(individual)) + node = individual[index] + slice_ = individual.searchSubtree(index) + + if node.arity == 0: # Terminal + term = np.random.choice(pset.terminals[node.ret]) + if isclass(term): + term = term() + individual[index] = term + else: # Primitive + # find next primitive if any + rindex = None + if index + 1 < len(individual): + for i, tmpnode in enumerate(individual[index+1:], index+ 1): + if isinstance(tmpnode, deap.gp.Primitive) and tmpnode.ret in tmpnode.args: + rindex = i + primitives = [p for p in pset.primitives[node.ret]] + if len(primitives) != 0: + new_node = np.random.choice(primitives) + new_subtree = [None] * len(new_node.args) + if rindex: + rnode = individual[rindex] + rslice = individual.searchSubtree(rindex) + # find position for passing return values to next operator + position = np.random.choice([i for i, a in enumerate(new_node.args) if a == rnode.ret]) + else: + position = None + for i, arg_type in enumerate(new_node.args): + if i != position: + term = np.random.choice(pset.terminals[arg_type]) + if isclass(term): + term = term() + new_subtree[i] = term + # paste the subtree to new node + if rindex: + new_subtree[position:position + 1] = individual[rslice] + # combine with primitives + new_subtree.insert(0, new_node) + individual[slice_] = new_subtree + return individual, + def _random_mutation_operator(self, individual): """Perform a replacement, insert, or shrink mutation on an individual @@ -624,16 +684,23 @@ def _random_mutation_operator(self, individual): Returns ------- - fitness: list + mut_ind: DEAP individual Returns the individual with one of the mutations applied to it """ + # debug usage + #print(str(individual)) + mutation_techniques = [ partial(gp.mutUniform, expr=self._toolbox.expr_mut, pset=self._pset), partial(gp.mutInsert, pset=self._pset), + partial(self._mutNodeReplacement, pset=self._pset), partial(gp.mutShrink) ] - return np.random.choice(mutation_techniques)(individual) + mut_ind = np.random.choice(mutation_techniques)(individual) + # debug usage + #print(str(mut_ind[0]),'\n') + return mut_ind def _gen_grow_safe(self, pset, min_, max_, type_=None): """Generate an expression where each leaf might have a different depth From eb919a4eea6125694b154dc6f5d60d555749aab9 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 28 Nov 2016 16:01:40 -0500 Subject: [PATCH 002/154] clean testfile --- test_point_mut.py | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 test_point_mut.py diff --git a/test_point_mut.py b/test_point_mut.py deleted file mode 100644 index 1c3a2824..00000000 --- a/test_point_mut.py +++ /dev/null @@ -1,9 +0,0 @@ -from sklearn.datasets import make_classification -from tpot import TPOTClassifier - -X, y = make_classification(n_samples=200, n_features=80, - n_informative=2, n_redundant=10, - random_state=42) - -tpot = TPOTClassifier(generations=5, crossover_rate= 0.5, population_size=20, verbosity = 2) -tpot.fit(X, y) From 2ef694a47fffc709d87101f3d25c94835aaa1b14 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 5 Dec 2016 16:51:37 -0500 Subject: [PATCH 003/154] add unit test and force mutation happen --- tests.py | 17 +++++++++++++++++ tpot/base.py | 12 ++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests.py b/tests.py index b0912cb7..aa85bf7d 100644 --- a/tests.py +++ b/tests.py @@ -314,6 +314,23 @@ def test_generate_import_code(): assert expected_code == generate_import_code(pipeline) +def test_mutNodeReplacement(): + """Assert that _mutNodeReplacement() returns the correct type of mutation node in a random pipeline""" + tpot_obj = TPOTClassifier() + pipeline = tpot_obj._toolbox.population(n =1)[0] # generated with gen_grow_safe function + old_ret_type_list = [node.ret for node in pipeline] + old_prims_list = [node for node in pipeline if node.arity != 0] + mut_ind = tpot_obj._mutNodeReplacement(pipeline, pset = tpot_obj._pset) + + new_ret_type_list = [node.ret for node in mut_ind[0]] + new_prims_list = [node for node in mut_ind[0] if node.arity != 0] + if new_prims_list == old_prims_list: # Terminal mutated + assert new_ret_type_list == old_ret_type_list + else: # Primitive mutated + diff_prims = list(set(new_prims_list).symmetric_difference(old_prims_list)) + assert diff_prims[0].ret == diff_prims[1].ret + assert mut_ind[0][0].ret == Output_DF + def test_export_pipeline(): """Assert that exported_pipeline() generated a compile source file as expected given a fixed pipeline""" diff --git a/tpot/base.py b/tpot/base.py index a64b167c..441ce4dd 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -648,7 +648,10 @@ def _mutNodeReplacement(self, individual, pset): for i, tmpnode in enumerate(individual[index+1:], index+ 1): if isinstance(tmpnode, deap.gp.Primitive) and tmpnode.ret in tmpnode.args: rindex = i - primitives = [p for p in pset.primitives[node.ret]] + #pset.primitives[node.ret] can get a list of the type of node + # for example: if op.root is True then the node.ret is Output_DF object + # based on the function _setup_pset. Then primitives is the list of classifor or regressor + primitives = pset.primitives[node.ret] if len(primitives) != 0: new_node = np.random.choice(primitives) new_subtree = [None] * len(new_node.args) @@ -690,14 +693,15 @@ def _random_mutation_operator(self, individual): """ # debug usage #print(str(individual)) - + old_ind = str(individual) + mut_ind = (str(individual),) mutation_techniques = [ - partial(gp.mutUniform, expr=self._toolbox.expr_mut, pset=self._pset), partial(gp.mutInsert, pset=self._pset), partial(self._mutNodeReplacement, pset=self._pset), partial(gp.mutShrink) ] - mut_ind = np.random.choice(mutation_techniques)(individual) + while str(mut_ind[0]) == old_ind: # infinite loop to make sure mutation happen + mut_ind = np.random.choice(mutation_techniques)(individual) # debug usage #print(str(mut_ind[0]),'\n') return mut_ind From 6029384b4d21bd980802be5620e0456857acc787 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 7 Dec 2016 15:24:14 -0500 Subject: [PATCH 004/154] new unit tests on a fixed pipeline --- tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests.py b/tests.py index aa85bf7d..405e0397 100644 --- a/tests.py +++ b/tests.py @@ -315,13 +315,15 @@ def test_generate_import_code(): assert expected_code == generate_import_code(pipeline) def test_mutNodeReplacement(): - """Assert that _mutNodeReplacement() returns the correct type of mutation node in a random pipeline""" + """Assert that _mutNodeReplacement() returns the correct type of mutation node in a fixed pipeline""" tpot_obj = TPOTClassifier() - pipeline = tpot_obj._toolbox.population(n =1)[0] # generated with gen_grow_safe function + pipeline = creator.Individual.\ + from_string("KNeighborsClassifier(CombineDFs(GradientBoostingClassifier(input_matrix, 38.0, 0.87), SelectKBest(input_matrix, 5)), 18, 33)", tpot_obj._pset) + # change the last operato's type to Output_DF as op.root = True + pipeline[0].ret = Output_DF old_ret_type_list = [node.ret for node in pipeline] old_prims_list = [node for node in pipeline if node.arity != 0] mut_ind = tpot_obj._mutNodeReplacement(pipeline, pset = tpot_obj._pset) - new_ret_type_list = [node.ret for node in mut_ind[0]] new_prims_list = [node for node in mut_ind[0] if node.arity != 0] if new_prims_list == old_prims_list: # Terminal mutated From 3415e2a76bf9022d1065c9f9d4a737c0e60f3998 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 19 Dec 2016 16:31:17 -0500 Subject: [PATCH 005/154] move mutation to deap_gp --- tests.py | 5 ++-- tpot/base.py | 66 ++----------------------------------------------- tpot/gp_deap.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 70 insertions(+), 67 deletions(-) diff --git a/tests.py b/tests.py index 7c27e9d7..7d3362f1 100644 --- a/tests.py +++ b/tests.py @@ -10,6 +10,7 @@ from tpot.export_utils import export_pipeline, generate_import_code, _indent, generate_pipeline_code from tpot.decorators import _gp_new_generation from tpot.gp_types import Output_DF +from tpot.gp_deap import mutNodeReplacement from tpot.operators import Operator from tpot.operators.selectors import TPOTSelectKBest @@ -354,7 +355,7 @@ def test_generate_import_code(): assert expected_code == generate_import_code(pipeline) def test_mutNodeReplacement(): - """Assert that _mutNodeReplacement() returns the correct type of mutation node in a fixed pipeline""" + """Assert that mutNodeReplacement() returns the correct type of mutation node in a fixed pipeline""" tpot_obj = TPOTClassifier() pipeline = creator.Individual.\ from_string("KNeighborsClassifier(CombineDFs(GradientBoostingClassifier(input_matrix, 38.0, 0.87), SelectKBest(input_matrix, 5)), 18, 33)", tpot_obj._pset) @@ -362,7 +363,7 @@ def test_mutNodeReplacement(): pipeline[0].ret = Output_DF old_ret_type_list = [node.ret for node in pipeline] old_prims_list = [node for node in pipeline if node.arity != 0] - mut_ind = tpot_obj._mutNodeReplacement(pipeline, pset = tpot_obj._pset) + mut_ind = mutNodeReplacement(pipeline, pset = tpot_obj._pset) new_ret_type_list = [node.ret for node in mut_ind[0]] new_prims_list = [node for node in mut_ind[0] if node.arity != 0] if new_prims_list == old_prims_list: # Terminal mutated diff --git a/tpot/base.py b/tpot/base.py index d1d63fdc..1b9dd397 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -48,7 +48,7 @@ from .operators import CombineDFs from .gp_types import Bool, Output_DF from .metrics import SCORERS -from .gp_deap import eaSimple +from .gp_deap import eaSimple, mutNodeReplacement # hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS @@ -669,68 +669,6 @@ def _combined_selection_operator(self, individuals, k): """ return tools.selNSGA2(individuals, int(k / 5.)) * 5 - # point mutation function - def _mutNodeReplacement(self, individual, pset): - """Replaces a randomly chosen primitive from *individual* by a randomly - chosen primitive no matter if it has the same number of arguments from the :attr:`pset` - attribute of the individual. - Parameters - ---------- - individual: DEAP individual - A list of pipeline operators and model parameters that can be - compiled by DEAP into a callable function - - Returns - ------- - individual: DEAP individual - Returns the individual with one of point mutation applied to it - - """ - - index = random.randrange(len(individual)) - node = individual[index] - slice_ = individual.searchSubtree(index) - - if node.arity == 0: # Terminal - term = np.random.choice(pset.terminals[node.ret]) - if isclass(term): - term = term() - individual[index] = term - else: # Primitive - # find next primitive if any - rindex = None - if index + 1 < len(individual): - for i, tmpnode in enumerate(individual[index+1:], index+ 1): - if isinstance(tmpnode, deap.gp.Primitive) and tmpnode.ret in tmpnode.args: - rindex = i - #pset.primitives[node.ret] can get a list of the type of node - # for example: if op.root is True then the node.ret is Output_DF object - # based on the function _setup_pset. Then primitives is the list of classifor or regressor - primitives = pset.primitives[node.ret] - if len(primitives) != 0: - new_node = np.random.choice(primitives) - new_subtree = [None] * len(new_node.args) - if rindex: - rnode = individual[rindex] - rslice = individual.searchSubtree(rindex) - # find position for passing return values to next operator - position = np.random.choice([i for i, a in enumerate(new_node.args) if a == rnode.ret]) - else: - position = None - for i, arg_type in enumerate(new_node.args): - if i != position: - term = np.random.choice(pset.terminals[arg_type]) - if isclass(term): - term = term() - new_subtree[i] = term - # paste the subtree to new node - if rindex: - new_subtree[position:position + 1] = individual[rslice] - # combine with primitives - new_subtree.insert(0, new_node) - individual[slice_] = new_subtree - return individual, - def _random_mutation_operator(self, individual): """Perform a replacement, insert, or shrink mutation on an individual @@ -753,7 +691,7 @@ def _random_mutation_operator(self, individual): mut_ind = (str(individual),) mutation_techniques = [ partial(gp.mutInsert, pset=self._pset), - partial(self._mutNodeReplacement, pset=self._pset), + partial(mutNodeReplacement, pset=self._pset), partial(gp.mutShrink) ] while str(mut_ind[0]) == old_ind: # infinite loop to make sure mutation happen diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index c517ea88..85a246c9 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -18,8 +18,9 @@ """ import random - +import numpy as np from deap import tools +from inspect import isclass def varAnd(population, toolbox, cxpb, mutpb): """Part of an evolutionary algorithm applying only the variation part @@ -172,3 +173,66 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, print(logbook.stream) return population, logbook + + +# point mutation function +def mutNodeReplacement(individual, pset): + """Replaces a randomly chosen primitive from *individual* by a randomly + chosen primitive no matter if it has the same number of arguments from the :attr:`pset` + attribute of the individual. + Parameters + ---------- + individual: DEAP individual + A list of pipeline operators and model parameters that can be + compiled by DEAP into a callable function + + Returns + ------- + individual: DEAP individual + Returns the individual with one of point mutation applied to it + + """ + + index = random.randrange(len(individual)) + node = individual[index] + slice_ = individual.searchSubtree(index) + + if node.arity == 0: # Terminal + term = np.random.choice(pset.terminals[node.ret]) + if isclass(term): + term = term() + individual[index] = term + else: # Primitive + # find next primitive if any + rindex = None + if index + 1 < len(individual): + for i, tmpnode in enumerate(individual[index+1:], index+ 1): + if isinstance(tmpnode, deap.gp.Primitive) and tmpnode.ret in tmpnode.args: + rindex = i + #pset.primitives[node.ret] can get a list of the type of node + # for example: if op.root is True then the node.ret is Output_DF object + # based on the function _setup_pset. Then primitives is the list of classifor or regressor + primitives = pset.primitives[node.ret] + if len(primitives) != 0: + new_node = np.random.choice(primitives) + new_subtree = [None] * len(new_node.args) + if rindex: + rnode = individual[rindex] + rslice = individual.searchSubtree(rindex) + # find position for passing return values to next operator + position = np.random.choice([i for i, a in enumerate(new_node.args) if a == rnode.ret]) + else: + position = None + for i, arg_type in enumerate(new_node.args): + if i != position: + term = np.random.choice(pset.terminals[arg_type]) + if isclass(term): + term = term() + new_subtree[i] = term + # paste the subtree to new node + if rindex: + new_subtree[position:position + 1] = individual[rslice] + # combine with primitives + new_subtree.insert(0, new_node) + individual[slice_] = new_subtree + return individual, From c2173a267bfd1313f56c1c8e2104a44fad25fee1 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 19 Dec 2016 16:41:48 -0500 Subject: [PATCH 006/154] clean codes --- tpot/gp_deap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 85a246c9..8dc5c015 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -19,7 +19,7 @@ import random import numpy as np -from deap import tools +from deap import tools, gp from inspect import isclass def varAnd(population, toolbox, cxpb, mutpb): @@ -207,7 +207,7 @@ def mutNodeReplacement(individual, pset): rindex = None if index + 1 < len(individual): for i, tmpnode in enumerate(individual[index+1:], index+ 1): - if isinstance(tmpnode, deap.gp.Primitive) and tmpnode.ret in tmpnode.args: + if isinstance(tmpnode, gp.Primitive) and tmpnode.ret in tmpnode.args: rindex = i #pset.primitives[node.ret] can get a list of the type of node # for example: if op.root is True then the node.ret is Output_DF object From 106659cb0bfbf0fb52fbc7cbecebd2061c8bd61a Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 20 Dec 2016 11:02:03 -0500 Subject: [PATCH 007/154] catch none pipeline error --- tpot/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 1b9dd397..e62007bb 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -388,15 +388,15 @@ def pareto_eq(ind1, ind2): self._optimized_pipeline = pipeline # It won't raise error for a small test like in a unit test becasue a few pipeline sometimes # may fail due to the training data does not fit the operator's requirement. - if self.generations*self.population_size > 5 and not self._optimized_pipeline: - raise ValueError('There was an error in the TPOT optimization ' + if not self._optimized_pipeline: + print('There was an error in the TPOT optimization ' 'process. This could be because the data was ' 'not formatted properly, or because data for ' 'a regression problem was provided to the ' 'TPOTClassifier object. Please make sure you ' 'passed the data to TPOT correctly.') - - self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) + else: + self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) with warnings.catch_warnings(): warnings.simplefilter('ignore') From d13937b5e355181cf33744eb1ae37d040a6e7de2 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 20 Dec 2016 11:08:54 -0500 Subject: [PATCH 008/154] clean codes --- tpot/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index e62007bb..cc006998 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -398,9 +398,9 @@ def pareto_eq(ind1, ind2): else: self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self._fitted_pipeline.fit(features, classes) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self._fitted_pipeline.fit(features, classes) if self.verbosity in [1, 2] and self._optimized_pipeline: # Add an extra line of spacing if the progress bar was used From a6b58fb2bea1a01308fead13d0cfb6cc05672a37 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 20 Dec 2016 11:12:46 -0500 Subject: [PATCH 009/154] fix output failed pipeline when verbosity >= 3 --- tpot/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index cc006998..5a7ce663 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -398,9 +398,9 @@ def pareto_eq(ind1, ind2): else: self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self._fitted_pipeline.fit(features, classes) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self._fitted_pipeline.fit(features, classes) if self.verbosity in [1, 2] and self._optimized_pipeline: # Add an extra line of spacing if the progress bar was used @@ -409,7 +409,7 @@ def pareto_eq(ind1, ind2): print('Best pipeline: {}'.format(self._optimized_pipeline)) # Store and fit the entire Pareto front if sciencing - elif self.verbosity >= 3 and self._pareto_front: + elif self.verbosity >= 3 and self._optimized_pipeline: self._pareto_front_fitted_pipelines = {} for pipeline in self._pareto_front.items: From dca89b29d43a1916be1995c6cd25eced477bf09b Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 20 Dec 2016 11:17:49 -0500 Subject: [PATCH 010/154] clean codes --- tpot/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index 5a7ce663..7057d13f 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -386,7 +386,6 @@ def pareto_eq(ind1, ind2): for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline - # It won't raise error for a small test like in a unit test becasue a few pipeline sometimes # may fail due to the training data does not fit the operator's requirement. if not self._optimized_pipeline: print('There was an error in the TPOT optimization ' From bf0fa9494399db413a4782f916c80c10c3ff331d Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 20 Dec 2016 11:30:29 -0500 Subject: [PATCH 011/154] failed pipeline cannot run fit() --- tpot/base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 7057d13f..10708984 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -396,10 +396,9 @@ def pareto_eq(ind1, ind2): 'passed the data to TPOT correctly.') else: self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self._fitted_pipeline.fit(features, classes) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self._fitted_pipeline.fit(features, classes) if self.verbosity in [1, 2] and self._optimized_pipeline: # Add an extra line of spacing if the progress bar was used From 1f2a1dd63e6e2d658a5d80c93108e9d200fc9e8c Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 23 Dec 2016 15:06:45 -0500 Subject: [PATCH 012/154] add multiprocessing --- tpot/base.py | 172 +++++++++++++++++++++++++++++++++++++++++------- tpot/gp_deap.py | 17 +++-- tpot_test.py | 20 ++++++ 3 files changed, 180 insertions(+), 29 deletions(-) create mode 100644 tpot_test.py diff --git a/tpot/base.py b/tpot/base.py index 63565d56..6b4c9434 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -25,6 +25,8 @@ import sys from functools import partial from datetime import datetime +from pathos.multiprocessing import Pool +#from joblib import Parallel, delayed import numpy as np import deap @@ -39,6 +41,7 @@ from sklearn.metrics.scorer import make_scorer from update_checker import update_check +from joblib import Parallel, delayed from ._version import __version__ from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code @@ -50,6 +53,8 @@ from .gp_deap import eaSimple + + # hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS if sys.platform.startswith('win'): import win32api @@ -64,7 +69,7 @@ def handler(dwCtrlType, hook_sigint=_thread.interrupt_main): return 0 win32api.SetConsoleCtrlHandler(handler, 1) # add time limit for imported function -cross_val_score = _timeout(cross_val_score) +#cross_val_score = _timeout(cross_val_score) class TPOTBase(BaseEstimator): @@ -315,7 +320,7 @@ def fit(self, features, classes, sample_weight = None): self._start_datetime = datetime.now() - self._toolbox.register('evaluate', self._evaluate_individual, features=features, classes=classes, sample_weight=sample_weight) + self._toolbox.register('evaluate', self._evaluate_individuals, features=features, classes=classes, sample_weight=sample_weight) # assign population, self._pop can only be not None if warm_start is enabled if self._pop: @@ -323,28 +328,10 @@ def fit(self, features, classes, sample_weight = None): else: pop = self._toolbox.population(n=self.population_size) - def pareto_eq(ind1, ind2): - """Determines whether two individuals are equal on the Pareto front - - Parameters - ---------- - ind1: DEAP individual from the GP population - First individual to compare - ind2: DEAP individual from the GP population - Second individual to compare - - Returns - ---------- - individuals_equal: bool - Boolean indicating whether the two individuals are equal on - the Pareto front - - """ - return np.all(ind1.fitness.values == ind2.fitness.values) # generate new pareto front if it doesn't already exist for warm start if not self.warm_start or not self._pareto_front: - self._pareto_front = tools.ParetoFront(similar=pareto_eq) + self._pareto_front = tools.ParetoFront(similar=self._pareto_eq) # Start the progress bar if self.max_time_mins: @@ -383,10 +370,11 @@ def pareto_eq(ind1, ind2): if self._pareto_front: top_score = -float('inf') for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): + print(pipeline_scores.wvalues) if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline # It won't raise error for a small test like in a unit test becasue a few pipeline sometimes - # may fail due to the training data does not fit the operator's requirement. + # may fail due to the training data does not fit the operator's requirement. if self.generations*self.population_size > 5 and not self._optimized_pipeline: raise ValueError('There was an error in the TPOT optimization ' 'process. This could be because the data was ' @@ -648,6 +636,106 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No else: raise ValueError('Scoring function does not return a float') + def _evaluate_individuals(self, individuals, features, classes, sample_weight = None): + """Determines the `individual`'s fitness + + Parameters + ---------- + individuals: a list of DEAP individual + One individual is a list of pipeline operators and model parameters that can be + compiled by DEAP into a callable function + features: numpy.ndarray {n_samples, n_features} + A numpy matrix containing the training and testing features for the + `individual`'s evaluation + classes: numpy.ndarray {n_samples, } + A numpy matrix containing the training and testing classes for the + `individual`'s evaluation + + Returns + ------- + fitness: float + Returns a float value indicating the `individual`'s fitness + according to its performance on the provided data + + """ + if self.max_time_mins: + total_mins_elapsed = (datetime.now() - self._start_datetime).total_seconds() / 60. + if total_mins_elapsed >= self.max_time_mins: + raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) + # return individuals with fitness scores + ret_individuals = [] + # 3 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing + eval_individuals = [] + sklearn_pipeline_list = [] + operator_count_list = [] + for individual in individuals: + # Disallow certain combinations of operators because they will take too long or take up too much RAM + # This is a fairly hacky way to prevent TPOT from getting stuck on bad pipelines and should be improved in a future release + individual_str = str(individual) + if (individual_str.count('PolynomialFeatures') > 1): + print('Invalid pipeline -- skipping its evaluation') + individual.fitness.value = (max(1, operator_count), resulting_score) + ret_individuals.append(individual) + if not self._pbar.disable: + self._pbar.update(1) + + # check if the individual are evaluated before + elif individual_str in self.eval_ind: + # get fitness score from previous evaluation + individual.fitness.value = self.eval_ind[individual_str] + if self.verbosity == 3: + self._pbar.write("Pipeline #{0} has been evaluated previously. " + "Continuing to the next pipeline.".format(self._pbar.n + 1)) + ret_individuals.append(individual) + if not self._pbar.disable: + self._pbar.update(1) + + else: + # Transform the tree expression into an sklearn pipeline + sklearn_pipeline = self._toolbox.compile(expr=individual) + + # Fix random state when the operator allows and build sample weight dictionary + sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) + + # Count the number of pipeline operators as a measure of pipeline complexity + operator_count = 0 + # add time limit for evaluation of pipeline + for i in range(len(individual)): + node = individual[i] + if ((type(node) is deap.gp.Terminal) or + type(node) is deap.gp.Primitive and node.name == 'CombineDFs'): + continue + operator_count += 1 + + eval_individuals.append(individual) + operator_count_list.append(operator_count) + sklearn_pipeline_list.append(sklearn_pipeline) + + # make partial for pool.map + partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, + num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) + + pool = Pool(processes=2) + """parallel = Parallel(n_jobs=2, verbose=0) + resulting_score_list = parallel(delayed(wrapped_cross_val_score)(sklearn_pipeline, features, classes, + self.num_cv_folds, self.scoring_function, sample_weight_dict) for sklearn_pipeline in sklearn_pipeline_list)""" + resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + #print(resulting_score_list) + + for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): + individual_str = str(individual) + if type(resulting_score) in [float, np.float64, np.float32]: + self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) + individual.fitness.value = self.eval_ind[individual_str] + else: + raise ValueError('Scoring function does not return a float') + ret_individuals.append(individual) + + for ind in ret_individuals: + print(ind.fitness.value) + return ret_individuals + + @_gp_new_generation def _combined_selection_operator(self, individuals, k): """Perform NSGA2 selection on the population according to their Pareto fitness @@ -668,6 +756,8 @@ def _combined_selection_operator(self, individuals, k): return tools.selNSGA2(individuals, int(k / 5.)) * 5 + + def _random_mutation_operator(self, individual): """Perform a replacement, insert, or shrink mutation on an individual @@ -780,3 +870,41 @@ def _generate(self, pset, min_, max_, condition, type_=None): stack.append((depth+1, arg)) return expr + + def _pareto_eq(self, ind1, ind2): + """Determines whether two individuals are equal on the Pareto front + + Parameters + ---------- + ind1: DEAP individual from the GP population + First individual to compare + ind2: DEAP individual from the GP population + Second individual to compare + + Returns + ---------- + individuals_equal: bool + Boolean indicating whether the two individuals are equal on + the Pareto front + + """ + return np.all(ind1.fitness.values == ind2.fitness.values) + + + def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_folds, scoring_function, sample_weight_dict): + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + cv_scores = cross_val_score(sklearn_pipeline, features, classes, + cv=num_cv_folds, scoring=scoring_function, + n_jobs=1, fit_params=sample_weight_dict) + try: + resulting_score = np.mean(cv_scores) + except TypeError: + raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') + except: + resulting_score = -float('inf') + print(resulting_score) + if not self._pbar.disable: + self._pbar.update(1) + return resulting_score diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index c517ea88..b5015b44 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -17,10 +17,11 @@ with the TPOT library. If not, see http://www.gnu.org/licenses/. """ -import random +import numpy as np from deap import tools + def varAnd(population, toolbox, cxpb, mutpb): """Part of an evolutionary algorithm applying only the variation part (crossover **and** mutation). The modified individuals have their @@ -57,7 +58,7 @@ def varAnd(population, toolbox, cxpb, mutpb): # Apply crossover and mutation on the offspring for i in range(1, len(offspring), 2): - if random.random() < cxpb: + if np.random.random() < cxpb: ind1, ind2 = str(offspring[i - 1]), str(offspring[i]) offspring[i - 1], offspring[i] = toolbox.mate(offspring[i - 1], offspring[i]) for child in [offspring[i - 1], offspring[i]]: @@ -66,7 +67,7 @@ def varAnd(population, toolbox, cxpb, mutpb): del child.fitness.values for i in range(len(offspring)): - if random.random() < mutpb: + if np.random.random() < mutpb: tmpind = str(offspring[i]) offspring[i], = toolbox.mutate(offspring[i]) if tmpind != str(offspring[i]): @@ -132,9 +133,10 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in population if not ind.fitness.valid] - fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) + """fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): - ind.fitness.values = fit + ind.fitness.values = fit""" + invalid_ind = toolbox.evaluate(invalid_ind) if halloffame is not None: halloffame.update(population) @@ -154,9 +156,10 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in offspring if not ind.fitness.valid] - fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) + """fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): - ind.fitness.values = fit + ind.fitness.values = fit""" + invalid_ind = toolbox.evaluate(invalid_ind) # Update the hall of fame with the generated individuals if halloffame is not None: diff --git a/tpot_test.py b/tpot_test.py new file mode 100644 index 00000000..4aa6a1d4 --- /dev/null +++ b/tpot_test.py @@ -0,0 +1,20 @@ +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split +import time + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.25, test_size=0.75) + +#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) +#time_start = time.time() +#tpot.fit(X_train, y_train) +#print(tpot.score(X_test, y_test)) +#print('\nTime used with num_cpu = 1:',time.time()-time_start) + +tpot = TPOTClassifier(generations=1, population_size=5, verbosity=0, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used with num_cpu = 2:',time.time()-time_start) From 5a0324536369f2a3d2a4d556494e5d7f3362cb55 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 31 Dec 2016 13:02:58 -0500 Subject: [PATCH 013/154] fit bugs in pool and null fitness scores --- tpot/base.py | 62 +++++++++++++++++++++++++++++-------------------- tpot/gp_deap.py | 9 +++++-- tpot_test.py | 2 +- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 6b4c9434..760d9b56 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -327,6 +327,7 @@ def fit(self, features, classes, sample_weight = None): pop = self._pop else: pop = self._toolbox.population(n=self.population_size) + print(self.population_size, len(pop)) # generate new pareto front if it doesn't already exist for warm start @@ -370,7 +371,7 @@ def fit(self, features, classes, sample_weight = None): if self._pareto_front: top_score = -float('inf') for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): - print(pipeline_scores.wvalues) + print(pipeline,pipeline_scores.wvalues) if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline # It won't raise error for a small test like in a unit test becasue a few pipeline sometimes @@ -664,6 +665,9 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) # return individuals with fitness scores ret_individuals = [] + fitnesses = [] + num_ind = len(individuals) + print(num_ind) # 3 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing eval_individuals = [] sklearn_pipeline_list = [] @@ -674,15 +678,20 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): print('Invalid pipeline -- skipping its evaluation') - individual.fitness.value = (max(1, operator_count), resulting_score) - ret_individuals.append(individual) + #individual.fitness.value = (max(1, operator_count), resulting_score) + fitness = (max(1, operator_count), resulting_score) + fitnesses.append(fitness) + #ret_individuals.append(individual) if not self._pbar.disable: self._pbar.update(1) # check if the individual are evaluated before elif individual_str in self.eval_ind: # get fitness score from previous evaluation - individual.fitness.value = self.eval_ind[individual_str] + #individual.fitness.value = self.eval_ind[individual_str] + fitness = self.eval_ind[individual_str] + fitnesses.append(fitness) + print('duplicated pipeline', self.eval_ind[individual_str]) if self.verbosity == 3: self._pbar.write("Pipeline #{0} has been evaluated previously. " "Continuing to the next pipeline.".format(self._pbar.n + 1)) @@ -712,28 +721,31 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = sklearn_pipeline_list.append(sklearn_pipeline) # make partial for pool.map - partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, - num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) - - pool = Pool(processes=2) - """parallel = Parallel(n_jobs=2, verbose=0) - resulting_score_list = parallel(delayed(wrapped_cross_val_score)(sklearn_pipeline, features, classes, - self.num_cv_folds, self.scoring_function, sample_weight_dict) for sklearn_pipeline in sklearn_pipeline_list)""" - resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) - #print(resulting_score_list) - - for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): - individual_str = str(individual) - if type(resulting_score) in [float, np.float64, np.float32]: - self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) - individual.fitness.value = self.eval_ind[individual_str] - else: - raise ValueError('Scoring function does not return a float') - ret_individuals.append(individual) + """for ind in ret_individuals: + print(ind.fitness.value)""" + partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, + num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) + + pool = Pool(processes=2) + """parallel = Parallel(n_jobs=2, verbose=0) + resulting_score_list = parallel(delayed(wrapped_cross_val_score)(sklearn_pipeline, features, classes, + self.num_cv_folds, self.scoring_function, sample_weight_dict) for sklearn_pipeline in sklearn_pipeline_list)""" + resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + print(len(resulting_score_list),resulting_score_list) + + print('after_evaluation',len(resulting_score_list), len(operator_count_list)) + for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): + individual_str = str(individual) + if type(resulting_score) in [float, np.float64, np.float32]: + self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) + fitness = self.eval_ind[individual_str] + fitnesses.append(fitness) + else: + raise ValueError('Scoring function does not return a float') - for ind in ret_individuals: - print(ind.fitness.value) - return ret_individuals + print('eval_done') + #return ret_individuals + return fitnesses @_gp_new_generation diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index b5015b44..a95fc032 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -136,7 +136,9 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, """fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit""" - invalid_ind = toolbox.evaluate(invalid_ind) + fitnesses = toolbox.evaluate(invalid_ind) + for ind, fit in zip(invalid_ind, fitnesses): + ind.fitness.values = fit if halloffame is not None: halloffame.update(population) @@ -148,6 +150,7 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, # Begin the generational process for gen in range(1, ngen + 1): + print('Gen:', gen) # Select the next generation individuals offspring = toolbox.select(population, len(population)) @@ -159,7 +162,9 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, """fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit""" - invalid_ind = toolbox.evaluate(invalid_ind) + fitnesses = toolbox.evaluate(invalid_ind) + for ind, fit in zip(invalid_ind, fitnesses): + ind.fitness.values = fit # Update the hall of fame with the generated individuals if halloffame is not None: diff --git a/tpot_test.py b/tpot_test.py index 4aa6a1d4..0fe2a2c8 100644 --- a/tpot_test.py +++ b/tpot_test.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=1, population_size=5, verbosity=0, random_state = 42) +tpot = TPOTClassifier(generations=2, population_size=5, verbosity=0, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From da190da58bc02ecd64e86a790554e710fe525bd2 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 31 Dec 2016 13:32:32 -0500 Subject: [PATCH 014/154] clean codes, pbar works and fix sample_weight_dict bug --- tpot/base.py | 166 +++++++++--------------------------------------- tpot/gp_deap.py | 7 -- tpot_test.py | 4 +- 3 files changed, 32 insertions(+), 145 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 760d9b56..848189fe 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -327,7 +327,6 @@ def fit(self, features, classes, sample_weight = None): pop = self._pop else: pop = self._toolbox.population(n=self.population_size) - print(self.population_size, len(pop)) # generate new pareto front if it doesn't already exist for warm start @@ -371,41 +370,40 @@ def fit(self, features, classes, sample_weight = None): if self._pareto_front: top_score = -float('inf') for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): - print(pipeline,pipeline_scores.wvalues) if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline # It won't raise error for a small test like in a unit test becasue a few pipeline sometimes # may fail due to the training data does not fit the operator's requirement. - if self.generations*self.population_size > 5 and not self._optimized_pipeline: - raise ValueError('There was an error in the TPOT optimization ' + if not self._optimized_pipeline: + print('There was an error in the TPOT optimization ' 'process. This could be because the data was ' 'not formatted properly, or because data for ' 'a regression problem was provided to the ' 'TPOTClassifier object. Please make sure you ' 'passed the data to TPOT correctly.') + else: + self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) - self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self._fitted_pipeline.fit(features, classes) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self._fitted_pipeline.fit(features, classes) - if self.verbosity in [1, 2] and self._optimized_pipeline: - # Add an extra line of spacing if the progress bar was used - if self.verbosity >= 2: - print('') - print('Best pipeline: {}'.format(self._optimized_pipeline)) + if self.verbosity in [1, 2]: + # Add an extra line of spacing if the progress bar was used + if self.verbosity >= 2: + print('') + print('Best pipeline: {}'.format(self._optimized_pipeline)) - # Store and fit the entire Pareto front if sciencing - elif self.verbosity >= 3 and self._pareto_front: - self._pareto_front_fitted_pipelines = {} + # Store and fit the entire Pareto front if sciencing + elif self.verbosity >= 3 and self._pareto_front: + self._pareto_front_fitted_pipelines = {} - for pipeline in self._pareto_front.items: - self._pareto_front_fitted_pipelines[str(pipeline)] = self._toolbox.compile(expr=pipeline) + for pipeline in self._pareto_front.items: + self._pareto_front_fitted_pipelines[str(pipeline)] = self._toolbox.compile(expr=pipeline) - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self._pareto_front_fitted_pipelines[str(pipeline)].fit(features, classes) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self._pareto_front_fitted_pipelines[str(pipeline)].fit(features, classes) def predict(self, features): """Uses the optimized pipeline to predict the classes for a feature set @@ -552,91 +550,6 @@ def _set_param_recursive(self, pipeline_steps, parameter, value, sample_weight = return None - def _evaluate_individual(self, individual, features, classes, sample_weight = None): - """Determines the `individual`'s fitness - - Parameters - ---------- - individual: DEAP individual - A list of pipeline operators and model parameters that can be - compiled by DEAP into a callable function - features: numpy.ndarray {n_samples, n_features} - A numpy matrix containing the training and testing features for the - `individual`'s evaluation - classes: numpy.ndarray {n_samples, } - A numpy matrix containing the training and testing classes for the - `individual`'s evaluation - - Returns - ------- - fitness: float - Returns a float value indicating the `individual`'s fitness - according to its performance on the provided data - - """ - try: - if self.max_time_mins: - total_mins_elapsed = (datetime.now() - self._start_datetime).total_seconds() / 60. - if total_mins_elapsed >= self.max_time_mins: - raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) - - # Disallow certain combinations of operators because they will take too long or take up too much RAM - # This is a fairly hacky way to prevent TPOT from getting stuck on bad pipelines and should be improved in a future release - individual_str = str(individual) - if (individual_str.count('PolynomialFeatures') > 1): - raise ValueError('Invalid pipeline -- skipping its evaluation') - - # Transform the tree expression into an sklearn pipeline - sklearn_pipeline = self._toolbox.compile(expr=individual) - - # Fix random state when the operator allows and build sample weight dictionary - sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) - - # Count the number of pipeline operators as a measure of pipeline complexity - operator_count = 0 - - # check if the individual are evaluated before - if individual_str in self.eval_ind: - # get fitness score from previous evaluation - operator_count, resulting_score = self.eval_ind[individual_str] - if self.verbosity == 3: - self._pbar.write("Pipeline #{0} has been evaluated previously. " - "Continuing to the next pipeline.".format(self._pbar.n + 1)) - else: - # add time limit for evaluation of pipeline - for i in range(len(individual)): - node = individual[i] - if ((type(node) is deap.gp.Terminal) or - type(node) is deap.gp.Primitive and node.name == 'CombineDFs'): - continue - operator_count += 1 - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - cv_scores = cross_val_score(self, sklearn_pipeline, features, classes, - cv=self.num_cv_folds, scoring=self.scoring_function, - n_jobs=self.n_jobs, fit_params=sample_weight_dict) - try: - resulting_score = np.mean(cv_scores) - except TypeError: - raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') - - except Exception: - # Catch-all: Do not allow one pipeline that crashes to cause TPOT - # to crash. Instead, assign the crashing pipeline a poor fitness - # import traceback - # traceback.print_exc() - return 5000., -float('inf') - finally: - if not self._pbar.disable: - self._pbar.update(1) # One more pipeline evaluated - - if type(resulting_score) in [float, np.float64, np.float32]: - self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) - return max(1, operator_count), resulting_score - else: - raise ValueError('Scoring function does not return a float') - def _evaluate_individuals(self, individuals, features, classes, sample_weight = None): """Determines the `individual`'s fitness @@ -654,8 +567,8 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = Returns ------- - fitness: float - Returns a float value indicating the `individual`'s fitness + fitnesses: float + Returns a list of tuple value indicating the `individual`'s fitness according to its performance on the provided data """ @@ -663,11 +576,11 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = total_mins_elapsed = (datetime.now() - self._start_datetime).total_seconds() / 60. if total_mins_elapsed >= self.max_time_mins: raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) - # return individuals with fitness scores - ret_individuals = [] + if not sample_weight: + sample_weight_dict = None + + # return fitness scores fitnesses = [] - num_ind = len(individuals) - print(num_ind) # 3 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing eval_individuals = [] sklearn_pipeline_list = [] @@ -678,24 +591,18 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): print('Invalid pipeline -- skipping its evaluation') - #individual.fitness.value = (max(1, operator_count), resulting_score) fitness = (max(1, operator_count), resulting_score) fitnesses.append(fitness) - #ret_individuals.append(individual) if not self._pbar.disable: self._pbar.update(1) # check if the individual are evaluated before elif individual_str in self.eval_ind: # get fitness score from previous evaluation - #individual.fitness.value = self.eval_ind[individual_str] - fitness = self.eval_ind[individual_str] - fitnesses.append(fitness) - print('duplicated pipeline', self.eval_ind[individual_str]) + fitnesses.append(self.eval_ind[individual_str]) if self.verbosity == 3: self._pbar.write("Pipeline #{0} has been evaluated previously. " "Continuing to the next pipeline.".format(self._pbar.n + 1)) - ret_individuals.append(individual) if not self._pbar.disable: self._pbar.update(1) @@ -720,31 +627,19 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = operator_count_list.append(operator_count) sklearn_pipeline_list.append(sklearn_pipeline) - # make partial for pool.map - """for ind in ret_individuals: - print(ind.fitness.value)""" partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) - - pool = Pool(processes=2) - """parallel = Parallel(n_jobs=2, verbose=0) - resulting_score_list = parallel(delayed(wrapped_cross_val_score)(sklearn_pipeline, features, classes, - self.num_cv_folds, self.scoring_function, sample_weight_dict) for sklearn_pipeline in sklearn_pipeline_list)""" + # parallel computing in evaluation of pipeline + pool = Pool(processes=self.n_jobs) resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) - print(len(resulting_score_list),resulting_score_list) - print('after_evaluation',len(resulting_score_list), len(operator_count_list)) for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): individual_str = str(individual) if type(resulting_score) in [float, np.float64, np.float32]: self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) - fitness = self.eval_ind[individual_str] - fitnesses.append(fitness) + fitnesses.append(self.eval_ind[individual_str]) else: raise ValueError('Scoring function does not return a float') - - print('eval_done') - #return ret_individuals return fitnesses @@ -916,7 +811,6 @@ def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_f raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') except: resulting_score = -float('inf') - print(resulting_score) if not self._pbar.disable: self._pbar.update(1) return resulting_score diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index a95fc032..80838ea5 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -133,9 +133,6 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in population if not ind.fitness.valid] - """fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) - for ind, fit in zip(invalid_ind, fitnesses): - ind.fitness.values = fit""" fitnesses = toolbox.evaluate(invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit @@ -150,7 +147,6 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, # Begin the generational process for gen in range(1, ngen + 1): - print('Gen:', gen) # Select the next generation individuals offspring = toolbox.select(population, len(population)) @@ -159,9 +155,6 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in offspring if not ind.fitness.valid] - """fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) - for ind, fit in zip(invalid_ind, fitnesses): - ind.fitness.values = fit""" fitnesses = toolbox.evaluate(invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit diff --git a/tpot_test.py b/tpot_test.py index 0fe2a2c8..6f2ccd5a 100644 --- a/tpot_test.py +++ b/tpot_test.py @@ -13,8 +13,8 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=2, population_size=5, verbosity=0, random_state = 42) +tpot = TPOTClassifier(generations=2, population_size=5, verbosity=3, n_jobs = 3, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 2:',time.time()-time_start) +print('\nTime used with num_cpu = 3:',time.time()-time_start) From 9188ed23d723d64187b88c38e5d14eab483b1017 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 31 Dec 2016 13:40:24 -0500 Subject: [PATCH 015/154] clean codes --- tpot/base.py | 1 + tpot_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index 848189fe..35dcccaa 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -778,6 +778,7 @@ def _generate(self, pset, min_, max_, condition, type_=None): return expr + # make the function pickleable def _pareto_eq(self, ind1, ind2): """Determines whether two individuals are equal on the Pareto front diff --git a/tpot_test.py b/tpot_test.py index 6f2ccd5a..8ff842e1 100644 --- a/tpot_test.py +++ b/tpot_test.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=2, population_size=5, verbosity=3, n_jobs = 3, random_state = 42) +tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, n_jobs = 3, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From 5fb96ba503974605774f2d77d466217d27f8672f Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 31 Dec 2016 13:41:19 -0500 Subject: [PATCH 016/154] clean codes --- tpot_test.py => tpot_test_multi_process.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tpot_test.py => tpot_test_multi_process.py (100%) diff --git a/tpot_test.py b/tpot_test_multi_process.py similarity index 100% rename from tpot_test.py rename to tpot_test_multi_process.py From 0c83431f47a1500f400f53b8d8fff1e0bc8947b0 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 3 Jan 2017 22:06:30 -1000 Subject: [PATCH 017/154] timeout works --- tpot/base.py | 92 +++++++++++++++----------- tpot/decorators.py | 131 +++++++++++++++++++++++++------------ tpot_test_multi_process.py | 2 +- 3 files changed, 145 insertions(+), 80 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 35dcccaa..7f3d5fb7 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -25,7 +25,7 @@ import sys from functools import partial from datetime import datetime -from pathos.multiprocessing import Pool +from pathos.multiprocessing import ProcessPool #from joblib import Parallel, delayed import numpy as np @@ -168,6 +168,10 @@ def __init__(self, population_size=100, generations=100, self.max_time_mins = max_time_mins self.max_eval_time_mins = max_eval_time_mins + global max_e_time_mins + max_e_time_mins = max_eval_time_mins + + # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out if not (max_time_mins is None): @@ -567,7 +571,7 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = Returns ------- - fitnesses: float + fitnesses_ordered: float Returns a list of tuple value indicating the `individual`'s fitness according to its performance on the provided data @@ -581,18 +585,22 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = # return fitness scores fitnesses = [] - # 3 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing - eval_individuals = [] + orderlist = [] + # 4 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing + eval_individuals_str = [] sklearn_pipeline_list = [] operator_count_list = [] - for individual in individuals: + test_idx_list = [] + for indidx in range(len(individuals)): # Disallow certain combinations of operators because they will take too long or take up too much RAM # This is a fairly hacky way to prevent TPOT from getting stuck on bad pipelines and should be improved in a future release + individual = individuals[indidx] individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): print('Invalid pipeline -- skipping its evaluation') - fitness = (max(1, operator_count), resulting_score) + fitness = (5000., -float('inf')) ## need reorder !!! fitnesses.append(fitness) + orderlist.append(indidx) if not self._pbar.disable: self._pbar.update(1) @@ -600,48 +608,57 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = elif individual_str in self.eval_ind: # get fitness score from previous evaluation fitnesses.append(self.eval_ind[individual_str]) + orderlist.append(indidx) if self.verbosity == 3: self._pbar.write("Pipeline #{0} has been evaluated previously. " "Continuing to the next pipeline.".format(self._pbar.n + 1)) if not self._pbar.disable: self._pbar.update(1) - else: - # Transform the tree expression into an sklearn pipeline - sklearn_pipeline = self._toolbox.compile(expr=individual) - - # Fix random state when the operator allows and build sample weight dictionary - sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) - - # Count the number of pipeline operators as a measure of pipeline complexity - operator_count = 0 - # add time limit for evaluation of pipeline - for i in range(len(individual)): - node = individual[i] - if ((type(node) is deap.gp.Terminal) or - type(node) is deap.gp.Primitive and node.name == 'CombineDFs'): - continue - operator_count += 1 - - eval_individuals.append(individual) + try: + # Transform the tree expression into an sklearn pipeline + sklearn_pipeline = self._toolbox.compile(expr=individual) + + # Fix random state when the operator allows and build sample weight dictionary + sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) + + # Count the number of pipeline operators as a measure of pipeline complexity + operator_count = 0 + # add time limit for evaluation of pipeline + for i in range(len(individual)): + node = individual[i] + if ((type(node) is deap.gp.Terminal) or + type(node) is deap.gp.Primitive and node.name == 'CombineDFs'): + continue + operator_count += 1 + except: + fitness = (5000., -float('inf')) ## need reorder !!! + fitnesses.append(fitness) + orderlist.append(indidx) + if not self._pbar.disable: + self._pbar.update(1) + continue + eval_individuals_str.append(individual_str) operator_count_list.append(operator_count) sklearn_pipeline_list.append(sklearn_pipeline) - - partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, + test_idx_list.append(indidx) + partial_cross_val_score = partial(self._wrapped_cross_val_score, self, features=features, classes=classes, num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) # parallel computing in evaluation of pipeline - pool = Pool(processes=self.n_jobs) + pool = ProcessPool(processes=self.n_jobs) resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) - for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): - individual_str = str(individual) + for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) fitnesses.append(self.eval_ind[individual_str]) + orderlist.append(test_idx) else: raise ValueError('Scoring function does not return a float') - return fitnesses - + fitnesses_ordered = [None] * len(individuals) + for idx, fit in zip(orderlist, fitnesses): + fitnesses_ordered[idx] = fit + return fitnesses_ordered @_gp_new_generation def _combined_selection_operator(self, individuals, k): @@ -778,7 +795,7 @@ def _generate(self, pset, min_, max_, condition, type_=None): return expr - # make the function pickleable + # make the function pickleable def _pareto_eq(self, ind1, ind2): """Determines whether two individuals are equal on the Pareto front @@ -798,7 +815,7 @@ def _pareto_eq(self, ind1, ind2): """ return np.all(ind1.fitness.values == ind2.fitness.values) - + @_timeout def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_folds, scoring_function, sample_weight_dict): try: with warnings.catch_warnings(): @@ -806,10 +823,11 @@ def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_f cv_scores = cross_val_score(sklearn_pipeline, features, classes, cv=num_cv_folds, scoring=scoring_function, n_jobs=1, fit_params=sample_weight_dict) - try: - resulting_score = np.mean(cv_scores) - except TypeError: - raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') + resulting_score = np.mean(cv_scores) + except RuntimeError: + if self.verbosity > 1: + self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') + resulting_score = -float('inf') except: resulting_score = -float('inf') if not self._pbar.disable: diff --git a/tpot/decorators.py b/tpot/decorators.py index e70d85e6..33badd87 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -18,7 +18,7 @@ """ - +from threading import Thread, current_thread from functools import wraps import sys @@ -71,6 +71,38 @@ def wrapped_func(self, *args, **kwargs): return wrapped_func +def convert_mins_to_secs(time_minute): + """Convert time from minutes to seconds""" + second = int(time_minute * 60) + # time limit should be at least 1 second + return max(second, 1) + + +class InterruptableThread(Thread): + def __init__(self, args, kwargs): + Thread.__init__(self) + self.args = args + self.kwargs = kwargs + self.result = -float('inf') + self.daemon = True + def stop(self): + self._stop() + def run(self): + try: + # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) + # Note: Need attention if using parallel execution model of scikit-learn + current_thread().name = 'MainThread' + self.result = func(*self.args, **self.kwargs) + except Exception: + pass + +def timeout_signal_handler(signum, frame): + """ + signal handler for _timeout function + rasie TIMEOUT exception + """ + raise RuntimeError("Time Out!") + def _timeout(func): """Runs a function with time limit @@ -90,25 +122,27 @@ def _timeout(func): limitedTime: function Wrapped function that raises a timeout exception if the time limit is exceeded """ - def convert_mins_to_secs(time_minute): - """Convert time from minutes to seconds""" - second = int(time_minute * 60) - # time limit should be at least 1 second - return max(second, 1) - class TIMEOUT(RuntimeError): - """ - Inhertis from RuntimeError - """ - pass - - def timeout_signal_handler(signum, frame): - """ - signal handler for _timeout function - rasie TIMEOUT exception - """ - raise TIMEOUT("Time Out!") - if sys.platform.startswith('linux'): - from signal import SIGXCPU, signal, getsignal + if not sys.platform.startswith('win'): + import signal + @wraps(func) + def limitedTime(self, *args, **kw): + old_signal_hander = signal.signal(signal.SIGALRM, timeout_signal_handler) + max_time_seconds = convert_mins_to_secs(self.max_eval_time_mins) + signal.alarm(max_time_seconds) + try: + ret = func(*args, **kw) + except RuntimeError: + raise RuntimeError("Time Out!") + """print('timeout!!') + ret = -float('inf') + if self.verbosity > 1: + self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') """ # f() always returns, in this scheme + finally: + signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored + signal.alarm(0) # Alarm removed + return ret + #return limitedTime + """from signal import SIGXCPU, signal, getsignal from resource import getrlimit, setrlimit, RLIMIT_CPU, getrusage, RUSAGE_SELF # timeout uses the CPU time @wraps(func) @@ -136,29 +170,12 @@ def limitedTime(self,*args, **kw): sys.tracebacklimit=1000 # reset signal signal(SIGXCPU, old_signal_hander) - return ret + return ret""" + else: - from threading import Thread, current_thread - class InterruptableThread(Thread): - def __init__(self, args, kwargs): - Thread.__init__(self) - self.args = args - self.kwargs = kwargs - self.result = None - self.daemon = True - def stop(self): - self._stop() - def run(self): - try: - # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) - # Note: Need attention if using parallel execution model of scikit-learn - current_thread().name = 'MainThread' - self.result = func(*self.args, **self.kwargs) - except Exception: - pass @wraps(func) def limitedTime(self, *args, **kw): - sys.tracebacklimit = 0 + #sys.tracebacklimit = 0 max_time_seconds = convert_mins_to_secs(self.max_eval_time_mins) # start thread tmp_it = InterruptableThread(args, kw) @@ -168,8 +185,38 @@ def limitedTime(self, *args, **kw): if tmp_it.isAlive(): if self.verbosity > 1: self._pbar.write('Timeout during evaluation of pipeline #{0}. Skipping to the next pipeline.'.format(self._pbar.n + 1)) - sys.tracebacklimit=1000 + #sys.tracebacklimit=1000 return tmp_it.result tmp_it.stop() - # return func + # return func return limitedTime + #return decorate + + +"""class TimedOutExc(Exception): + + +import signal +def timeout(timeout): + + def decorate(f): + + def handler(signum, frame): + raise TimedOutExc() + + def new_f(*args, **kwargs): + + old_handler = signal.signal(signal.SIGALRM, handler) + signal.alarm(timeout) + + result = f(*args, **kwargs) # f() always returns, in this scheme + + signal.signal(signal.SIGALRM, old_handler) # Old signal handler is restored + signal.alarm(0) # Alarm removed + + return result + + #new_f.func_name = f.func_name + return new_f + + return decorate""" diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py index 8ff842e1..7e9ac260 100644 --- a/tpot_test_multi_process.py +++ b/tpot_test_multi_process.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, n_jobs = 3, random_state = 42) +tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 3, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From e91cfe91007bc7b9ba5216ceea07820dd98f9b69 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 3 Jan 2017 22:06:59 -1000 Subject: [PATCH 018/154] clean codes --- tpot/decorators.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/tpot/decorators.py b/tpot/decorators.py index 33badd87..5734da5f 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -190,33 +190,3 @@ def limitedTime(self, *args, **kw): tmp_it.stop() # return func return limitedTime - #return decorate - - -"""class TimedOutExc(Exception): - - -import signal -def timeout(timeout): - - def decorate(f): - - def handler(signum, frame): - raise TimedOutExc() - - def new_f(*args, **kwargs): - - old_handler = signal.signal(signal.SIGALRM, handler) - signal.alarm(timeout) - - result = f(*args, **kwargs) # f() always returns, in this scheme - - signal.signal(signal.SIGALRM, old_handler) # Old signal handler is restored - signal.alarm(0) # Alarm removed - - return result - - #new_f.func_name = f.func_name - return new_f - - return decorate""" From 48920853c72ad10bc1266f660139b40fd993fa95 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 3 Jan 2017 22:35:27 -1000 Subject: [PATCH 019/154] windows works and clean codes --- tpot/base.py | 10 +++-- tpot/decorators.py | 76 ++++++++++---------------------------- tpot_test_multi_process.py | 20 ---------- 3 files changed, 26 insertions(+), 80 deletions(-) delete mode 100644 tpot_test_multi_process.py diff --git a/tpot/base.py b/tpot/base.py index 7f3d5fb7..a26689ef 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -25,7 +25,7 @@ import sys from functools import partial from datetime import datetime -from pathos.multiprocessing import ProcessPool +from pathos.multiprocessing import Pool #from joblib import Parallel, delayed import numpy as np @@ -41,7 +41,6 @@ from sklearn.metrics.scorer import make_scorer from update_checker import update_check -from joblib import Parallel, delayed from ._version import __version__ from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code @@ -645,8 +644,11 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = partial_cross_val_score = partial(self._wrapped_cross_val_score, self, features=features, classes=classes, num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) # parallel computing in evaluation of pipeline - pool = ProcessPool(processes=self.n_jobs) - resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + if not sys.platform.startswith('win'): + pool = Pool(processes=self.n_jobs) + resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + else: + resulting_score_list = map(partial_cross_val_score, sklearn_pipeline_list) for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: diff --git a/tpot/decorators.py b/tpot/decorators.py index 5734da5f..7dbf6734 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -78,24 +78,6 @@ def convert_mins_to_secs(time_minute): return max(second, 1) -class InterruptableThread(Thread): - def __init__(self, args, kwargs): - Thread.__init__(self) - self.args = args - self.kwargs = kwargs - self.result = -float('inf') - self.daemon = True - def stop(self): - self._stop() - def run(self): - try: - # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) - # Note: Need attention if using parallel execution model of scikit-learn - current_thread().name = 'MainThread' - self.result = func(*self.args, **self.kwargs) - except Exception: - pass - def timeout_signal_handler(signum, frame): """ signal handler for _timeout function @@ -133,49 +115,31 @@ def limitedTime(self, *args, **kw): ret = func(*args, **kw) except RuntimeError: raise RuntimeError("Time Out!") - """print('timeout!!') - ret = -float('inf') - if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') """ # f() always returns, in this scheme finally: signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored signal.alarm(0) # Alarm removed return ret - #return limitedTime - """from signal import SIGXCPU, signal, getsignal - from resource import getrlimit, setrlimit, RLIMIT_CPU, getrusage, RUSAGE_SELF - # timeout uses the CPU time - @wraps(func) - def limitedTime(self,*args, **kw): - # don't show traceback - sys.tracebacklimit=0 - # save old signal - old_signal_hander = getsignal(SIGXCPU) - # change signal - signal(SIGXCPU, timeout_signal_handler) - max_time_second = convert_mins_to_secs(self.max_eval_time_mins) - r = getrusage(RUSAGE_SELF) - cpu_time = r.ru_utime + r.ru_stime - current = getrlimit(RLIMIT_CPU) - try: - setrlimit(RLIMIT_CPU, (cpu_time+max_time_second, current[1])) - ret = func(*args, **kw) - except RuntimeError: - if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of pipeline #{0}. Skipping to the next pipeline.'.format(self._pbar.n + 1)) - ret = None - finally: - # reset cpu time limit and trackback - setrlimit(RLIMIT_CPU, current) - sys.tracebacklimit=1000 - # reset signal - signal(SIGXCPU, old_signal_hander) - return ret""" - else: + class InterruptableThread(Thread): + def __init__(self, args, kwargs): + Thread.__init__(self) + self.args = args + self.kwargs = kwargs + self.result = -float('inf') + self.daemon = True + def stop(self): + self._stop() + def run(self): + try: + # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) + # Note: Need attention if using parallel execution model of scikit-learn + current_thread().name = 'MainThread' + self.result = func(*self.args, **self.kwargs) + except Exception: + pass @wraps(func) def limitedTime(self, *args, **kw): - #sys.tracebacklimit = 0 + sys.tracebacklimit = 0 max_time_seconds = convert_mins_to_secs(self.max_eval_time_mins) # start thread tmp_it = InterruptableThread(args, kw) @@ -184,8 +148,8 @@ def limitedTime(self, *args, **kw): tmp_it.join(max_time_seconds) if tmp_it.isAlive(): if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of pipeline #{0}. Skipping to the next pipeline.'.format(self._pbar.n + 1)) - #sys.tracebacklimit=1000 + self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') + sys.tracebacklimit=1000 return tmp_it.result tmp_it.stop() # return func diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py deleted file mode 100644 index 7e9ac260..00000000 --- a/tpot_test_multi_process.py +++ /dev/null @@ -1,20 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split -import time - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.25, test_size=0.75) - -#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) -#time_start = time.time() -#tpot.fit(X_train, y_train) -#print(tpot.score(X_test, y_test)) -#print('\nTime used with num_cpu = 1:',time.time()-time_start) - -tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 3, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 3:',time.time()-time_start) From bbc7202b786702f2ff8823e61d1523f2f8d95005 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 3 Jan 2017 22:45:12 -1000 Subject: [PATCH 020/154] clean codes --- tpot/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index a26689ef..f1925343 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -26,7 +26,6 @@ from functools import partial from datetime import datetime from pathos.multiprocessing import Pool -#from joblib import Parallel, delayed import numpy as np import deap From 6af022d40030114ab36ef7ec29d6eb223e6dc6d5 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 4 Jan 2017 07:03:36 -1000 Subject: [PATCH 021/154] make a pickable class --- tpot/base.py | 4 ++-- tpot/decorators.py | 11 ++++++++--- tpot_test_multi_process.py | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 tpot_test_multi_process.py diff --git a/tpot/base.py b/tpot/base.py index f1925343..8291a9b4 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -43,7 +43,7 @@ from ._version import __version__ from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code -from .decorators import _gp_new_generation, _timeout +from .decorators import _gp_new_generation, _timeout, TimedOutExc from . import operators from .operators import CombineDFs from .gp_types import Bool, Output_DF @@ -825,7 +825,7 @@ def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_f cv=num_cv_folds, scoring=scoring_function, n_jobs=1, fit_params=sample_weight_dict) resulting_score = np.mean(cv_scores) - except RuntimeError: + except TimedOutExc: if self.verbosity > 1: self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') resulting_score = -float('inf') diff --git a/tpot/decorators.py b/tpot/decorators.py index 7dbf6734..7a9d2e70 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -78,12 +78,17 @@ def convert_mins_to_secs(time_minute): return max(second, 1) +class TimedOutExc(RuntimeError): + """ + Raised when a timeout happens + """ + def timeout_signal_handler(signum, frame): """ signal handler for _timeout function rasie TIMEOUT exception """ - raise RuntimeError("Time Out!") + raise TimedOutExc("Time Out!") def _timeout(func): """Runs a function with time limit @@ -113,8 +118,8 @@ def limitedTime(self, *args, **kw): signal.alarm(max_time_seconds) try: ret = func(*args, **kw) - except RuntimeError: - raise RuntimeError("Time Out!") + except: + raise TimedOutExc("Time Out!") finally: signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored signal.alarm(0) # Alarm removed diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py new file mode 100644 index 00000000..7e9ac260 --- /dev/null +++ b/tpot_test_multi_process.py @@ -0,0 +1,20 @@ +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split +import time + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.25, test_size=0.75) + +#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) +#time_start = time.time() +#tpot.fit(X_train, y_train) +#print(tpot.score(X_test, y_test)) +#print('\nTime used with num_cpu = 1:',time.time()-time_start) + +tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 3, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used with num_cpu = 3:',time.time()-time_start) From 736973a15669b2e96550e142d472ed7b05fa8f3a Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Tue, 17 Jan 2017 09:52:21 -0500 Subject: [PATCH 022/154] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ceccb6d..6d07b45e 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,6 @@ Alternatively, you can cite the repository directly with the following DOI: ## Support for TPOT -TPOT was developed in the [Computational Genetics Lab](http://epistasis.org) with funding from the [NIH](http://www.nih.gov). We're incredibly grateful for their support during the development of this project. +TPOT was developed in the [Computational Genetics Lab](http://epistasis.org) with funding from the [NIH](http://www.nih.gov) under grant R01 AI117694. We're incredibly grateful for their support during the development of this project. The TPOT logo was designed by Todd Newmuis, who generously donated his time to the project. From f06fefe10c261bf3bac8804af5a5ca11b55401ab Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Tue, 17 Jan 2017 09:53:12 -0500 Subject: [PATCH 023/154] Update support.md --- docs_sources/support.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_sources/support.md b/docs_sources/support.md index f307042c..ba1ca553 100644 --- a/docs_sources/support.md +++ b/docs_sources/support.md @@ -1,3 +1,3 @@ -TPOT was developed in the [Computational Genetics Lab](http://epistasis.org) with funding from the [NIH](http://www.nih.gov). We're incredibly grateful for their support during the development of this project. +TPOT was developed in the [Computational Genetics Lab](http://epistasis.org) with funding from the [NIH](http://www.nih.gov) under grant R01 AI117694. We're incredibly grateful for their support during the development of this project. The TPOT logo was designed by Todd Newmuis, who generously donated his time to the project. From aa6f673a1cf223f29fb8bef66a4c7af7600f2817 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Tue, 17 Jan 2017 10:02:53 -0500 Subject: [PATCH 024/154] Update docs --- docs/index.html | 4 ++-- docs/mkdocs/search_index.json | 8 ++++---- docs/sitemap.xml | 22 +++++++++++----------- docs/support/index.html | 2 +- docs/using/index.html | 19 ++++++++++++++++++- 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/docs/index.html b/docs/index.html index 8367658a..cc25acda 100644 --- a/docs/index.html +++ b/docs/index.html @@ -244,6 +244,6 @@ diff --git a/docs/mkdocs/search_index.json b/docs/mkdocs/search_index.json index 79801034..fc742ba0 100644 --- a/docs/mkdocs/search_index.json +++ b/docs/mkdocs/search_index.json @@ -12,17 +12,17 @@ }, { "location": "/using/", - "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >2\n\n\nThe number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. This setting will override the GENERATIONS parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngeneration\n\n\nAny positive integer\n\n\nThe number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nThe number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nThe mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nThe crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We don't recommend that you tweak this parameter unless you know what you're doing.\n\n\n\n\n\n\nnum_cv_folds\n\n\n[2, 10]\n\n\nThe number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. This setting will override the generations parameter.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nThe random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit().\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)", + "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nThe number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nThe number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. This setting will override the GENERATIONS parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngeneration\n\n\nAny positive integer\n\n\nThe number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nThe number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nThe mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nThe crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We don't recommend that you tweak this parameter unless you know what you're doing.\n\n\n\n\n\n\nnum_cv_folds\n\n\n[2, 10]\n\n\nThe number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nThe number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. This setting will override the generations parameter.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nThe random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit().\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether TPOT will reuse models from previous calls\nto fit() for faster operation\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)", "title": "Using TPOT" }, { "location": "/using/#tpot-on-the-command-line", - "text": "To use TPOT via the command line, enter the following command with a path to the data file: tpot /path_to/data_file.csv TPOT offers several arguments that can be provided at the command line: Argument Parameter Valid values Effect -is INPUT_SEPARATOR Any string Character used to separate columns in the input file. -target TARGET_NAME Any string Name of the target column in the input file. -mode TPOT_MODE ['classification', 'regression'] Whether TPOT is being used for a classification or regression problem. -o OUTPUT_FILE String path to a file File to export the code for the final optimized pipeline. -g GENERATIONS Any positive integer Number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. -p POPULATION_SIZE Any positive integer Number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. -mr MUTATION_RATE [0.0, 1.0] GP mutation rate. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. -xr CROSSOVER_RATE [0.0, 1.0] GP crossover rate in the range [0.0, 1.0]. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms. -cv NUM_CV_FOLDS Any integer >2 The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. -scoring SCORING_FN 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. -maxtime MAX_TIME_MINS Any positive integer How many minutes TPOT has to optimize the pipeline. This setting will override the GENERATIONS parameter and allow TPOT to run until it runs out of time. -maxeval MAX_EVAL_MINS Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. -s RANDOM_STATE Any positive integer Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future. -v VERBOSITY {0, 1, 2, 3} How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. --no-update-check N/A Flag indicating whether the TPOT version checker should be disabled. --version N/A Show TPOT's version number and exit. --help N/A Show TPOT's help documentation and exit. An example command-line call to TPOT may look like: tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2", + "text": "To use TPOT via the command line, enter the following command with a path to the data file: tpot /path_to/data_file.csv TPOT offers several arguments that can be provided at the command line: Argument Parameter Valid values Effect -is INPUT_SEPARATOR Any string Character used to separate columns in the input file. -target TARGET_NAME Any string Name of the target column in the input file. -mode TPOT_MODE ['classification', 'regression'] Whether TPOT is being used for a classification or regression problem. -o OUTPUT_FILE String path to a file File to export the code for the final optimized pipeline. -g GENERATIONS Any positive integer Number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. -p POPULATION_SIZE Any positive integer Number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. -mr MUTATION_RATE [0.0, 1.0] GP mutation rate. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. -xr CROSSOVER_RATE [0.0, 1.0] GP crossover rate in the range [0.0, 1.0]. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms. -cv NUM_CV_FOLDS Any integer >1 The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. -njobs NUM_JOBS Any positive integer or -1 The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation. -scoring SCORING_FN 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. -maxtime MAX_TIME_MINS Any positive integer How many minutes TPOT has to optimize the pipeline. This setting will override the GENERATIONS parameter and allow TPOT to run until it runs out of time. -maxeval MAX_EVAL_MINS Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. -s RANDOM_STATE Any positive integer Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future. -v VERBOSITY {0, 1, 2, 3} How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. --no-update-check N/A Flag indicating whether the TPOT version checker should be disabled. --version N/A Show TPOT's version number and exit. --help N/A Show TPOT's help documentation and exit. An example command-line call to TPOT may look like: tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2", "title": "TPOT on the command line" }, { "location": "/using/#tpot-with-code", - "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn. TPOT can be imported just like any regular Python module. To import TPOT, type: from tpot import TPOTClassifier then create an instance of TPOT as follows: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier() It's also possible to use TPOT for regression problems with the TPOTRegressor class. Other than the class name, a TPOTRegressor is used the same way as a TPOTClassifier . Note that you can pass several parameters to the TPOT instantiation call: Parameter Valid values Effect generation Any positive integer The number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. population_size Any positive integer The number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. mutation_rate [0.0, 1.0] The mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing. crossover_rate [0.0, 1.0] The crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We don't recommend that you tweak this parameter unless you know what you're doing. num_cv_folds [2, 10] The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. max_time_mins Any positive integer How many minutes TPOT has to optimize the pipeline. This setting will override the generations parameter. max_eval_time_mins Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. random_state Any positive integer The random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit(). disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. Some example code with custom TPOT parameters might look like: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2) Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes) The fit() function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score. You can then proceed to evaluate the final pipeline on the testing set with the score() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes)) Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py') Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline. Check our examples to see TPOT applied to some specific data sets.", + "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn. TPOT can be imported just like any regular Python module. To import TPOT, type: from tpot import TPOTClassifier then create an instance of TPOT as follows: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier() It's also possible to use TPOT for regression problems with the TPOTRegressor class. Other than the class name, a TPOTRegressor is used the same way as a TPOTClassifier . Note that you can pass several parameters to the TPOT instantiation call: Parameter Valid values Effect generation Any positive integer The number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. population_size Any positive integer The number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. mutation_rate [0.0, 1.0] The mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing. crossover_rate [0.0, 1.0] The crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We don't recommend that you tweak this parameter unless you know what you're doing. num_cv_folds [2, 10] The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. n_jobs Any positive integer or -1 The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation. scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. max_time_mins Any positive integer How many minutes TPOT has to optimize the pipeline. This setting will override the generations parameter. max_eval_time_mins Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. random_state Any positive integer The random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit(). disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. warm_start [True, False] Flag indicating whether TPOT will reuse models from previous calls\nto fit() for faster operation Some example code with custom TPOT parameters might look like: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2) Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes) The fit() function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score. You can then proceed to evaluate the final pipeline on the testing set with the score() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes)) Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py') Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline. Check our examples to see TPOT applied to some specific data sets.", "title": "TPOT with code" }, { @@ -122,7 +122,7 @@ }, { "location": "/support/", - "text": "TPOT was developed in the \nComputational Genetics Lab\n with funding from the \nNIH\n. We're incredibly grateful for their support during the development of this project.\n\n\nThe TPOT logo was designed by Todd Newmuis, who generously donated his time to the project.", + "text": "TPOT was developed in the \nComputational Genetics Lab\n with funding from the \nNIH\n under grant R01 AI117694. We're incredibly grateful for their support during the development of this project.\n\n\nThe TPOT logo was designed by Todd Newmuis, who generously donated his time to the project.", "title": "Support" } ] diff --git a/docs/sitemap.xml b/docs/sitemap.xml index a7fd3f25..e1135dd1 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -4,7 +4,7 @@ http://rhiever.github.io/tpot/ - 2016-12-20 + 2017-01-17 daily @@ -12,7 +12,7 @@ http://rhiever.github.io/tpot/installing/ - 2016-12-20 + 2017-01-17 daily @@ -20,7 +20,7 @@ http://rhiever.github.io/tpot/using/ - 2016-12-20 + 2017-01-17 daily @@ -29,25 +29,25 @@ http://rhiever.github.io/tpot/examples/MNIST_Example/ - 2016-12-20 + 2017-01-17 daily http://rhiever.github.io/tpot/examples/IRIS_Example/ - 2016-12-20 + 2017-01-17 daily http://rhiever.github.io/tpot/examples/Boston_Example/ - 2016-12-20 + 2017-01-17 daily http://rhiever.github.io/tpot/examples/Titanic_Kaggle_Example/ - 2016-12-20 + 2017-01-17 daily @@ -56,7 +56,7 @@ http://rhiever.github.io/tpot/contributing/ - 2016-12-20 + 2017-01-17 daily @@ -64,7 +64,7 @@ http://rhiever.github.io/tpot/releases/ - 2016-12-20 + 2017-01-17 daily @@ -72,7 +72,7 @@ http://rhiever.github.io/tpot/citing/ - 2016-12-20 + 2017-01-17 daily @@ -80,7 +80,7 @@ http://rhiever.github.io/tpot/support/ - 2016-12-20 + 2017-01-17 daily diff --git a/docs/support/index.html b/docs/support/index.html index 3909622d..5d97923e 100644 --- a/docs/support/index.html +++ b/docs/support/index.html @@ -170,7 +170,7 @@
-

TPOT was developed in the Computational Genetics Lab with funding from the NIH. We're incredibly grateful for their support during the development of this project.

+

TPOT was developed in the Computational Genetics Lab with funding from the NIH under grant R01 AI117694. We're incredibly grateful for their support during the development of this project.

The TPOT logo was designed by Todd Newmuis, who generously donated his time to the project.

diff --git a/docs/using/index.html b/docs/using/index.html index d99f035b..774a5692 100644 --- a/docs/using/index.html +++ b/docs/using/index.html @@ -242,10 +242,16 @@

TPOT on the command line

-cv NUM_CV_FOLDS -Any integer >2 +Any integer >1 The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. +-njobs +NUM_JOBS +Any positive integer or -1 +The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation. + + -scoring SCORING_FN 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' @@ -342,6 +348,11 @@

TPOT with code

The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. +n_jobs +Any positive integer or -1 +The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation. + + scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. @@ -371,6 +382,12 @@

TPOT with code

[True, False] Flag indicating whether the TPOT version checker should be disabled. + +warm_start +[True, False] +Flag indicating whether TPOT will reuse models from previous calls +to fit() for faster operation +

Some example code with custom TPOT parameters might look like:

From 4a6211fe35163aeff15217f53bc0d91a4d5ae0a1 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 19 Jan 2017 13:13:59 -0500 Subject: [PATCH 025/154] fix windows support --- tpot/decorators.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tpot/decorators.py b/tpot/decorators.py index 7a9d2e70..4d4ab6d7 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -152,8 +152,7 @@ def limitedTime(self, *args, **kw): #timer = Timer(max_time_seconds, interrupt_main) tmp_it.join(max_time_seconds) if tmp_it.isAlive(): - if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') + raise TimedOutExc("Time Out!") sys.tracebacklimit=1000 return tmp_it.result tmp_it.stop() From 1da881d600299d0d7c813740801914cf5817fbfe Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 27 Jan 2017 13:01:59 -0500 Subject: [PATCH 026/154] code clean and add install pathos in Travis --- ci/.travis_install.sh | 1 + tpot_test_multi_process.py | 20 -------------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 tpot_test_multi_process.py diff --git a/ci/.travis_install.sh b/ci/.travis_install.sh index 267d8e60..23a2b0ac 100755 --- a/ci/.travis_install.sh +++ b/ci/.travis_install.sh @@ -53,6 +53,7 @@ fi pip install update_checker pip install tqdm +pip install pathos if [[ "$COVERAGE" == "true" ]]; then pip install coverage coveralls diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py deleted file mode 100644 index 7e9ac260..00000000 --- a/tpot_test_multi_process.py +++ /dev/null @@ -1,20 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split -import time - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.25, test_size=0.75) - -#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) -#time_start = time.time() -#tpot.fit(X_train, y_train) -#print(tpot.score(X_test, y_test)) -#print('\nTime used with num_cpu = 1:',time.time()-time_start) - -tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 3, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 3:',time.time()-time_start) From 8cbff159e7bb7d7f98c9db726ded05ed55a74565 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 9 Feb 2017 12:43:28 -0500 Subject: [PATCH 027/154] remove pbar decorators --- tests.py | 20 +------------------ tpot/base.py | 12 +++++------- tpot/decorators.py | 49 ---------------------------------------------- tpot/gp_deap.py | 28 +++++++++++++++++++++++++- 4 files changed, 33 insertions(+), 76 deletions(-) diff --git a/tests.py b/tests.py index 04580fb7..997bbed7 100644 --- a/tests.py +++ b/tests.py @@ -8,7 +8,6 @@ from tpot.base import TPOTBase from tpot.driver import positive_integer, float_range from tpot.export_utils import export_pipeline, generate_import_code, _indent, generate_pipeline_code -from tpot.decorators import _gp_new_generation from tpot.gp_types import Output_DF from tpot.operators import Operator @@ -232,7 +231,7 @@ def test_predict_proba(): assert result.shape == (testing_features.shape[0], num_labels) - + def test_predict_proba2(): """Assert that the TPOT predict_proba function returns a numpy matrix filled with probabilities (float)""" @@ -282,23 +281,6 @@ def test_fit(): assert not (tpot_obj._start_datetime is None) -def test_gp_new_generation(): - """Assert that the gp_generation count gets incremented when _gp_new_generation is called""" - tpot_obj = TPOTClassifier() - tpot_obj._pbar = tqdm(total=1, disable=True) - - assert tpot_obj._gp_generation == 0 - - # Since _gp_new_generation is a decorator, and we dont want to run a full - # fit(), decorate a dummy function and then call the dummy function. - @_gp_new_generation - def dummy_function(self, foo): - pass - - dummy_function(tpot_obj, None) - - assert tpot_obj._gp_generation == 1 - def check_export(op): """Assert that a TPOT operator exports as expected""" diff --git a/tpot/base.py b/tpot/base.py index 8b7754a9..c043f3f9 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -42,7 +42,7 @@ from ._version import __version__ from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code -from .decorators import _gp_new_generation, _timeout +from .decorators import _timeout from . import operators from .operators import CombineDFs from .gp_types import Bool, Output_DF @@ -356,10 +356,9 @@ def pareto_eq(ind1, ind2): disable=not (self.verbosity >= 2), desc='Optimization Progress') try: - pop, _ = eaSimple( - population=pop, toolbox=self._toolbox, cxpb=self.crossover_rate, - mutpb=self.mutation_rate, ngen=self.generations, - halloffame=self._pareto_front, verbose=False) + pop, _ = eaSimple(self, population=pop, toolbox=self._toolbox, + cxpb=self.crossover_rate, mutpb=self.mutation_rate, + ngen=self.generations,halloffame=self._pareto_front, verbose=False) # store population for the next call if self.warm_start: @@ -388,7 +387,7 @@ def pareto_eq(ind1, ind2): top_score = pipeline_scores.wvalues[1] # It won't raise error for a small test like in a unit test becasue a few pipeline sometimes - # may fail due to the training data does not fit the operator's requirement. + # may fail due to the training data does not fit the operator's requirement. if self.generations*self.population_size > 5 and not self._optimized_pipeline: raise ValueError('There was an error in the TPOT optimization ' 'process. This could be because the data was ' @@ -671,7 +670,6 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No else: raise ValueError('Scoring function does not return a float') - @_gp_new_generation def _combined_selection_operator(self, individuals, k): """Perform NSGA2 selection on the population according to their Pareto fitness diff --git a/tpot/decorators.py b/tpot/decorators.py index e70d85e6..75b659e5 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -22,55 +22,6 @@ from functools import wraps import sys - -def _gp_new_generation(func): - """Decorator that wraps functions that indicate the beginning of a new GP generation. - - Parameters - ---------- - func: function - The function being decorated - - Returns - ------- - wrapped_func: function - A wrapper function around the func parameter - """ - @wraps(func) - def wrapped_func(self, *args, **kwargs): - """Increment _gp_generation and bump pipeline count if necessary""" - ret = func(self, *args, **kwargs) - self._gp_generation += 1 - if not self._pbar.disable: - # Print only the best individual fitness - if self.verbosity == 2: - high_score = abs(max([self._pareto_front.keys[x].wvalues[1] for x in range(len(self._pareto_front.keys))])) - self._pbar.write('Generation {0} - Current best internal CV score: {1}'.format(self._gp_generation, high_score)) - - # Print the entire Pareto front - elif self.verbosity == 3: - self._pbar.write('Generation {} - Current Pareto front scores:'.format(self._gp_generation)) - for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): - self._pbar.write('{}\t{}\t{}'.format(int(abs(pipeline_scores.wvalues[0])), - abs(pipeline_scores.wvalues[1]), - pipeline)) - self._pbar.write('') - - # Sometimes the actual evaluated pipeline count does not match the - # supposed count because DEAP can cache pipelines. Here any missed - # evaluations are added back to the progress bar. - if self._pbar.n < self._gp_generation * self.population_size: - missing_pipelines = (self._gp_generation * self.population_size) - self._pbar.n - self._pbar.update(missing_pipelines) - - if not (self.max_time_mins is None) and self._pbar.n >= self._pbar.total: - self._pbar.total += self.population_size - - return ret # Pass back return value of func - - return wrapped_func - - def _timeout(func): """Runs a function with time limit diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index c517ea88..6cc75ac4 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -75,7 +75,7 @@ def varAnd(population, toolbox, cxpb, mutpb): return offspring -def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, +def eaSimple(self, population, toolbox, cxpb, mutpb, ngen, stats=None, halloffame=None, verbose=__debug__): """This algorithm reproduce the simplest evolutionary algorithm as presented in chapter 7 of [Back2000]_. @@ -148,6 +148,32 @@ def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, for gen in range(1, ngen + 1): # Select the next generation individuals offspring = toolbox.select(population, len(population)) + # pbar process + self._gp_generation += 1 + if not self._pbar.disable: + # Print only the best individual fitness + if self.verbosity == 2: + high_score = abs(max([self._pareto_front.keys[x].wvalues[1] for x in range(len(self._pareto_front.keys))])) + self._pbar.write('Generation {0} - Current best internal CV score: {1}'.format(self._gp_generation, high_score)) + + # Print the entire Pareto front + elif self.verbosity == 3: + self._pbar.write('Generation {} - Current Pareto front scores:'.format(self._gp_generation)) + for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): + self._pbar.write('{}\t{}\t{}'.format(int(abs(pipeline_scores.wvalues[0])), + abs(pipeline_scores.wvalues[1]), + pipeline)) + self._pbar.write('') + + # Sometimes the actual evaluated pipeline count does not match the + # supposed count because DEAP can cache pipelines. Here any missed + # evaluations are added back to the progress bar. + if self._pbar.n < self._gp_generation * self.population_size: + missing_pipelines = (self._gp_generation * self.population_size) - self._pbar.n + self._pbar.update(missing_pipelines) + + if not (self.max_time_mins is None) and self._pbar.n >= self._pbar.total: + self._pbar.total += self.population_size # Vary the pool of individuals offspring = varAnd(offspring, toolbox, cxpb, mutpb) From 2b88c847a301cc0c1bc1a0d3765a2b5360006d3a Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 9 Feb 2017 12:58:52 -0500 Subject: [PATCH 028/154] fix unit tests --- tpot/base.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index c043f3f9..63a758ca 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -388,36 +388,36 @@ def pareto_eq(ind1, ind2): # It won't raise error for a small test like in a unit test becasue a few pipeline sometimes # may fail due to the training data does not fit the operator's requirement. - if self.generations*self.population_size > 5 and not self._optimized_pipeline: - raise ValueError('There was an error in the TPOT optimization ' + if not self._optimized_pipeline: + print('There was an error in the TPOT optimization ' 'process. This could be because the data was ' 'not formatted properly, or because data for ' 'a regression problem was provided to the ' 'TPOTClassifier object. Please make sure you ' 'passed the data to TPOT correctly.') + else: + self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) - self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self._fitted_pipeline.fit(features, classes) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self._fitted_pipeline.fit(features, classes) - if self.verbosity in [1, 2] and self._optimized_pipeline: - # Add an extra line of spacing if the progress bar was used - if self.verbosity >= 2: - print('') - print('Best pipeline: {}'.format(self._optimized_pipeline)) + if self.verbosity in [1, 2]: + # Add an extra line of spacing if the progress bar was used + if self.verbosity >= 2: + print('') + print('Best pipeline: {}'.format(self._optimized_pipeline)) - # Store and fit the entire Pareto front if sciencing - elif self.verbosity >= 3 and self._pareto_front: - self._pareto_front_fitted_pipelines = {} + # Store and fit the entire Pareto front if sciencing + elif self.verbosity >= 3 and self._pareto_front: + self._pareto_front_fitted_pipelines = {} - for pipeline in self._pareto_front.items: - self._pareto_front_fitted_pipelines[str(pipeline)] = self._toolbox.compile(expr=pipeline) + for pipeline in self._pareto_front.items: + self._pareto_front_fitted_pipelines[str(pipeline)] = self._toolbox.compile(expr=pipeline) - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self._pareto_front_fitted_pipelines[str(pipeline)].fit(features, classes) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self._pareto_front_fitted_pipelines[str(pipeline)].fit(features, classes) def predict(self, features): """Uses the optimized pipeline to predict the classes for a feature set From 5298d9a75bda8bf9d79a15895448d163d19da721 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 9 Feb 2017 13:07:01 -0500 Subject: [PATCH 029/154] fix unit tests 2 --- tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 997bbed7..d8b94a40 100644 --- a/tests.py +++ b/tests.py @@ -273,7 +273,7 @@ def test_warm_start(): def test_fit(): """Assert that the TPOT fit function provides an optimized pipeline""" - tpot_obj = TPOTClassifier(random_state=42, population_size=1, generations=1, verbosity=0) + tpot_obj = TPOTClassifier(random_state=42, population_size=5, generations=1, verbosity=0) tpot_obj.fit(training_features, training_classes) assert isinstance(tpot_obj._optimized_pipeline, creator.Individual) From 6822628f24b4c750d9b4254c0a876167ab2d542f Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 10 Feb 2017 17:28:23 -0500 Subject: [PATCH 030/154] new GP algorithms works --- tpot/base.py | 32 ++++-- tpot/driver.py | 8 +- tpot/gp_deap.py | 214 ++++++++++++++++++--------------------- tpot_test_rework_pbar.py | 26 +++++ 4 files changed, 153 insertions(+), 127 deletions(-) create mode 100644 tpot_test_rework_pbar.py diff --git a/tpot/base.py b/tpot/base.py index 63a758ca..b339bd4d 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -47,7 +47,7 @@ from .operators import CombineDFs from .gp_types import Bool, Output_DF from .metrics import SCORERS -from .gp_deap import eaSimple +from .gp_deap import eaMuPlusLambda # hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS @@ -70,9 +70,9 @@ def handler(dwCtrlType, hook_sigint=_thread.interrupt_main): class TPOTBase(BaseEstimator): """TPOT automatically creates and optimizes machine learning pipelines using genetic programming""" - def __init__(self, population_size=100, generations=100, + def __init__(self, population_size=100, lamda=200, generations=100, mutation_rate=0.9, crossover_rate=0.05, - scoring=None, num_cv_folds=5, n_jobs=1, + scoring=None, cv=5, n_jobs=1, max_time_mins=None, max_eval_time_mins=5, random_state=None, verbosity=0, disable_update_check=False, warm_start=False): @@ -84,6 +84,8 @@ def __init__(self, population_size=100, generations=100, The number of pipelines in the genetic algorithm population. Must be > 0.The more pipelines in the population, the slower TPOT will run, but it's also more likely to find better pipelines. + lamda: int (default: twice of population_size) + The number of children to produce at each generation. generations: int (default: 100) The number of generations to run pipeline optimization for. Must be > 0. The more generations you give TPOT to run, the longer it @@ -113,7 +115,7 @@ def __init__(self, population_size=100, generations=100, 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'] - num_cv_folds: int (default: 5) + cv: int (default: 5) The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process n_jobs: int (default: 1) @@ -162,6 +164,8 @@ def __init__(self, population_size=100, generations=100, self.generations = generations self.max_time_mins = max_time_mins self.max_eval_time_mins = max_eval_time_mins + # set lamda equal to twice of population_size by default + self.lamda = population_size*2 # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out @@ -170,6 +174,12 @@ def __init__(self, population_size=100, generations=100, self.mutation_rate = mutation_rate self.crossover_rate = crossover_rate + + # check if mutation_rate + crossover_rate > 1 + if self.mutation_rate + self.crossover_rate > 1: + raise TypeError("The sum of the crossover and mutation probabilities must be smaller " + "or equal to 1.0.") + self.verbosity = verbosity self.operators_context = { 'make_pipeline': make_pipeline, @@ -179,7 +189,7 @@ def __init__(self, population_size=100, generations=100, } self._pbar = None - self._gp_generation = 0 + #self._gp_generation = 0 # a dictionary of individual which has already evaluated in previous generation. self.eval_ind = {} @@ -200,7 +210,7 @@ def __init__(self, population_size=100, generations=100, else: self.scoring_function = scoring - self.num_cv_folds = num_cv_folds + self.cv = cv # If the OS is windows, reset cpu number to 1 since the OS did not have multiprocessing module if sys.platform.startswith('win') and n_jobs > 1: print('Warning: Parallelizing cross validation is not supported in Windows OS.', @@ -356,9 +366,11 @@ def pareto_eq(ind1, ind2): disable=not (self.verbosity >= 2), desc='Optimization Progress') try: - pop, _ = eaSimple(self, population=pop, toolbox=self._toolbox, + pop, _ = eaMuPlusLambda(population=pop, toolbox=self._toolbox, + mu = self.population_size, lambda_=self.lamda, cxpb=self.crossover_rate, mutpb=self.mutation_rate, - ngen=self.generations,halloffame=self._pareto_front, verbose=False) + ngen=self.generations,pbar = self._pbar, halloffame=self._pareto_front, + verbose=self.verbosity, max_time_mins = self.max_time_mins) # store population for the next call if self.warm_start: @@ -376,7 +388,7 @@ def pareto_eq(ind1, ind2): self._pbar.close() # Reset gp_generation counter to restore initial state - self._gp_generation = 0 + #self._gp_generation = 0 # Store the pipeline with the highest internal testing score if self._pareto_front: @@ -647,7 +659,7 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No with warnings.catch_warnings(): warnings.simplefilter('ignore') cv_scores = cross_val_score(self, sklearn_pipeline, features, classes, - cv=self.num_cv_folds, scoring=self.scoring_function, + cv=self.cv, scoring=self.scoring_function, n_jobs=self.n_jobs, fit_params=sample_weight_dict) try: resulting_score = np.mean(cv_scores) diff --git a/tpot/driver.py b/tpot/driver.py index e981a687..a0363811 100644 --- a/tpot/driver.py +++ b/tpot/driver.py @@ -100,6 +100,10 @@ def main(): 'you give it more generations (and therefore time) to optimize over. ' 'TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.') + parser.add_argument('-lamda', action='store', dest='LAMBDA', default=200, + type=positive_integer, help='The number of children to produce ' + 'at each generation.') + parser.add_argument('-p', action='store', dest='POPULATION_SIZE', default=100, type=positive_integer, help='Number of individuals in the GP population.\n' 'Generally, TPOT will work better when you give it more individuals ' @@ -197,8 +201,8 @@ def main(): tpot_type = TPOTRegressor tpot = tpot_type(generations=args.GENERATIONS, population_size=args.POPULATION_SIZE, - mutation_rate=args.MUTATION_RATE, crossover_rate=args.CROSSOVER_RATE, - num_cv_folds=args.NUM_CV_FOLDS, n_jobs=args.NUM_JOBS, + lamda=args.LAMBDA, mutation_rate=args.MUTATION_RATE, crossover_rate=args.CROSSOVER_RATE, + cv=args.NUM_CV_FOLDS, n_jobs=args.NUM_JOBS, scoring=args.SCORING_FN, max_time_mins=args.MAX_TIME_MINS, max_eval_time_mins=args.MAX_EVAL_MINS, random_state=args.RANDOM_STATE, verbosity=args.VERBOSITY, diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 6cc75ac4..ebb10044 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -18,73 +18,75 @@ """ import random - from deap import tools -def varAnd(population, toolbox, cxpb, mutpb): +def varOr(population, toolbox, lambda_, cxpb, mutpb): """Part of an evolutionary algorithm applying only the variation part - (crossover **and** mutation). The modified individuals have their - fitness invalidated. The individuals are cloned so returned population is - independent of the input population. + (crossover, mutation **or** reproduction). The modified individuals have + their fitness invalidated. The individuals are cloned so returned + population is independent of the input population. :param population: A list of individuals to vary. :param toolbox: A :class:`~deap.base.Toolbox` that contains the evolution operators. + :param lambda\_: The number of children to produce :param cxpb: The probability of mating two individuals. :param mutpb: The probability of mutating an individual. - :returns: A list of varied individuals that are independent of their - parents. - The variation goes as follow. First, the parental population - :math:`P_\mathrm{p}` is duplicated using the :meth:`toolbox.clone` method - and the result is put into the offspring population :math:`P_\mathrm{o}`. - A first loop over :math:`P_\mathrm{o}` is executed to mate pairs of consecutive - individuals. According to the crossover probability *cxpb*, the - individuals :math:`\mathbf{x}_i` and :math:`\mathbf{x}_{i+1}` are mated - using the :meth:`toolbox.mate` method. The resulting children - :math:`\mathbf{y}_i` and :math:`\mathbf{y}_{i+1}` replace their respective - parents in :math:`P_\mathrm{o}`. A second loop over the resulting - :math:`P_\mathrm{o}` is executed to mutate every individual with a - probability *mutpb*. When an individual is mutated it replaces its not - mutated version in :math:`P_\mathrm{o}`. The resulting - :math:`P_\mathrm{o}` is returned. - This variation is named *And* beceause of its propention to apply both - crossover and mutation on the individuals. Note that both operators are - not applied systematicaly, the resulting individuals can be generated from - crossover only, mutation only, crossover and mutation, and reproduction - according to the given probabilities. Both probabilities should be in - :math:`[0, 1]`. + :returns: The final population + :returns: A class:`~deap.tools.Logbook` with the statistics of the + evolution + The variation goes as follow. On each of the *lambda_* iteration, it + selects one of the three operations; crossover, mutation or reproduction. + In the case of a crossover, two individuals are selected at random from + the parental population :math:`P_\mathrm{p}`, those individuals are cloned + using the :meth:`toolbox.clone` method and then mated using the + :meth:`toolbox.mate` method. Only the first child is appended to the + offspring population :math:`P_\mathrm{o}`, the second child is discarded. + In the case of a mutation, one individual is selected at random from + :math:`P_\mathrm{p}`, it is cloned and then mutated using using the + :meth:`toolbox.mutate` method. The resulting mutant is appended to + :math:`P_\mathrm{o}`. In the case of a reproduction, one individual is + selected at random from :math:`P_\mathrm{p}`, cloned and appended to + :math:`P_\mathrm{o}`. + This variation is named *Or* beceause an offspring will never result from + both operations crossover and mutation. The sum of both probabilities + shall be in :math:`[0, 1]`, the reproduction probability is + 1 - *cxpb* - *mutpb*. """ - offspring = [toolbox.clone(ind) for ind in population] - - # Apply crossover and mutation on the offspring - for i in range(1, len(offspring), 2): - if random.random() < cxpb: - ind1, ind2 = str(offspring[i - 1]), str(offspring[i]) - offspring[i - 1], offspring[i] = toolbox.mate(offspring[i - 1], offspring[i]) - for child in [offspring[i - 1], offspring[i]]: - # check if child is the same as their parents - if str(child) != ind1 and str(child) != ind2: - del child.fitness.values - - for i in range(len(offspring)): - if random.random() < mutpb: - tmpind = str(offspring[i]) - offspring[i], = toolbox.mutate(offspring[i]) - if tmpind != str(offspring[i]): - del offspring[i].fitness.values - return offspring + offspring = [] + for _ in range(lambda_): + op_choice = random.random() + if op_choice < cxpb: # Apply crossover + ind1, ind2 = map(toolbox.clone, random.sample(population, 2)) + ind_str = str(ind1) + ind1, ind2 = toolbox.mate(ind1, ind2) + if ind_str != str(ind1): # check if crossover generated a new pipeline + del ind1.fitness.values + offspring.append(ind1) + elif op_choice < cxpb + mutpb: # Apply mutation + ind = toolbox.clone(random.choice(population)) + ind_str = str(ind) + ind, = toolbox.mutate(ind) + if ind_str != str(ind): # check if mutation happend + del ind.fitness.values + offspring.append(ind) + else: # Apply reproduction + offspring.append(random.choice(population)) + return offspring -def eaSimple(self, population, toolbox, cxpb, mutpb, ngen, stats=None, - halloffame=None, verbose=__debug__): - """This algorithm reproduce the simplest evolutionary algorithm as - presented in chapter 7 of [Back2000]_. +def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, + stats=None, halloffame=None, verbose=__debug__, max_time_mins = None): + """This is the :math:`(\mu + \lambda)` evolutionary algorithm. :param population: A list of individuals. :param toolbox: A :class:`~deap.base.Toolbox` that contains the evolution operators. - :param cxpb: The probability of mating two individuals. - :param mutpb: The probability of mutating an individual. + :param mu: The number of individuals to select for the next generation. + :param lambda\_: The number of children to produce at each generation. + :param cxpb: The probability that an offspring is produced by crossover. + :param mutpb: The probability that an offspring is produced by mutation. :param ngen: The number of generation. + :param pbar: processing bar :param stats: A :class:`~deap.tools.Statistics` object that is updated inplace, optional. :param halloffame: A :class:`~deap.tools.HallOfFame` object that will @@ -92,40 +94,30 @@ def eaSimple(self, population, toolbox, cxpb, mutpb, ngen, stats=None, :param verbose: Whether or not to log the statistics. :returns: The final population :returns: A class:`~deap.tools.Logbook` with the statistics of the - evolution + evolution. The algorithm takes in a population and evolves it in place using the - :meth:`varAnd` method. It returns the optimized population and a + :func:`varOr` function. It returns the optimized population and a :class:`~deap.tools.Logbook` with the statistics of the evolution. The logbook will contain the generation number, the number of evalutions for each generation and the statistics if a :class:`~deap.tools.Statistics` is given as argument. The *cxpb* and *mutpb* arguments are passed to the - :func:`varAnd` function. The pseudocode goes as follow :: + :func:`varOr` function. The pseudocode goes as follow :: evaluate(population) for g in range(ngen): - population = select(population, len(population)) - offspring = varAnd(population, toolbox, cxpb, mutpb) + offspring = varOr(population, toolbox, lambda_, cxpb, mutpb) evaluate(offspring) - population = offspring - As stated in the pseudocode above, the algorithm goes as follow. First, it - evaluates the individuals with an invalid fitness. Second, it enters the - generational loop where the selection procedure is applied to entirely - replace the parental population. The 1:1 replacement ratio of this - algorithm **requires** the selection procedure to be stochastic and to - select multiple times the same individual, for example, - :func:`~deap.tools.selTournament` and :func:`~deap.tools.selRoulette`. - Third, it applies the :func:`varAnd` function to produce the next - generation population. Fourth, it evaluates the new individuals and - compute the statistics on this population. Finally, when *ngen* - generations are done, the algorithm returns a tuple with the final + population = select(population + offspring, mu) + First, the individuals having an invalid fitness are evaluated. Second, + the evolutionary loop begins by producing *lambda_* offspring from the + population, the offspring are generated by the :func:`varOr` function. The + offspring are then evaluated and the next generation population is + selected from both the offspring **and** the population. Finally, when + *ngen* generations are done, the algorithm returns a tuple with the final population and a :class:`~deap.tools.Logbook` of the evolution. - .. note:: - Using a non-stochastic selection method will result in no selection as - the operator selects *n* individuals from a pool of *n*. - This function expects the :meth:`toolbox.mate`, :meth:`toolbox.mutate`, + This function expects :meth:`toolbox.mate`, :meth:`toolbox.mutate`, :meth:`toolbox.select` and :meth:`toolbox.evaluate` aliases to be - registered in the toolbox. - .. [Back2000] Back, Fogel and Michalewicz, "Evolutionary Computation 1 : - Basic Algorithms and Operators", 2000. + registered in the toolbox. This algorithm uses the :func:`varOr` + variation. """ logbook = tools.Logbook() logbook.header = ['gen', 'nevals'] + (stats.fields if stats else []) @@ -139,44 +131,13 @@ def eaSimple(self, population, toolbox, cxpb, mutpb, ngen, stats=None, if halloffame is not None: halloffame.update(population) - record = stats.compile(population) if stats else {} + record = stats.compile(population) if stats is not None else {} logbook.record(gen=0, nevals=len(invalid_ind), **record) - if verbose: - print(logbook.stream) # Begin the generational process for gen in range(1, ngen + 1): - # Select the next generation individuals - offspring = toolbox.select(population, len(population)) - # pbar process - self._gp_generation += 1 - if not self._pbar.disable: - # Print only the best individual fitness - if self.verbosity == 2: - high_score = abs(max([self._pareto_front.keys[x].wvalues[1] for x in range(len(self._pareto_front.keys))])) - self._pbar.write('Generation {0} - Current best internal CV score: {1}'.format(self._gp_generation, high_score)) - - # Print the entire Pareto front - elif self.verbosity == 3: - self._pbar.write('Generation {} - Current Pareto front scores:'.format(self._gp_generation)) - for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): - self._pbar.write('{}\t{}\t{}'.format(int(abs(pipeline_scores.wvalues[0])), - abs(pipeline_scores.wvalues[1]), - pipeline)) - self._pbar.write('') - - # Sometimes the actual evaluated pipeline count does not match the - # supposed count because DEAP can cache pipelines. Here any missed - # evaluations are added back to the progress bar. - if self._pbar.n < self._gp_generation * self.population_size: - missing_pipelines = (self._gp_generation * self.population_size) - self._pbar.n - self._pbar.update(missing_pipelines) - - if not (self.max_time_mins is None) and self._pbar.n >= self._pbar.total: - self._pbar.total += self.population_size - - # Vary the pool of individuals - offspring = varAnd(offspring, toolbox, cxpb, mutpb) + # Vary the population + offspring = varOr(population, toolbox, lambda_, cxpb, mutpb) # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in offspring if not ind.fitness.valid] @@ -188,13 +149,36 @@ def eaSimple(self, population, toolbox, cxpb, mutpb, ngen, stats=None, if halloffame is not None: halloffame.update(offspring) - # Replace the current population by the offspring - population[:] = offspring + # Select the next generation population + population[:] = toolbox.select(population + offspring, mu) + + # pbar process + if not pbar.disable: + # Print only the best individual fitness + if verbose == 2: + high_score = abs(max([halloffame.keys[x].wvalues[1] for x in range(len(halloffame.keys))])) + pbar.write('Generation {0} - Current best internal CV score: {1}'.format(gen, high_score)) + + # Print the entire Pareto front + elif verbose == 3: + pbar.write('Generation {} - Current Pareto front scores:'.format(gen)) + for pipeline, pipeline_scores in zip(halloffame.items, reversed(halloffame.keys)): + pbar.write('{}\t{}\t{}'.format(int(abs(pipeline_scores.wvalues[0])), + abs(pipeline_scores.wvalues[1]), + pipeline)) + pbar.write('') + # Sometimes the actual evaluated pipeline count does not match the + # supposed count because DEAP can cache pipelines. Here any missed + # evaluations are added back to the progress bar. + if pbar.n < gen * mu: + missing_pipelines = (gen * mu) - pbar.n + pbar.update(missing_pipelines) + + if not (max_time_mins is None) and pbar.n >= pbar.total: + pbar.total += mu - # Append the current generation statistics to the logbook - record = stats.compile(population) if stats else {} + # Update the statistics with the new population + record = stats.compile(population) if stats is not None else {} logbook.record(gen=gen, nevals=len(invalid_ind), **record) - if verbose: - print(logbook.stream) return population, logbook diff --git a/tpot_test_rework_pbar.py b/tpot_test_rework_pbar.py new file mode 100644 index 00000000..da28267d --- /dev/null +++ b/tpot_test_rework_pbar.py @@ -0,0 +1,26 @@ +from tpot import TPOTClassifier +from sklearn.datasets import make_classification +from sklearn.model_selection import train_test_split +import time + +X, y = make_classification(n_samples=100, n_features=10, random_state=42) +X_train, X_test, y_train, y_test = train_test_split(X, y, + train_size=0.75, test_size=0.25) + +tpot = TPOTClassifier(generations=3, population_size=10, lamda=20, verbosity=1, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used',time.time()-time_start) + +tpot = TPOTClassifier(generations=3, population_size=10, lamda=20, verbosity=2, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used',time.time()-time_start) + +tpot = TPOTClassifier(generations=3, population_size=10, lamda=20, verbosity=3, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used',time.time()-time_start) From cdeaef3a1b11246c34cc1b18db118360d411d183 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 11 Feb 2017 16:29:46 -0500 Subject: [PATCH 031/154] fix unit test --- tests.py | 13 +++++++------ tpot/base.py | 12 +++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tests.py b/tests.py index d8b94a40..af90910b 100644 --- a/tests.py +++ b/tests.py @@ -41,18 +41,19 @@ def test_init_custom_parameters(): """Assert that the TPOT instantiator stores the TPOT variables properly""" - tpot_obj = TPOTClassifier(population_size=500, generations=1000, + tpot_obj = TPOTClassifier(population_size=500, generations=1000, lamda=2000, mutation_rate=0.05, crossover_rate=0.9, - scoring='accuracy', num_cv_folds=10, + scoring='accuracy', cv=10, verbosity=1, random_state=42, disable_update_check=True, warm_start=True) assert tpot_obj.population_size == 500 assert tpot_obj.generations == 1000 + assert tpot_obj.lamda == 2000 assert tpot_obj.mutation_rate == 0.05 assert tpot_obj.crossover_rate == 0.9 assert tpot_obj.scoring_function == 'accuracy' - assert tpot_obj.num_cv_folds == 10 + assert tpot_obj.cv == 10 assert tpot_obj.max_time_mins is None assert tpot_obj.warm_start is True assert tpot_obj.verbosity == 1 @@ -84,6 +85,7 @@ def test_get_params(): kwargs = { 'population_size': 500, 'generations': 1000, + 'lamda': 2000, 'verbosity': 1 } @@ -256,7 +258,7 @@ def test_predict_proba2(): def test_warm_start(): """Assert that the TPOT warm_start flag stores the pop and pareto_front from the first run""" - tpot_obj = TPOTClassifier(random_state=42, population_size=1, generations=1, verbosity=0, warm_start=True) + tpot_obj = TPOTClassifier(random_state=42, population_size=2, lamda=4, generations=1, verbosity=0, warm_start=True) tpot_obj.fit(training_features, training_classes) assert tpot_obj._pop != None @@ -273,11 +275,10 @@ def test_warm_start(): def test_fit(): """Assert that the TPOT fit function provides an optimized pipeline""" - tpot_obj = TPOTClassifier(random_state=42, population_size=5, generations=1, verbosity=0) + tpot_obj = TPOTClassifier(random_state=42, population_size=2, lamda=4, generations=1, verbosity=0) tpot_obj.fit(training_features, training_classes) assert isinstance(tpot_obj._optimized_pipeline, creator.Individual) - assert tpot_obj._gp_generation == 0 assert not (tpot_obj._start_datetime is None) diff --git a/tpot/base.py b/tpot/base.py index b339bd4d..4fd9aab1 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -70,7 +70,7 @@ def handler(dwCtrlType, hook_sigint=_thread.interrupt_main): class TPOTBase(BaseEstimator): """TPOT automatically creates and optimizes machine learning pipelines using genetic programming""" - def __init__(self, population_size=100, lamda=200, generations=100, + def __init__(self, population_size=100, generations=100, lamda=200, mutation_rate=0.9, crossover_rate=0.05, scoring=None, cv=5, n_jobs=1, max_time_mins=None, max_eval_time_mins=5, @@ -165,7 +165,7 @@ def __init__(self, population_size=100, lamda=200, generations=100, self.max_time_mins = max_time_mins self.max_eval_time_mins = max_eval_time_mins # set lamda equal to twice of population_size by default - self.lamda = population_size*2 + self.lamda = lamda # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out @@ -177,8 +177,8 @@ def __init__(self, population_size=100, lamda=200, generations=100, # check if mutation_rate + crossover_rate > 1 if self.mutation_rate + self.crossover_rate > 1: - raise TypeError("The sum of the crossover and mutation probabilities must be smaller " - "or equal to 1.0.") + raise TypeError('The sum of the crossover and mutation probabilities must be smaller ' + 'or equal to 1.0.') self.verbosity = verbosity self.operators_context = { @@ -189,7 +189,7 @@ def __init__(self, population_size=100, lamda=200, generations=100, } self._pbar = None - #self._gp_generation = 0 + # a dictionary of individual which has already evaluated in previous generation. self.eval_ind = {} @@ -387,8 +387,6 @@ def pareto_eq(ind1, ind2): if not isinstance(self._pbar, type(None)): self._pbar.close() - # Reset gp_generation counter to restore initial state - #self._gp_generation = 0 # Store the pipeline with the highest internal testing score if self._pareto_front: From 7c85c2803fe69abb071d6cc7bbad9438f3c36cd8 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 11 Feb 2017 16:45:56 -0500 Subject: [PATCH 032/154] clean codes --- tpot/base.py | 9 ++++++--- tpot_test_rework_pbar.py | 26 -------------------------- 2 files changed, 6 insertions(+), 29 deletions(-) delete mode 100644 tpot_test_rework_pbar.py diff --git a/tpot/base.py b/tpot/base.py index 4fd9aab1..77ff5fc5 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -70,7 +70,7 @@ def handler(dwCtrlType, hook_sigint=_thread.interrupt_main): class TPOTBase(BaseEstimator): """TPOT automatically creates and optimizes machine learning pipelines using genetic programming""" - def __init__(self, population_size=100, generations=100, lamda=200, + def __init__(self, population_size=100, generations=100, lamda=None, mutation_rate=0.9, crossover_rate=0.05, scoring=None, cv=5, n_jobs=1, max_time_mins=None, max_eval_time_mins=5, @@ -84,7 +84,7 @@ def __init__(self, population_size=100, generations=100, lamda=200, The number of pipelines in the genetic algorithm population. Must be > 0.The more pipelines in the population, the slower TPOT will run, but it's also more likely to find better pipelines. - lamda: int (default: twice of population_size) + lamda: int (default: None) The number of children to produce at each generation. generations: int (default: 100) The number of generations to run pipeline optimization for. Must @@ -165,7 +165,10 @@ def __init__(self, population_size=100, generations=100, lamda=200, self.max_time_mins = max_time_mins self.max_eval_time_mins = max_eval_time_mins # set lamda equal to twice of population_size by default - self.lamda = lamda + if lamda: + self.lamda = lamda + else: + self.lamda = population_size # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out diff --git a/tpot_test_rework_pbar.py b/tpot_test_rework_pbar.py deleted file mode 100644 index da28267d..00000000 --- a/tpot_test_rework_pbar.py +++ /dev/null @@ -1,26 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import make_classification -from sklearn.model_selection import train_test_split -import time - -X, y = make_classification(n_samples=100, n_features=10, random_state=42) -X_train, X_test, y_train, y_test = train_test_split(X, y, - train_size=0.75, test_size=0.25) - -tpot = TPOTClassifier(generations=3, population_size=10, lamda=20, verbosity=1, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used',time.time()-time_start) - -tpot = TPOTClassifier(generations=3, population_size=10, lamda=20, verbosity=2, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used',time.time()-time_start) - -tpot = TPOTClassifier(generations=3, population_size=10, lamda=20, verbosity=3, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used',time.time()-time_start) From 5e81cee59228979af3b6e294adff062fb97a3463 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 13 Feb 2017 13:29:34 -0500 Subject: [PATCH 033/154] selection rework --- tpot/base.py | 20 +------------------- tpot/gp_deap.py | 2 ++ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 77ff5fc5..988c2131 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -291,7 +291,7 @@ def _setup_toolbox(self): self._toolbox.register('individual', tools.initIterate, creator.Individual, self._toolbox.expr) self._toolbox.register('population', tools.initRepeat, list, self._toolbox.individual) self._toolbox.register('compile', self._compile_to_sklearn) - self._toolbox.register('select', self._combined_selection_operator) + self._toolbox.register('select', tools.selNSGA2) self._toolbox.register('mate', gp.cxOnePoint) self._toolbox.register('expr_mut', self._gen_grow_safe, min_=1, max_=4) self._toolbox.register('mutate', self._random_mutation_operator) @@ -683,24 +683,6 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No else: raise ValueError('Scoring function does not return a float') - def _combined_selection_operator(self, individuals, k): - """Perform NSGA2 selection on the population according to their Pareto fitness - - Parameters - ---------- - individuals: list - A list of individuals to perform selection on - k: int - The number of individuals to return from the selection phase - - Returns - ------- - fitness: list - Returns a list of individuals that were selected - - """ - return tools.selNSGA2(individuals, int(k / 5.)) * 5 - def _random_mutation_operator(self, individual): """Perform a replacement, insert, or shrink mutation on an individual diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index ebb10044..5eaad3dc 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -151,6 +151,8 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, # Select the next generation population population[:] = toolbox.select(population + offspring, mu) + print(len(population)) + print(len(population+offspring)) # pbar process if not pbar.disable: From 9add96c585091c5dab04dc2205e51946651c81c2 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 13 Feb 2017 13:45:58 -0500 Subject: [PATCH 034/154] spring --- tpot/base.py | 2 +- tpot/gp_deap.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 988c2131..101b3d31 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -164,7 +164,7 @@ def __init__(self, population_size=100, generations=100, lamda=None, self.generations = generations self.max_time_mins = max_time_mins self.max_eval_time_mins = max_eval_time_mins - # set lamda equal to twice of population_size by default + # set lamda equal to population_size by default if lamda: self.lamda = lamda else: diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 5eaad3dc..ebb10044 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -151,8 +151,6 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, # Select the next generation population population[:] = toolbox.select(population + offspring, mu) - print(len(population)) - print(len(population+offspring)) # pbar process if not pbar.disable: From c5e715badec4677901cb6161b99b7abe916b3ade Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 13 Feb 2017 13:51:54 -0500 Subject: [PATCH 035/154] rename --- tests.py | 10 +++++----- tpot/base.py | 14 +++++++------- tpot/driver.py | 10 +++++----- tpot/gp_deap.py | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests.py b/tests.py index af90910b..5662b30f 100644 --- a/tests.py +++ b/tests.py @@ -41,7 +41,7 @@ def test_init_custom_parameters(): """Assert that the TPOT instantiator stores the TPOT variables properly""" - tpot_obj = TPOTClassifier(population_size=500, generations=1000, lamda=2000, + tpot_obj = TPOTClassifier(population_size=500, generations=1000, offspring_size=2000, mutation_rate=0.05, crossover_rate=0.9, scoring='accuracy', cv=10, verbosity=1, random_state=42, @@ -49,7 +49,7 @@ def test_init_custom_parameters(): assert tpot_obj.population_size == 500 assert tpot_obj.generations == 1000 - assert tpot_obj.lamda == 2000 + assert tpot_obj.offspring_size == 2000 assert tpot_obj.mutation_rate == 0.05 assert tpot_obj.crossover_rate == 0.9 assert tpot_obj.scoring_function == 'accuracy' @@ -85,7 +85,7 @@ def test_get_params(): kwargs = { 'population_size': 500, 'generations': 1000, - 'lamda': 2000, + 'offspring_size': 2000, 'verbosity': 1 } @@ -258,7 +258,7 @@ def test_predict_proba2(): def test_warm_start(): """Assert that the TPOT warm_start flag stores the pop and pareto_front from the first run""" - tpot_obj = TPOTClassifier(random_state=42, population_size=2, lamda=4, generations=1, verbosity=0, warm_start=True) + tpot_obj = TPOTClassifier(random_state=42, population_size=2, offspring_size=4, generations=1, verbosity=0, warm_start=True) tpot_obj.fit(training_features, training_classes) assert tpot_obj._pop != None @@ -275,7 +275,7 @@ def test_warm_start(): def test_fit(): """Assert that the TPOT fit function provides an optimized pipeline""" - tpot_obj = TPOTClassifier(random_state=42, population_size=2, lamda=4, generations=1, verbosity=0) + tpot_obj = TPOTClassifier(random_state=42, population_size=2, offspring_size=4, generations=1, verbosity=0) tpot_obj.fit(training_features, training_classes) assert isinstance(tpot_obj._optimized_pipeline, creator.Individual) diff --git a/tpot/base.py b/tpot/base.py index 101b3d31..84db8deb 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -70,7 +70,7 @@ def handler(dwCtrlType, hook_sigint=_thread.interrupt_main): class TPOTBase(BaseEstimator): """TPOT automatically creates and optimizes machine learning pipelines using genetic programming""" - def __init__(self, population_size=100, generations=100, lamda=None, + def __init__(self, population_size=100, generations=100, offspring_size=None, mutation_rate=0.9, crossover_rate=0.05, scoring=None, cv=5, n_jobs=1, max_time_mins=None, max_eval_time_mins=5, @@ -84,7 +84,7 @@ def __init__(self, population_size=100, generations=100, lamda=None, The number of pipelines in the genetic algorithm population. Must be > 0.The more pipelines in the population, the slower TPOT will run, but it's also more likely to find better pipelines. - lamda: int (default: None) + offspring_size: int (default: None) The number of children to produce at each generation. generations: int (default: 100) The number of generations to run pipeline optimization for. Must @@ -164,11 +164,11 @@ def __init__(self, population_size=100, generations=100, lamda=None, self.generations = generations self.max_time_mins = max_time_mins self.max_eval_time_mins = max_eval_time_mins - # set lamda equal to population_size by default - if lamda: - self.lamda = lamda + # set offspring_size equal to population_size by default + if offspring_size: + self.offspring_size = offspring_size else: - self.lamda = population_size + self.offspring_size = population_size # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out @@ -370,7 +370,7 @@ def pareto_eq(ind1, ind2): try: pop, _ = eaMuPlusLambda(population=pop, toolbox=self._toolbox, - mu = self.population_size, lambda_=self.lamda, + mu = self.population_size, lambda_=self.offspring_size, cxpb=self.crossover_rate, mutpb=self.mutation_rate, ngen=self.generations,pbar = self._pbar, halloffame=self._pareto_front, verbose=self.verbosity, max_time_mins = self.max_time_mins) diff --git a/tpot/driver.py b/tpot/driver.py index a0363811..2272163a 100644 --- a/tpot/driver.py +++ b/tpot/driver.py @@ -100,16 +100,16 @@ def main(): 'you give it more generations (and therefore time) to optimize over. ' 'TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.') - parser.add_argument('-lamda', action='store', dest='LAMBDA', default=200, - type=positive_integer, help='The number of children to produce ' - 'at each generation.') - parser.add_argument('-p', action='store', dest='POPULATION_SIZE', default=100, type=positive_integer, help='Number of individuals in the GP population.\n' 'Generally, TPOT will work better when you give it more individuals ' '(and therefore time) to optimize over. TPOT will evaluate ' 'GENERATIONS x POPULATION_SIZE number of pipelines in total.') + parser.add_argument('-os', action='store', dest='OFFSPRING_SIZE', default=100, + type=positive_integer, help='The number of children to produce ' + 'at each generation.') + parser.add_argument('-mr', action='store', dest='MUTATION_RATE', default=0.9, type=float_range, help='GP mutation rate in the range [0.0, 1.0]. We ' 'recommend using the default parameter unless you ' @@ -201,7 +201,7 @@ def main(): tpot_type = TPOTRegressor tpot = tpot_type(generations=args.GENERATIONS, population_size=args.POPULATION_SIZE, - lamda=args.LAMBDA, mutation_rate=args.MUTATION_RATE, crossover_rate=args.CROSSOVER_RATE, + offspring_size=args.OFFSPRING_SIZE, mutation_rate=args.MUTATION_RATE, crossover_rate=args.CROSSOVER_RATE, cv=args.NUM_CV_FOLDS, n_jobs=args.NUM_JOBS, scoring=args.SCORING_FN, max_time_mins=args.MAX_TIME_MINS, max_eval_time_mins=args.MAX_EVAL_MINS, diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index ebb10044..51b3044d 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -76,7 +76,7 @@ def varOr(population, toolbox, lambda_, cxpb, mutpb): return offspring def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, - stats=None, halloffame=None, verbose=__debug__, max_time_mins = None): + stats=None, halloffame=None, verbose=0, max_time_mins = None): """This is the :math:`(\mu + \lambda)` evolutionary algorithm. :param population: A list of individuals. :param toolbox: A :class:`~deap.base.Toolbox` that contains the evolution From 14a8173f06366fcf62f9f7ff070ee789a0a20305 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Mon, 13 Feb 2017 15:11:15 -0500 Subject: [PATCH 036/154] Update base.py --- tpot/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index a335d61d..67c80064 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -48,7 +48,6 @@ from .operators import CombineDFs from .gp_types import Bool, Output_DF from .metrics import SCORERS -from .gp_deap import eaSimple from .gp_deap import eaMuPlusLambda, mutNodeReplacement From 2eb2c3e2c6093c3d5d1f8e6ae747e628f1836e14 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 11 Nov 2016 14:30:34 -0500 Subject: [PATCH 037/154] test_dict_demo --- opt_dict/opt_demo.txt | 8 ++++++++ opt_dict/opt_input_control.py | 1 + 2 files changed, 9 insertions(+) create mode 100644 opt_dict/opt_demo.txt create mode 100644 opt_dict/opt_input_control.py diff --git a/opt_dict/opt_demo.txt b/opt_dict/opt_demo.txt new file mode 100644 index 00000000..10538541 --- /dev/null +++ b/opt_dict/opt_demo.txt @@ -0,0 +1,8 @@ +>DecisionTree +Type: ['Root', 'Classifor'] #--> op.regression +Params: { +criterion: ["gini", "entropy"] +max_depth: range(1, 11) +min_samples_split: range(2, 21) +min_samples_leaf: range(1, 21) +} diff --git a/opt_dict/opt_input_control.py b/opt_dict/opt_input_control.py new file mode 100644 index 00000000..1bfc4fd5 --- /dev/null +++ b/opt_dict/opt_input_control.py @@ -0,0 +1 @@ +# import control of opt From 711e51745a15b412c92a2bd131e22d483d8474e2 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 11 Nov 2016 14:45:35 -0500 Subject: [PATCH 038/154] a litt more --- opt_dict/opt_demo.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/opt_dict/opt_demo.txt b/opt_dict/opt_demo.txt index 10538541..3903cbc7 100644 --- a/opt_dict/opt_demo.txt +++ b/opt_dict/opt_demo.txt @@ -1,8 +1,16 @@ >DecisionTree -Type: ['Root', 'Classifor'] #--> op.regression +Type: ['Root', 'Classifor'] #--> op.root = True, op.classification = True, op.regression = False Params: { criterion: ["gini", "entropy"] max_depth: range(1, 11) min_samples_split: range(2, 21) min_samples_leaf: range(1, 21) } + + +>BernoulliNB +Type: ['Root', 'Classifor'] #--> op.root = True, op.classification = True, op.regression = False +Params: { +alpha: [1e-3, 1e-2, 1e-1, 1., 10., 100.] +fit_prior: [True, False] +} From 3f763e73e6c97aadad192261977a42a7e4288542 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 16 Nov 2016 09:33:35 -0500 Subject: [PATCH 039/154] dict_demo --- opt_dict/opt_input_control.py | 1 + 1 file changed, 1 insertion(+) diff --git a/opt_dict/opt_input_control.py b/opt_dict/opt_input_control.py index 1bfc4fd5..77c890d3 100644 --- a/opt_dict/opt_input_control.py +++ b/opt_dict/opt_input_control.py @@ -1 +1,2 @@ # import control of opt +# exec() From 70c2e47bc0a4558b7d325aba78ae1805e31e9ebe Mon Sep 17 00:00:00 2001 From: Sahil Shah Date: Mon, 19 Dec 2016 12:34:59 -0500 Subject: [PATCH 040/154] Initial support for user-defined conig fix conflict with new development branch --- opt_dict/config_classifier.txt | 86 ++++++++++++++++++ opt_dict/config_regressor.txt | 74 +++++++++++++++ opt_dict/opt_demo.txt | 34 ++++--- tpot/base.py | 73 ++++++++++++--- tpot/config_classifier.py | 158 +++++++++++++++++++++++++++++++++ tpot/config_regressor.py | 76 ++++++++++++++++ 6 files changed, 478 insertions(+), 23 deletions(-) create mode 100644 opt_dict/config_classifier.txt create mode 100644 opt_dict/config_regressor.txt create mode 100644 tpot/config_classifier.py create mode 100644 tpot/config_regressor.py diff --git a/opt_dict/config_classifier.txt b/opt_dict/config_classifier.txt new file mode 100644 index 00000000..7fb86045 --- /dev/null +++ b/opt_dict/config_classifier.txt @@ -0,0 +1,86 @@ +{ + + 'sklearn.tree.DecisionTreeClassifier': { + 'criterion': ['gini', 'entropy'], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + }, + + 'sklearn.naive_bayes.BernoulliNB': { + 'criterion': ['gini', 'entropy'], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + }, + + 'sklearn.naive_bayes.GaussianNB': { + "criterion": ['gini', 'entropy'], + "max_features": [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + "bootstrap": [True, False] + }, + + 'sklearn.ensemble.ExtraTreesClassifier': { + "criterion": ["gini", "entropy"], + "max_features": [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + "bootstrap": [True, False] + }, + + 'sklearn.ensemble.GradientBoostingClassifier': { + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'subsample': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.] + }, + + 'sklearn.neighbors.KNeighborsClassifier': { + 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'weights': ["uniform", "distance"], + 'p': [1, 2] + }, + + 'sklearn.svm.LinearSVC': { + 'penalty': ["l1", "l2"], + 'loss': ["hinge", "squared_hinge"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] + }, + + + 'sklearn.linear_model.LogisticRegression': { + 'penalty': ["l1", "l2"], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'dual': [True, False] + }, + + 'sklearn.naive_bayes.MultinomialNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + }, + + 'sklearn.ensemble.RandomForestClassifier': { + 'criterion': ["gini", "entropy"], + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], + 'bootstrap': [True, False] + } +} + diff --git a/opt_dict/config_regressor.txt b/opt_dict/config_regressor.txt new file mode 100644 index 00000000..65e4a417 --- /dev/null +++ b/opt_dict/config_regressor.txt @@ -0,0 +1,74 @@ +{ + + 'sklearn.linear_model.ElasticNetCV': { + 'l1_ratio': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1] + }, + + 'sklearn.ensemble.ExtraTreesRegressor': { + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'bootstrap': [True, False] + }, + + 'sklearn.ensemble.GradientBoostingRegressor': { + 'loss': ["ls", "lad", "huber", "quantile"], + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'subsample':[0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'alpha': [0.75, 0.8, 0.85, 0.9, 0.95, 0.99] + }, + + 'sklearn.ensemble.AdaBoostRegressor': { + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'loss': ["linear", "square", "exponential"], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + }, + + 'sklearn.tree.DecisionTreeRegressor': { + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + }, + + 'sklearn.neighbors.KNeighborsRegressor': { + 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'weights': ["uniform", "distance"], + 'p': [1, 2] + }, + + 'sklearn.linear_model.LassoLarsCV': { + 'normalize': [True, False] + }, + + 'sklearn.svm.LinearSVR': { + 'loss': ["epsilon_insensitive", "squared_epsilon_insensitive"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'epsilon': [1e-4, 1e-3, 1e-2, 1e-1, 1.] + }, + + 'sklearn.ensemble.RandomForestRegressor': { + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'bootstrap': [True, False] + }, + + 'sklearn.linear_model.RidgeCV': {} +} \ No newline at end of file diff --git a/opt_dict/opt_demo.txt b/opt_dict/opt_demo.txt index 3903cbc7..95109218 100644 --- a/opt_dict/opt_demo.txt +++ b/opt_dict/opt_demo.txt @@ -1,16 +1,24 @@ ->DecisionTree -Type: ['Root', 'Classifor'] #--> op.root = True, op.classification = True, op.regression = False -Params: { -criterion: ["gini", "entropy"] -max_depth: range(1, 11) -min_samples_split: range(2, 21) -min_samples_leaf: range(1, 21) -} +{ + sklearn...'DecisionTree': { + 'Type': ['Root', 'Classifier'], #--> op.root = True, op.classification = True, op.regression = False + 'Params': { + 'criterion': ["gini", "entropy"], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + } + }, + + 'BernoulliNB': { + 'Type': ['Root', 'Classifier'], #--> op.root = True, op.classification = True, op.regression = False + 'Params': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False], + } + }, + 'AdaBoost': { + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], ->BernoulliNB -Type: ['Root', 'Classifor'] #--> op.root = True, op.classification = True, op.regression = False -Params: { -alpha: [1e-3, 1e-2, 1e-1, 1., 10., 100.] -fit_prior: [True, False] + } } diff --git a/tpot/base.py b/tpot/base.py index 67c80064..b73e3117 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -33,6 +33,8 @@ from tqdm import tqdm from sklearn.base import BaseEstimator +from sklearn.base import ClassifierMixin +from sklearn.base import RegressorMixin from sklearn.model_selection import cross_val_score from sklearn.pipeline import make_pipeline, make_union from sklearn.preprocessing import FunctionTransformer @@ -49,7 +51,13 @@ from .gp_types import Bool, Output_DF from .metrics import SCORERS from .gp_deap import eaMuPlusLambda, mutNodeReplacement +from .config_classifier import classifier_config_dict +from .config_regressor import regressor_config_dict +#Create another param for init method: string or dict +#If string: import lite vs actual +#Lite will use a subset of TPOT normal - models that are simple to learn; nothing expensive +#If actual dictionary - means user wants to specify their own models/params etc. # hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS if sys.platform.startswith('win'): @@ -142,6 +150,9 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, Flag indicating whether TPOT will reuse models from previous calls to fit() for faster operation + config: dictionary or string (default: classifier_config_dict) + Sci-kit learn classifiers or regressors, and respective params to include in pipelines + Returns ------- None @@ -223,6 +234,14 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, else: self.n_jobs = n_jobs + if type(config) is dict: + self.operators = config + else: + with open(config, 'r') as f: + data = f.read().replace('\n', ' ') + + self.operators = eval(data) + self._setup_pset() self._setup_toolbox() @@ -232,6 +251,43 @@ def _setup_pset(self): # Rename pipeline input to "input_df" self._pset.renameArguments(ARG0='input_matrix') + # Add all specified operator to primitive set + # Add from imported dictionary + for key, value in self.operators.items(): + l = key.split('.') + op_str = l.pop() + op = eval(op_str) + import_hash = l.join('.') + + if key.startswith('tpot.'): + exec('from {} import {}'.format(import_hash[4:], op_str)) + else: + exec('from {} import {}'.format(import_hash, op_str)) + + input_arg_types = [] + + for arg_name, arg_vals in value.items(): + + input_arg_types = input_arg_types + [type(arg_vals[0])] + # First argument is always a DataFrame + input_arg_types = [np.ndarray] + input_arg_types + + # Add Terminals + for val in arg_vals: + self._pset.addTerminal(val, type(val)) + + if issubclass(op, ClassifierMixin) or issubclass(op, RegressorMixin): + # We need to add rooted primitives twice so that they can + # return both an Output_DF (and thus be the root of the tree), + # and return a np.ndarray so they can exist elsewhere in the tree. + self._pset.addPrimitive(op, input_arg_types, Output_DF) + + return_type = np.ndarray + self._pset.addPrimitive(op, input_arg_types, return_type) + + self._pset.addPrimitive(CombineDFs(), [np.ndarray, np.ndarray], np.ndarray) + + # Add all operators to the primitive set for op in operators.Operator.inheritors(): if self._ignore_operator(op): @@ -399,16 +455,15 @@ def pareto_eq(ind1, ind2): if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline top_score = pipeline_scores.wvalues[1] - # It won't raise error for a small test like in a unit test because a few pipeline sometimes # may fail due to the training data does not fit the operator's requirement. if not self._optimized_pipeline: print('There was an error in the TPOT optimization ' - 'process. This could be because the data was ' - 'not formatted properly, or because data for ' - 'a regression problem was provided to the ' - 'TPOTClassifier object. Please make sure you ' - 'passed the data to TPOT correctly.') + 'process. This could be because the data was ' + 'not formatted properly, or because data for ' + 'a regression problem was provided to the ' + 'TPOTClassifier object. Please make sure you ' + 'passed the data to TPOT correctly.') else: self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) with warnings.catch_warnings(): @@ -709,12 +764,10 @@ def _random_mutation_operator(self, individual): partial(mutNodeReplacement, pset=self._pset), partial(gp.mutShrink) ] - while str(mut_ind[0]) == old_ind: # infinite loop to make sure mutation happen - mut_ind = np.random.choice(mutation_techniques)(individual) - # debug usage - #print(str(mut_ind[0]),'\n') + mut_ind = np.random.choice(mutation_techniques)(individual) return mut_ind + def _gen_grow_safe(self, pset, min_, max_, type_=None): """Generate an expression where each leaf might have a different depth between *min* and *max*. diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py new file mode 100644 index 00000000..b7631141 --- /dev/null +++ b/tpot/config_classifier.py @@ -0,0 +1,158 @@ +# TODO: figure out xg_boost because it does not import directly from sklearn clf class +classifier_config_dict = { + + 'sklearn.tree.DecisionTreeClassifier': { + 'criterion': ['gini', 'entropy'], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + }, + + 'sklearn.naive_bayes.BernoulliNB': { + 'criterion': ['gini', 'entropy'], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + }, + + 'sklearn.naive_bayes.GaussianNB': { + "criterion": ['gini', 'entropy'], + "max_features": [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + "bootstrap": [True, False] + }, + + 'sklearn.ensemble.ExtraTreesClassifier': { + "criterion": ["gini", "entropy"], + "max_features": [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + "bootstrap": [True, False] + }, + + 'sklearn.ensemble.GradientBoostingClassifier': { + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'subsample': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.] + }, + + 'sklearn.neighbors.KNeighborsClassifier': { + 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'weights': ["uniform", "distance"], + 'p': [1, 2] + }, + + 'sklearn.svm.LinearSVC': { + 'penalty': ["l1", "l2"], + 'loss': ["hinge", "squared_hinge"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] + }, + + + 'sklearn.linear_model.LogisticRegression': { + 'penalty': ["l1", "l2"], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'dual': [True, False] + }, + + 'sklearn.naive_bayes.MultinomialNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + }, + + 'sklearn.ensemble.RandomForestClassifier': { + 'criterion': ["gini", "entropy"], + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], + 'bootstrap': [True, False] + }, + + # Preprocessors + 'sklearn.preprocessing.Binarizer': { + 'threshold': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.] + }, + + 'sklearn.decomposition.FastICA': { + 'tol': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.] + }, + + 'sklearn.cluster.FeatureAgglomeration': { + 'linkage': ['ward', 'complete', 'average'], + 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] + }, + + 'sklearn.preprocessing.MaxAbsScaler': { + + }, + + 'sklearn.preprocessing.MinMaxScaler': { + + }, + + 'sklearn.preprocessing.Normalizer': { + 'norm': ['l1', 'l2', 'max'] + }, + + 'sklearn.kernel_approximation.Nystroem': { + 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], + 'gamma': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'n_components': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + }, + + 'sklearn.decomposition.PCA': { + 'svd_solver': 'randomized', + 'iterated_power': [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] + }, + + 'sklearn.preprocessing.PolynomialFeatures': { + 'degree': 2, + 'include_bias': False, + 'interaction_only': False + }, + + 'sklearn.kernel_approximation.RBFSampler': { + 'gamma': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.] + }, + + 'sklearn.preprocessing.RobustScaler': { + + }, + + 'sklearn.preprocessing.StandardScaler': { + + }, + + 'tpot.operators.preprocessors.ZeroCount': { + + }, + + # Selectors + +} + + diff --git a/tpot/config_regressor.py b/tpot/config_regressor.py new file mode 100644 index 00000000..70c53417 --- /dev/null +++ b/tpot/config_regressor.py @@ -0,0 +1,76 @@ +# TODO: figure out xg_boost because it does not import directly from sklearn clf class + +regressor_config_dict = { + + 'sklearn.linear_model.ElasticNetCV': { + 'l1_ratio': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1] + }, + + 'sklearn.ensemble.ExtraTreesRegressor': { + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'bootstrap': [True, False] + }, + + 'sklearn.ensemble.GradientBoostingRegressor': { + 'loss': ["ls", "lad", "huber", "quantile"], + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'subsample':[0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'alpha': [0.75, 0.8, 0.85, 0.9, 0.95, 0.99] + }, + + 'sklearn.ensemble.AdaBoostRegressor': { + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'loss': ["linear", "square", "exponential"], + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + }, + + 'sklearn.tree.DecisionTreeRegressor': { + 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + }, + + 'sklearn.neighbors.KNeighborsRegressor': { + 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'weights': ["uniform", "distance"], + 'p': [1, 2] + }, + + 'sklearn.linear_model.LassoLarsCV': { + 'normalize': [True, False] + }, + + 'sklearn.svm.LinearSVR': { + 'loss': ["epsilon_insensitive", "squared_epsilon_insensitive"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'epsilon': [1e-4, 1e-3, 1e-2, 1e-1, 1.] + }, + + 'sklearn.ensemble.RandomForestRegressor': { + 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'bootstrap': [True, False] + }, + + 'sklearn.linear_model.RidgeCV': {} +} \ No newline at end of file From ac7312b0feb08a19b06abb4e9667283a636a1406 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 24 Jan 2017 11:37:19 -0500 Subject: [PATCH 041/154] config classifier reformated --- tpot/config_classifier.py | 277 ++++++++++++++++------------------- tpot/config_preporcessors.py | 71 +++++++++ 2 files changed, 199 insertions(+), 149 deletions(-) create mode 100644 tpot/config_preporcessors.py diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py index b7631141..1137ad26 100644 --- a/tpot/config_classifier.py +++ b/tpot/config_classifier.py @@ -1,158 +1,137 @@ -# TODO: figure out xg_boost because it does not import directly from sklearn clf class -classifier_config_dict = { - - 'sklearn.tree.DecisionTreeClassifier': { - 'criterion': ['gini', 'entropy'], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - }, - - 'sklearn.naive_bayes.BernoulliNB': { - 'criterion': ['gini', 'entropy'], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - }, - - 'sklearn.naive_bayes.GaussianNB': { - "criterion": ['gini', 'entropy'], - "max_features": [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - "bootstrap": [True, False] - }, - - 'sklearn.ensemble.ExtraTreesClassifier': { - "criterion": ["gini", "entropy"], - "max_features": [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - "bootstrap": [True, False] - }, - - 'sklearn.ensemble.GradientBoostingClassifier': { - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'subsample': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.] - }, - - 'sklearn.neighbors.KNeighborsClassifier': { - 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'weights': ["uniform", "distance"], - 'p': [1, 2] - }, - - 'sklearn.svm.LinearSVC': { - 'penalty': ["l1", "l2"], - 'loss': ["hinge", "squared_hinge"], - 'dual': [True, False], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] - }, - - - 'sklearn.linear_model.LogisticRegression': { - 'penalty': ["l1", "l2"], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], - 'dual': [True, False] - }, - - 'sklearn.naive_bayes.MultinomialNB': { - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False] - }, - - 'sklearn.ensemble.RandomForestClassifier': { - 'criterion': ["gini", "entropy"], - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], - 'bootstrap': [True, False] - }, - - # Preprocessors - 'sklearn.preprocessing.Binarizer': { - 'threshold': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.] - }, - - 'sklearn.decomposition.FastICA': { - 'tol': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.] - }, +# -*- coding: utf-8 -*- - 'sklearn.cluster.FeatureAgglomeration': { - 'linkage': ['ward', 'complete', 'average'], - 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] - }, +""" +Copyright 2016 Randal S. Olson - 'sklearn.preprocessing.MaxAbsScaler': { +This file is part of the TPOT library. - }, +The TPOT library is free software: you can redistribute it and/or +modify it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your option) +any later version. - 'sklearn.preprocessing.MinMaxScaler': { +The TPOT library is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. You should have received a copy of the GNU General Public License along +with the TPOT library. If not, see http://www.gnu.org/licenses/. - }, +""" - 'sklearn.preprocessing.Normalizer': { - 'norm': ['l1', 'l2', 'max'] - }, - - 'sklearn.kernel_approximation.Nystroem': { - 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], - 'gamma': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'n_components': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - }, - - 'sklearn.decomposition.PCA': { - 'svd_solver': 'randomized', - 'iterated_power': [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] - }, - - 'sklearn.preprocessing.PolynomialFeatures': { - 'degree': 2, - 'include_bias': False, - 'interaction_only': False - }, - - 'sklearn.kernel_approximation.RBFSampler': { - 'gamma': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.] - }, +""" +dictionary format (json-like format): +key: + operator name +value: + source: module source (e.g sklearn.tree) + dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency + params: a dictionary of parameter names (keys) and parameter ranges (values); None for no dependency +""" +import numpy as np - 'sklearn.preprocessing.RobustScaler': { - - }, - - 'sklearn.preprocessing.StandardScaler': { - - }, - - 'tpot.operators.preprocessors.ZeroCount': { - - }, - - # Selectors +classifier_config_dict = { + 'GaussianNB': { + 'source': 'sklearn.naive_bayes', + 'dependencies': None, + 'params': None + }, + + 'BernoulliNB': { + 'source': 'sklearn.naive_bayes', + 'dependencies': None, + 'params':{ + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + } + }, + + 'MultinomialNB': { + 'source': 'sklearn.naive_bayes', + 'dependencies': None, + 'params':{ + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + } + }, + + 'DecisionTreeClassifier': { + 'source': 'sklearn.tree', + 'dependencies': None, + 'params':{ + 'criterion': ["gini", "entropy"], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21) + } + }, + + 'ExtraTreesClassifier': { + 'source': 'sklearn.ensemble', + 'dependencies': None, + 'params':{ + 'criterion': ["gini", "entropy"], + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': ["True", "False"] + } + }, + + 'RandomForestClassifier': { + 'source': 'sklearn.ensemble', + 'dependencies': None, + 'params':{ + 'criterion': ["gini", "entropy"], + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': ["True", "False"] + } + }, + + 'GradientBoostingClassifier': { + 'source': 'sklearn.ensemble', + 'dependencies': None, + 'params':{ + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'subsample': np.arange(0.05, 1.01, 0.05), + 'max_features': np.arange(0, 1.01, 0.05) + } + }, + + 'KNeighborsClassifier': { + 'source': 'sklearn.neighbors', + 'dependencies': None, + 'params':{ + 'n_neighbors': range(1, 101), + 'weights': ["uniform", "distance"], + 'p': [1, 2] + } + }, + + 'LinearSVC': { + 'source': 'sklearn.svm', + 'dependencies': None, + 'params':{ + 'penalty': ["l1", "l2"], + 'loss': ["hinge", "squared_hinge"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] + } + }, + + 'LogisticRegression': { + 'source': 'sklearn.linear_model', + 'dependencies': None, + 'params':{ + 'penalty': ["l1", "l2"], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'dual': [True, False] + } + } } - - diff --git a/tpot/config_preporcessors.py b/tpot/config_preporcessors.py new file mode 100644 index 00000000..bff7ad49 --- /dev/null +++ b/tpot/config_preporcessors.py @@ -0,0 +1,71 @@ +# Preprocessors need to rework! + +'sklearn.preprocessing.Binarizer': { + 'threshold': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.] +}, + +'sklearn.decomposition.FastICA': { + 'tol': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.] +}, + +'sklearn.cluster.FeatureAgglomeration': { + 'linkage': ['ward', 'complete', 'average'], + 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] +}, + +'sklearn.preprocessing.MaxAbsScaler': { + +}, + +'sklearn.preprocessing.MinMaxScaler': { + +}, + +'sklearn.preprocessing.Normalizer': { + 'norm': ['l1', 'l2', 'max'] +}, + +'sklearn.kernel_approximation.Nystroem': { + 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], + 'gamma': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.], + 'n_components': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +}, + +'sklearn.decomposition.PCA': { + 'svd_solver': 'randomized', + 'iterated_power': [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] +}, + +'sklearn.preprocessing.PolynomialFeatures': { + 'degree': 2, + 'include_bias': False, + 'interaction_only': False +}, + +'sklearn.kernel_approximation.RBFSampler': { + 'gamma': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, + 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, + 0.9, 0.95, 1.] +}, + +'sklearn.preprocessing.RobustScaler': { + +}, + +'sklearn.preprocessing.StandardScaler': { + +}, + +'tpot.operators.preprocessors.ZeroCount': { + +}, + +# Selectors + +} From d70c474bc9b0c25cf8b1aa3d6840768f02fa24c5 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 24 Jan 2017 17:29:36 -0500 Subject: [PATCH 042/154] dict made --- tpot/build_in_operators.py | 66 ++++++++++++ tpot/config_classifier.py | 16 ++- tpot/config_preporcessor.py | 132 ++++++++++++++++++++++++ tpot/config_preporcessors.py | 71 ------------- tpot/config_regressor.py | 191 +++++++++++++++++++++++++---------- tpot/config_selector.py | 127 +++++++++++++++++++++++ 6 files changed, 475 insertions(+), 128 deletions(-) create mode 100644 tpot/build_in_operators.py create mode 100644 tpot/config_preporcessor.py delete mode 100644 tpot/config_preporcessors.py create mode 100644 tpot/config_selector.py diff --git a/tpot/build_in_operators.py b/tpot/build_in_operators.py new file mode 100644 index 00000000..b2bb7c85 --- /dev/null +++ b/tpot/build_in_operators.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +""" +Copyright 2016 Randal S. Olson + +This file is part of the TPOT library. + +The TPOT library is free software: you can redistribute it and/or +modify it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your option) +any later version. + +The TPOT library is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. You should have received a copy of the GNU General Public License along +with the TPOT library. If not, see http://www.gnu.org/licenses/. + +""" + +import numpy as np +from sklearn.base import BaseEstimator +from sklearn.utils import check_array + + +class ZeroCount(BaseEstimator): + + """Preprocessor that adds two virtual features to the dataset, one for the count of zero values in the feature set, and one for the count of non-zeros in the feature set""" + + def __init__(self): + pass + + def fit(self, X, y=None): + """Dummy function to fit in with the sklearn API""" + return self + + def transform(self, X, y=None): + """Transform data by adding two virtual features + + Parameters + ---------- + X: numpy ndarray, {n_samples, n_components} + New data, where n_samples is the number of samples and n_components + is the number of components. + y: None + Unused + + Returns + ------- + X_transformed: array-like, shape (n_samples, n_features) + The transformed feature set + """ + X = check_array(X) + n_features = X.shape[1] + + X_transformed = np.copy(X) + + non_zero = np.apply_along_axis(lambda row: np.count_nonzero(row), + axis=1, arr=X_transformed) + zero_col = np.apply_along_axis(lambda row: (n_features - np.count_nonzero(row)), + axis=1, arr=X_transformed) + + X_transformed = np.insert(X_transformed, n_features, non_zero, axis=1) + X_transformed = np.insert(X_transformed, n_features + 1, zero_col, axis=1) + + return X_transformed diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py index 1137ad26..90368310 100644 --- a/tpot/config_classifier.py +++ b/tpot/config_classifier.py @@ -74,7 +74,7 @@ 'max_features': np.arange(0, 1.01, 0.05), 'min_samples_split': range(2, 21), 'min_samples_leaf': range(1, 21), - 'bootstrap': ["True", "False"] + 'bootstrap': [True, False] } }, @@ -86,7 +86,7 @@ 'max_features': np.arange(0, 1.01, 0.05), 'min_samples_split': range(2, 21), 'min_samples_leaf': range(1, 21), - 'bootstrap': ["True", "False"] + 'bootstrap': [True, False] } }, @@ -133,5 +133,17 @@ 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], 'dual': [True, False] } + }, + + 'XGBClassifier': { + 'source': 'xgboost', + 'dependencies': None, + 'params':{ + 'max_depth': range(1, 11), + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'subsample': np.arange(0.05, 1.01, 0.05), + 'min_child_weight': range(1, 21) + } } + } diff --git a/tpot/config_preporcessor.py b/tpot/config_preporcessor.py new file mode 100644 index 00000000..54128e09 --- /dev/null +++ b/tpot/config_preporcessor.py @@ -0,0 +1,132 @@ +""" +Copyright 2016 Randal S. Olson + +This file is part of the TPOT library. + +The TPOT library is free software: you can redistribute it and/or +modify it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your option) +any later version. + +The TPOT library is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. You should have received a copy of the GNU General Public License along +with the TPOT library. If not, see http://www.gnu.org/licenses/. + +""" + +""" +dictionary format (json-like format): +key: + operator name +value: + source: module source (e.g sklearn.tree) + dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency + params: a dictionary of parameter names (keys) and parameter ranges (values); None for no params +""" +import numpy as np + +preprocessor_config_dict = { + + 'Binarizer': { + 'source': 'sklearn.preprocessing', + 'dependencies': None, + 'params':{ + 'threshold': np.arange(0.0, 1.01, 0.05) + } + }, + + 'FastICA': { + 'source': 'sklearn.decomposition', + 'dependencies': None, + 'params':{ + 'tol': np.arange(0.0, 1.01, 0.05) + } + }, + + 'FeatureAgglomeration': { + 'source': 'sklearn.cluster', + 'dependencies': None, + 'params':{ + 'linkage': ['ward', 'complete', 'average'], + 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] + } + }, + + 'MaxAbsScaler': { + 'source': 'sklearn.preprocessing', + 'dependencies': None, + 'params': None + }, + + 'MinMaxScaler': { + 'source': 'sklearn.preprocessing', + 'dependencies': None, + 'params': None + }, + + 'Normalizer': { + 'source': 'sklearn.preprocessing', + 'dependencies': None, + 'params': { + 'norm': ['l1', 'l2', 'max'] + } + }, + + 'Nystroem': { + 'source': 'sklearn.kernel_approximation', + 'dependencies': None, + 'params': { + 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], + 'gamma': np.arange(0.0, 1.01, 0.05), + 'n_components': range(1, 11) + } + }, + + 'PCA': { + 'source': 'sklearn.decomposition', + 'dependencies': None, + 'params': { + 'svd_solver': ['randomized'], + 'iterated_power': range(1, 11) + } + }, + + 'PolynomialFeatures': { + 'source': 'sklearn.preprocessing', + 'dependencies': None, + 'params': { + 'degree': [2], + 'include_bias': [False], + 'interaction_only': [False] + } + }, + + 'RBFSampler': { + 'source': 'sklearn.kernel_approximation', + 'dependencies': None, + 'params': { + 'gamma': np.arange(0.0, 1.01, 0.05) + } + }, + + 'RobustScaler': { + 'source': 'sklearn.preprocessing', + 'dependencies': None, + 'params': None + }, + + 'StandardScaler': { + 'source': 'sklearn.preprocessing', + 'dependencies': None, + 'params': None + } + + 'ZeroCount': { + 'source': 'tpot.build_in_operators', + 'dependencies': None, + 'params': None + } + +} diff --git a/tpot/config_preporcessors.py b/tpot/config_preporcessors.py deleted file mode 100644 index bff7ad49..00000000 --- a/tpot/config_preporcessors.py +++ /dev/null @@ -1,71 +0,0 @@ -# Preprocessors need to rework! - -'sklearn.preprocessing.Binarizer': { - 'threshold': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.] -}, - -'sklearn.decomposition.FastICA': { - 'tol': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.] -}, - -'sklearn.cluster.FeatureAgglomeration': { - 'linkage': ['ward', 'complete', 'average'], - 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] -}, - -'sklearn.preprocessing.MaxAbsScaler': { - -}, - -'sklearn.preprocessing.MinMaxScaler': { - -}, - -'sklearn.preprocessing.Normalizer': { - 'norm': ['l1', 'l2', 'max'] -}, - -'sklearn.kernel_approximation.Nystroem': { - 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], - 'gamma': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'n_components': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -}, - -'sklearn.decomposition.PCA': { - 'svd_solver': 'randomized', - 'iterated_power': [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30] -}, - -'sklearn.preprocessing.PolynomialFeatures': { - 'degree': 2, - 'include_bias': False, - 'interaction_only': False -}, - -'sklearn.kernel_approximation.RBFSampler': { - 'gamma': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.] -}, - -'sklearn.preprocessing.RobustScaler': { - -}, - -'sklearn.preprocessing.StandardScaler': { - -}, - -'tpot.operators.preprocessors.ZeroCount': { - -}, - -# Selectors - -} diff --git a/tpot/config_regressor.py b/tpot/config_regressor.py index 70c53417..05c6c36e 100644 --- a/tpot/config_regressor.py +++ b/tpot/config_regressor.py @@ -1,76 +1,157 @@ -# TODO: figure out xg_boost because it does not import directly from sklearn clf class +""" +Copyright 2016 Randal S. Olson + +This file is part of the TPOT library. + +The TPOT library is free software: you can redistribute it and/or +modify it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your option) +any later version. + +The TPOT library is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. You should have received a copy of the GNU General Public License along +with the TPOT library. If not, see http://www.gnu.org/licenses/. + +""" + +""" +dictionary format (json-like format): +key: + operator name +value: + source: module source (e.g sklearn.tree) + dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency + params: a dictionary of parameter names (keys) and parameter ranges (values); None for no params +""" +import numpy as np regressor_config_dict = { - 'sklearn.linear_model.ElasticNetCV': { - 'l1_ratio': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1] + 'LogisticRegression': { + 'source': 'sklearn.linear_model', + 'dependencies': None, + 'params':{ + 'penalty': ["l1", "l2"], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'dual': [True, False] + } + }, + + 'ElasticNetCV': { + 'source': 'sklearn.linear_model', + 'dependencies': None, + 'params':{ + 'l1_ratio': np.arange(0.0, 1.01, 0.05), + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1] + } + }, + + 'ExtraTreesRegressor': { + 'source': 'sklearn.ensemble', + 'dependencies': None, + 'params':{ + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] + } }, - 'sklearn.ensemble.ExtraTreesRegressor': { - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'bootstrap': [True, False] + 'GradientBoostingRegressor': { + 'source': 'sklearn.ensemble', + 'dependencies': None, + 'params':{ + 'loss': ["ls", "lad", "huber", "quantile"], + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'subsample': np.arange(0.05, 1.01, 0.05), + 'max_features': np.arange(0, 1.01, 0.05), + 'alpha': [0.75, 0.8, 0.85, 0.9, 0.95, 0.99] + } }, - 'sklearn.ensemble.GradientBoostingRegressor': { - 'loss': ["ls", "lad", "huber", "quantile"], - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'subsample':[0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'alpha': [0.75, 0.8, 0.85, 0.9, 0.95, 0.99] + 'AdaBoostRegressor': { + 'source': 'sklearn.ensemble', + 'dependencies': None, + 'params':{ + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'loss': ["linear", "square", "exponential"], + 'max_depth': range(1, 11) + } }, - 'sklearn.ensemble.AdaBoostRegressor': { - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'loss': ["linear", "square", "exponential"], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + 'DecisionTreeRegressor': { + 'source': 'sklearn.tree', + 'dependencies': None, + 'params':{ + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21) + } }, - 'sklearn.tree.DecisionTreeRegressor': { - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + 'KNeighborsRegressor': { + 'source': 'sklearn.neighbors', + 'dependencies': None, + 'params':{ + 'n_neighbors': range(1, 101), + 'weights': ["uniform", "distance"], + 'p': [1, 2] + } }, - 'sklearn.neighbors.KNeighborsRegressor': { - 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'weights': ["uniform", "distance"], - 'p': [1, 2] + 'LassoLarsCV': { + 'source': 'sklearn.linear_model', + 'dependencies': None, + 'params':{ + 'normalize': [True, False] + } }, - 'sklearn.linear_model.LassoLarsCV': { - 'normalize': [True, False] + 'LinearSVR': { + 'source': 'sklearn.svm', + 'dependencies': None, + 'params':{ + 'loss': ["epsilon_insensitive", "squared_epsilon_insensitive"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'epsilon': [1e-4, 1e-3, 1e-2, 1e-1, 1.] + } }, - 'sklearn.svm.LinearSVR': { - 'loss': ["epsilon_insensitive", "squared_epsilon_insensitive"], - 'dual': [True, False], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], - 'epsilon': [1e-4, 1e-3, 1e-2, 1e-1, 1.] + 'RandomForestRegressor': { + 'source': 'sklearn.ensemble', + 'dependencies': None, + 'params':{ + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] + } }, - 'sklearn.ensemble.RandomForestRegressor': { - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'bootstrap': [True, False] + 'RidgeCV': { + 'source': 'sklearn.linear_model', + 'dependencies': None, + 'params': None }, - 'sklearn.linear_model.RidgeCV': {} -} \ No newline at end of file + + 'XGBRegressor': { + 'source': 'xgboost', + 'dependencies': None, + 'params':{ + 'max_depth': range(1, 11), + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'subsample': np.arange(0.05, 1.01, 0.05), + 'min_child_weight': range(1, 21) + } + } + + +} diff --git a/tpot/config_selector.py b/tpot/config_selector.py new file mode 100644 index 00000000..a0fd1fb3 --- /dev/null +++ b/tpot/config_selector.py @@ -0,0 +1,127 @@ +""" +Copyright 2016 Randal S. Olson + +This file is part of the TPOT library. + +The TPOT library is free software: you can redistribute it and/or +modify it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your option) +any later version. + +The TPOT library is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. You should have received a copy of the GNU General Public License along +with the TPOT library. If not, see http://www.gnu.org/licenses/. + +""" + +""" +dictionary format (json-like format): +key: + alias: scikit-learn operator name, available for duplicted key in 'selector_config_dict' dictionary + operator name +value: + source: module source (e.g sklearn.tree) + dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency + params: a dictionary of parameter names (keys) and parameter ranges (values); None for no params +""" +import numpy as np + +selector_config_dict = { + 'RFE': { + 'source': 'sklearn.feature_selection', + 'dependencies': { + 'sklearn.svm.SVC': { + 'kernel': ['linear'], + 'random_state': [42] + } + 'classification': True + 'regression': False + }, + 'params':{ + 'step': np.arange(0.1, 1.01, 0.05), + 'estimator': 'sklearn.svm.SVC' # read from dependencies ! need add an exception in preprocess_args + } + }, + + 'SelectFromModel_R': { + 'alias': 'SelectFromModel', # need add an exception for this case + 'source': 'sklearn.feature_selection', + 'dependencies': { + 'sklearn.ensemble.ExtraTreesRegressor': { + 'max_features': np.arange(0, 1.01, 0.05) + } + 'classification': False + 'regression': True + }, + 'params':{ + 'threshold': np.arange(0, 1.01, 0.05), + 'estimator': 'sklearn.ensemble.ExtraTreesRegressor' # read from dependencies ! need add an exception in preprocess_args + } + }, + + 'SelectFromModel': { + 'source': 'sklearn.feature_selection', + 'dependencies': { + 'sklearn.ensemble.ExtraTreesClassifier': { + 'criterion': ['gini', 'entropy'], + 'max_features': np.arange(0, 1.01, 0.05) + } + 'classification': True + 'regression': False + }, + 'params':{ + 'threshold': np.arange(0, 1.01, 0.05), + 'estimator': 'sklearn.ensemble.ExtraTreesRegressor' # read from dependencies ! need add an exception in preprocess_args + } + }, + + 'SelectFwe': { + 'source': 'sklearn.feature_selection', + 'dependencies': { + 'sklearn.feature_selection.f_classif': None + 'classification': True + 'regression': True + }, + 'params':{ + 'alpha': np.arange(0, 0.05, 0.001), + 'score_func': 'sklearn.feature_selection.f_classif' # read from dependencies ! need add an exception in preprocess_args + } + }, + + 'SelectKBest': { + 'source': 'sklearn.feature_selection', + 'dependencies': { + 'sklearn.feature_selection.f_classif': None + 'classification': True + 'regression': True + }, + 'params':{ + 'k': range(1, 100), # need check range! + 'score_func': 'sklearn.feature_selection.f_classif' # read from dependencies ! need add an exception in preprocess_args + } + }, + + 'SelectPercentile': { + 'source': 'sklearn.feature_selection', + 'dependencies': { + 'sklearn.feature_selection.f_classif': None + 'classification': True + 'regression': True + }, + 'params':{ + 'percentile': range(1, 100), + 'score_func': 'sklearn.feature_selection.f_classif' # read from dependencies ! need add an exception in preprocess_args + } + }, + + 'VarianceThreshold': { + 'source': 'sklearn.feature_selection', + 'dependencies': None + 'params':{ + 'threshold': np.arange(0, 0.05, 0.001) + } + } + +} From 271c1cc60c7f80b28569854bcbc044e62e3e233e Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 25 Jan 2017 15:47:42 -0500 Subject: [PATCH 043/154] config class made fix conflict --- tpot/base.py | 7 +- ...preporcessor.py => config_preprocessor.py} | 0 tpot/config_regressor.py | 9 - tpot/config_selector.py | 61 ++-- tpot/operator_utils.py | 297 ++++++++++++++++++ 5 files changed, 325 insertions(+), 49 deletions(-) rename tpot/{config_preporcessor.py => config_preprocessor.py} (100%) create mode 100644 tpot/operator_utils.py diff --git a/tpot/base.py b/tpot/base.py index b73e3117..813ba721 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -33,8 +33,8 @@ from tqdm import tqdm from sklearn.base import BaseEstimator -from sklearn.base import ClassifierMixin -from sklearn.base import RegressorMixin +"""from sklearn.base import ClassifierMixin +from sklearn.base import RegressorMixin""" from sklearn.model_selection import cross_val_score from sklearn.pipeline import make_pipeline, make_union from sklearn.preprocessing import FunctionTransformer @@ -51,8 +51,7 @@ from .gp_types import Bool, Output_DF from .metrics import SCORERS from .gp_deap import eaMuPlusLambda, mutNodeReplacement -from .config_classifier import classifier_config_dict -from .config_regressor import regressor_config_dict + #Create another param for init method: string or dict #If string: import lite vs actual diff --git a/tpot/config_preporcessor.py b/tpot/config_preprocessor.py similarity index 100% rename from tpot/config_preporcessor.py rename to tpot/config_preprocessor.py diff --git a/tpot/config_regressor.py b/tpot/config_regressor.py index 05c6c36e..14790310 100644 --- a/tpot/config_regressor.py +++ b/tpot/config_regressor.py @@ -29,15 +29,6 @@ regressor_config_dict = { - 'LogisticRegression': { - 'source': 'sklearn.linear_model', - 'dependencies': None, - 'params':{ - 'penalty': ["l1", "l2"], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], - 'dual': [True, False] - } - }, 'ElasticNetCV': { 'source': 'sklearn.linear_model', diff --git a/tpot/config_selector.py b/tpot/config_selector.py index a0fd1fb3..23d544d8 100644 --- a/tpot/config_selector.py +++ b/tpot/config_selector.py @@ -19,8 +19,7 @@ """ dictionary format (json-like format): key: - alias: scikit-learn operator name, available for duplicted key in 'selector_config_dict' dictionary - operator name + unique operator name value: source: module source (e.g sklearn.tree) dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency @@ -29,96 +28,86 @@ import numpy as np selector_config_dict = { - 'RFE': { - 'source': 'sklearn.feature_selection', + 'TPOTRFE': { + 'source': 'sklearn.feature_selection.RFE', 'dependencies': { 'sklearn.svm.SVC': { 'kernel': ['linear'], 'random_state': [42] - } - 'classification': True + }, 'regression': False }, 'params':{ 'step': np.arange(0.1, 1.01, 0.05), - 'estimator': 'sklearn.svm.SVC' # read from dependencies ! need add an exception in preprocess_args + 'estimator': 'SVC(kernel=\'linear\', random_state=42)' # read from dependencies ! need add an exception in preprocess_args } }, - 'SelectFromModel_R': { - 'alias': 'SelectFromModel', # need add an exception for this case - 'source': 'sklearn.feature_selection', + 'TPOTSelectFromModelR': { + 'source': 'sklearn.feature_selection.SelectFromModel', 'dependencies': { 'sklearn.ensemble.ExtraTreesRegressor': { 'max_features': np.arange(0, 1.01, 0.05) - } + }, 'classification': False - 'regression': True }, 'params':{ 'threshold': np.arange(0, 1.01, 0.05), - 'estimator': 'sklearn.ensemble.ExtraTreesRegressor' # read from dependencies ! need add an exception in preprocess_args + 'estimator': 'ExtraTreesRegressor(max_features=max_features)' # read from dependencies ! need add an exception in preprocess_args } }, - 'SelectFromModel': { - 'source': 'sklearn.feature_selection', + 'TPOTSelectFromModel': { + 'source': 'sklearn.feature_selection.SelectFromModel', 'dependencies': { 'sklearn.ensemble.ExtraTreesClassifier': { 'criterion': ['gini', 'entropy'], 'max_features': np.arange(0, 1.01, 0.05) - } - 'classification': True + }, 'regression': False }, 'params':{ 'threshold': np.arange(0, 1.01, 0.05), - 'estimator': 'sklearn.ensemble.ExtraTreesRegressor' # read from dependencies ! need add an exception in preprocess_args + 'estimator': 'ExtraTreesClassifier(criterion=criterion_selection, max_features=max_features)' # read from dependencies ! need add an exception in preprocess_args } }, - 'SelectFwe': { - 'source': 'sklearn.feature_selection', + 'TPOTSelectFwe': { + 'source': 'sklearn.feature_selection.SelectFwe', 'dependencies': { 'sklearn.feature_selection.f_classif': None - 'classification': True - 'regression': True }, 'params':{ 'alpha': np.arange(0, 0.05, 0.001), - 'score_func': 'sklearn.feature_selection.f_classif' # read from dependencies ! need add an exception in preprocess_args + 'score_func': 'f_classif' # read from dependencies ! need add an exception in preprocess_args } }, - 'SelectKBest': { - 'source': 'sklearn.feature_selection', + 'TPOTSelectKBest': { + 'source': 'sklearn.feature_selection.SelectKBest', 'dependencies': { 'sklearn.feature_selection.f_classif': None - 'classification': True - 'regression': True }, 'params':{ 'k': range(1, 100), # need check range! - 'score_func': 'sklearn.feature_selection.f_classif' # read from dependencies ! need add an exception in preprocess_args + 'score_func': 'f_classif' # read from dependencies ! need add an exception in preprocess_args } }, - 'SelectPercentile': { - 'source': 'sklearn.feature_selection', + 'TPOTSelectPercentile': { + 'source': 'sklearn.feature_selection.SelectPercentile', 'dependencies': { 'sklearn.feature_selection.f_classif': None - 'classification': True - 'regression': True }, 'params':{ 'percentile': range(1, 100), - 'score_func': 'sklearn.feature_selection.f_classif' # read from dependencies ! need add an exception in preprocess_args + 'score_func': 'f_classif' # read from dependencies ! need add an exception in preprocess_args } }, - 'VarianceThreshold': { - 'source': 'sklearn.feature_selection', - 'dependencies': None + 'TPOTVarianceThreshold': { + 'source': 'sklearn.feature_selection.VarianceThreshold', + 'dependencies': None, 'params':{ 'threshold': np.arange(0, 0.05, 0.001) } diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py new file mode 100644 index 00000000..56acec05 --- /dev/null +++ b/tpot/operator_utils.py @@ -0,0 +1,297 @@ +# -*- coding: utf-8 -*- + +""" +Copyright 2016 Randal S. Olson + +This file is part of the TPOT library. + +The TPOT library is free software: you can redistribute it and/or +modify it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your option) +any later version. + +The TPOT library is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. You should have received a copy of the GNU General Public License along +with the TPOT library. If not, see http://www.gnu.org/licenses/. + +""" + +import numpy as np +from types import FunctionType +#from config_classifier import classifier_config_dict +#from config_regressor import regressor_config_dict +from config_selector import selector_config_dict +#from config_preprocessor import preprocessor_config_dict + +try: + from inspect import signature # Python 3 +except ImportError: + from inspect import getargspec # Python 2 + +class CombineDFs(object): + """Operator to combine two DataFrames""" + + @property + def __name__(self): + return self.__class__.__name__ + + +class Operator(object): + """Base class for operators in TPOT""" + ''' + def export(self, *args, **kwargs): # need rework !!! + """Represent the operator as a string so that it can be exported to a + file + + Parameters + ---------- + args, kwargs + Arbitrary arguments to be passed to the operator + + Returns + ------- + export_string: str + String representation of the sklearn class with its parameters in + the format: + SklearnClassName(param1="val1", param2=val2) + + """ + operator_args = self.preprocess_args(*args, **kwargs) #??? + + arguments = [] + for key in sorted(operator_args.keys()): + val = operator_args[key] + if isinstance(val, str): + val = '\"{}\"'.format(val) + elif isinstance(val, FunctionType): + val = val.__name__ + + arguments.append("{}={}".format(key, val)) + + return "{}({})".format(self.sklearn_class.__name__, ", ".join(arguments))''' + + @property + def __name__(self): + """Necessary for deap so that it can generate a string identifier for + each opeartor. + """ + return self.__class__.sklearn_class.__name__ + ''' + def parameter_types(self): # need rework !!! + """Return tuple of argument types for calling of the operator and the + return type of the operator + + Parameters + ---------- + None + + Returns + ------- + parameter_types: tuple + Tuple of the DEAP parameter types and the DEAP return type for the + operator + + """ + try: + # Python 3 + num_args = len(signature(self.preprocess_args).parameters.keys()) #??? + except NameError: + # Python 2 + + # Remove 'self' + num_args = len(getargspec(self.preprocess_args).args[1:]) #??? + + # Make sure the class has been written properly + if num_args != len(self.arg_types): + raise RuntimeError(("{}'s arg_types does not correspond to the " + "arguments defined for itself". + format(self.__name__))) + + # First argument is always a DataFrame + arg_types = [np.ndarray] + list(self.arg_types) + return_type = np.ndarray + + return (arg_types, return_type)''' + + @classmethod + def inheritors(cls): + """Returns set of all operators defined + + Parameters + ---------- + None + + Returns + ------- + operators: set + Set of all discovered operators that inherit from the base class + + """ + operators = set() + + # Search two levels deep and report leaves in inheritance tree + for operator_type in cls.__subclasses__(): + for operator in operator_type.__subclasses__(): + operators.add(operator()) # Instantiate class and append + + return operators + + @classmethod + def get_by_name(cls, name): + """Returns operator class instance by name + + Parameters + ---------- + name: str + Name of the sklearn class that belongs to a TPOT operator + + Returns + ------- + grandchild + An instance of the TPOT operator with a matching sklearn class name + + """ + for operator_type in cls.__subclasses__(): + for operator in operator_type.__subclasses__(): + if operator.sklearn_class.__name__ == name: + return operator() + +class TPOTOperator(Operator): + """ + A template of TPOT Operator Class + + """ + + root = False # Whether this operator type can be the root of the tree + regression = False # Whether this operator can be used in a regression problem + classification = False # Whether the operator can be used for classification + import_hash = None + sklearn_class = None + arg_types = None + + def preprocess_args(self): + pass + +class ARGType(object): + """Base class for parameter specifications""" + @classmethod + def inheritors(cls): + """Returns set of all parameter types defined + + Returns + ------- + operators: list + List of all discovered operators that inherit from the base class + """ + return [True, False].__subclasses__() + + +def source_decode(sourcecode): + """ Decode operator source and import operator class + Parameters + ---------- + sourcecode: string + a string of operator source (e.g 'sklearn.feature_selection.RFE') + + + Returns + ------- + import_str: string + a string of operator class source (e.g. 'sklearn.feature_selection') + op_str: string + a string of operator class (e.g. 'RFE') + op_obj: object + operator class (e.g. RFE) + + """ + tmp_path = sourcecode.split('.') + op_str = tmp_path.pop() + import_str = '.'.join(tmp_path) + if sourcecode.startswith('tpot.'): + exec('from {} import {}'.format(import_str[4:], op_str)) + else: + exec('from {} import {}'.format(import_str, op_str)) + op_obj = eval(op_str) + return import_str, op_str, op_obj + +def ARGTypeClassFactory(opname, params_dict, BaseClass=ARGType): + """ + Dynamically create parameter type class + """ + arg_class_dict = {} + for key, val in params_dict.items(): + if not isinstance(val, str): + classname = '{}_{}'.format(opname, key) + arg_class_dict[classname] = type(classname, (BaseClass,), {'values':val}) + print(arg_class_dict[classname].values) + return arg_class_dict + +def TPOTOperatorClassFactory(opname, opdict, optype, root, regression, classification, BaseClass=TPOTOperator): + """Dynamically create operator class + Parameters + ---------- + opname: string + operator name in config dictionary (key) + opdict: dictionary + operator profile in config dictionary (value) + BaseClass: Class + inherited BaseClass + Other params: operator profile + + Returns + ------- + newclass: Class + newclass for operators + """ + def __init__(self): + pass + @property + def op_type(self): + """Returns the type of the operator, e.g: + ("Classifier", "Regressor", "Selector", "Preprocessor") + """ + return optype + class_profile = {} + class_profile['__init__'] = __init__ + class_profile['type'] = op_type + class_profile['root'] = root + class_profile['regression'] = regression + class_profile['classification'] = classification + opsourse = opdict['source'] + import_str, op_str, op_obj = source_decode(opsourse) + sklearn_class = op_obj + import_hash = {} + import_hash[import_str] = [op_str] + arg_type_list = ARGTypeClassFactory(opname, opdict['params']) + + if opdict['dependencies']: + for key, val in opdict['dependencies'].items(): + if key.count('.'): # depended module class: + dep_import_str, dep_op_str, dep_op_obj = source_decode(key) + if dep_import_str in import_hash: + import_hash[import_str].append(dep_op_str) + else: + import_hash[dep_import_str] = [dep_op_str] + if val: + dep_opname = '{}_{}'.format(opname, dep_op_str) + dep_arg_type_list = ARGTypeClassFactory(dep_opname, val) + arg_type_list.update(dep_arg_type_list) + else: # exception info for regression or classification + class_profile[key] = val + class_profile['arg_types'] = tuple(arg_type_list) + class_profile['import_hash'] = import_hash + + return type(opname, (BaseClass,),class_profile) + +op_class_dict={} + +for key, val in selector_config_dict.items(): + print('Config: {}'.format(key)) + op_class_dict[key]=TPOTOperatorClassFactory(key, val, optype="Selector", + root=False, regression=True, classification=True) + print(op_class_dict[key].regression) + print(op_class_dict[key].classification) + print(op_class_dict[key].import_hash) + print(op_class_dict[key].arg_types) From 2ec636780fca01f5f680a1b865d1634e794a608d Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 26 Jan 2017 12:39:25 -0500 Subject: [PATCH 044/154] fix ind_dict conflict --- tpot/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index 813ba721..8abe4620 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -721,7 +721,6 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No except TypeError: raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') - except Exception: # Catch-all: Do not allow one pipeline that crashes to cause TPOT # to crash. Instead, assign the crashing pipeline a poor fitness From 19a1825c87c581da3d91f73823dd62ce6bd23dc4 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 25 Jan 2017 17:02:00 -0500 Subject: [PATCH 045/154] reformat dict --- tpot/config_selector.py | 118 +++++++++++++++++----------------------- tpot/operator_utils.py | 78 +++++++++++++------------- 2 files changed, 91 insertions(+), 105 deletions(-) diff --git a/tpot/config_selector.py b/tpot/config_selector.py index 23d544d8..b983a7f2 100644 --- a/tpot/config_selector.py +++ b/tpot/config_selector.py @@ -28,36 +28,62 @@ import numpy as np selector_config_dict = { - 'TPOTRFE': { - 'source': 'sklearn.feature_selection.RFE', - 'dependencies': { - 'sklearn.svm.SVC': { - 'kernel': ['linear'], - 'random_state': [42] - }, - 'regression': False - }, - 'params':{ - 'step': np.arange(0.1, 1.01, 0.05), - 'estimator': 'SVC(kernel=\'linear\', random_state=42)' # read from dependencies ! need add an exception in preprocess_args + + 'sklearn.feature_selection.SelectFromModel': { + 'threshold': np.arange(0, 1.01, 0.05), + 'estimator': { + 'sklearn.ensemble.ExtraTreesRegressor': { + 'max_features': np.arange(0, 1.01, 0.05) + } + } + + }, + + 'sklearn.feature_selection.SelectFwe': { + 'alpha': np.arange(0, 0.05, 0.001), + 'score_func': { + 'sklearn.feature_selection.f_classif': None + } # read from dependencies ! need add an exception in preprocess_args + + }, + + 'sklearn.feature_selection.SelectKBest': { + 'k': range(1, 100), # need check range! + 'score_func': { + 'sklearn.feature_selection.f_classif': None } }, - 'TPOTSelectFromModelR': { - 'source': 'sklearn.feature_selection.SelectFromModel', - 'dependencies': { - 'sklearn.ensemble.ExtraTreesRegressor': { - 'max_features': np.arange(0, 1.01, 0.05) - }, - 'classification': False - }, - 'params':{ - 'threshold': np.arange(0, 1.01, 0.05), - 'estimator': 'ExtraTreesRegressor(max_features=max_features)' # read from dependencies ! need add an exception in preprocess_args + 'sklearn.feature_selection.SelectPercentile': { + 'percentile': range(1, 100), + 'score_func': { + 'sklearn.feature_selection.f_classif': None } }, - 'TPOTSelectFromModel': { + 'sklearn.feature_selection.VarianceThreshold': { + 'threshold': np.arange(0, 0.05, 0.001) + } + +} + +"""'TPOTRFE': { + 'source': 'sklearn.feature_selection.RFE', + 'dependencies': { + 'sklearn.svm.SVC': { + 'kernel': ['linear'], + 'random_state': [42] + }, + 'regression': False + }, + 'params':{ + 'step': np.arange(0.1, 1.01, 0.05), + 'estimator': 'SVC(kernel=\'linear\', random_state=42)' # read from dependencies ! need add an exception in preprocess_args + } +},""" + + +""" 'TPOTSelectFromModel': { 'source': 'sklearn.feature_selection.SelectFromModel', 'dependencies': { 'sklearn.ensemble.ExtraTreesClassifier': { @@ -71,46 +97,4 @@ 'estimator': 'ExtraTreesClassifier(criterion=criterion_selection, max_features=max_features)' # read from dependencies ! need add an exception in preprocess_args } }, - - 'TPOTSelectFwe': { - 'source': 'sklearn.feature_selection.SelectFwe', - 'dependencies': { - 'sklearn.feature_selection.f_classif': None - }, - 'params':{ - 'alpha': np.arange(0, 0.05, 0.001), - 'score_func': 'f_classif' # read from dependencies ! need add an exception in preprocess_args - } - }, - - 'TPOTSelectKBest': { - 'source': 'sklearn.feature_selection.SelectKBest', - 'dependencies': { - 'sklearn.feature_selection.f_classif': None - }, - 'params':{ - 'k': range(1, 100), # need check range! - 'score_func': 'f_classif' # read from dependencies ! need add an exception in preprocess_args - } - }, - - 'TPOTSelectPercentile': { - 'source': 'sklearn.feature_selection.SelectPercentile', - 'dependencies': { - 'sklearn.feature_selection.f_classif': None - }, - 'params':{ - 'percentile': range(1, 100), - 'score_func': 'f_classif' # read from dependencies ! need add an exception in preprocess_args - } - }, - - 'TPOTVarianceThreshold': { - 'source': 'sklearn.feature_selection.VarianceThreshold', - 'dependencies': None, - 'params':{ - 'threshold': np.arange(0, 0.05, 0.001) - } - } - -} +""" diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 56acec05..adacab28 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -20,6 +20,9 @@ import numpy as np from types import FunctionType + +from sklearn.base import ClassifierMixin +from sklearn.base import RegressorMixin #from config_classifier import classifier_config_dict #from config_regressor import regressor_config_dict from config_selector import selector_config_dict @@ -164,7 +167,7 @@ class TPOTOperator(Operator): """ - root = False # Whether this operator type can be the root of the tree + root = True # Whether this operator type can be the root of the tree regression = False # Whether this operator can be used in a regression problem classification = False # Whether the operator can be used for classification import_hash = None @@ -216,26 +219,21 @@ def source_decode(sourcecode): op_obj = eval(op_str) return import_str, op_str, op_obj -def ARGTypeClassFactory(opname, params_dict, BaseClass=ARGType): +def ARGTypeClassFactory(opname, pname, prange, BaseClass=ARGType): """ Dynamically create parameter type class """ - arg_class_dict = {} - for key, val in params_dict.items(): - if not isinstance(val, str): - classname = '{}_{}'.format(opname, key) - arg_class_dict[classname] = type(classname, (BaseClass,), {'values':val}) - print(arg_class_dict[classname].values) - return arg_class_dict - -def TPOTOperatorClassFactory(opname, opdict, optype, root, regression, classification, BaseClass=TPOTOperator): + classname = '{}_{}'.format(opname, pname) + return type(classname, (BaseClass,), {'values':prange}) + +def TPOTOperatorClassFactory(opsourse, opdict, root, regression, classification, BaseClass=TPOTOperator): """Dynamically create operator class Parameters ---------- - opname: string - operator name in config dictionary (key) + opsourse: string + operator source in config dictionary (key) opdict: dictionary - operator profile in config dictionary (value) + operator params in config dictionary (value) BaseClass: Class inherited BaseClass Other params: operator profile @@ -247,51 +245,55 @@ def TPOTOperatorClassFactory(opname, opdict, optype, root, regression, classific """ def __init__(self): pass + + class_profile = {} + class_profile['__init__'] = __init__ + class_profile['regression'] = regression + class_profile['classification'] = classification + import_str, op_str, op_obj = source_decode(opsourse) + if not issubclass(op_obj, ClassifierMixin): + class_profile['root'] = False + optype = "Preprocessor or Selector" @property def op_type(self): """Returns the type of the operator, e.g: ("Classifier", "Regressor", "Selector", "Preprocessor") """ return optype - class_profile = {} - class_profile['__init__'] = __init__ class_profile['type'] = op_type - class_profile['root'] = root - class_profile['regression'] = regression - class_profile['classification'] = classification - opsourse = opdict['source'] - import_str, op_str, op_obj = source_decode(opsourse) + sklearn_class = op_obj import_hash = {} import_hash[import_str] = [op_str] - arg_type_list = ARGTypeClassFactory(opname, opdict['params']) - - if opdict['dependencies']: - for key, val in opdict['dependencies'].items(): - if key.count('.'): # depended module class: - dep_import_str, dep_op_str, dep_op_obj = source_decode(key) + arg_type_dict = {} + for pname, prange in opdict.items(): + if not isinstance(prange, dict): + classname = '{}_{}'.format(op_str, pname) + arg_type_dict[classname] = ARGTypeClassFactory(op_str, pname, prange) + else: + for dkey, dval in prange.items(): + dep_import_str, dep_op_str, dep_op_obj = source_decode(dkey) if dep_import_str in import_hash: import_hash[import_str].append(dep_op_str) else: import_hash[dep_import_str] = [dep_op_str] - if val: - dep_opname = '{}_{}'.format(opname, dep_op_str) - dep_arg_type_list = ARGTypeClassFactory(dep_opname, val) - arg_type_list.update(dep_arg_type_list) - else: # exception info for regression or classification - class_profile[key] = val - class_profile['arg_types'] = tuple(arg_type_list) + if dval: + for dpname, dprange in dval.items(): + classname = '{}_{}'.format(dep_op_str, dpname) + arg_type_dict[classname] = ARGTypeClassFactory(dep_op_str, dpname, dprange) + + class_profile['arg_types'] = tuple(arg_type_dict.values()) class_profile['import_hash'] = import_hash - return type(opname, (BaseClass,),class_profile) + return type(op_str, (BaseClass,),class_profile) op_class_dict={} for key, val in selector_config_dict.items(): print('Config: {}'.format(key)) - op_class_dict[key]=TPOTOperatorClassFactory(key, val, optype="Selector", - root=False, regression=True, classification=True) + op_class_dict[key]=TPOTOperatorClassFactory(key, val, root=False, + regression=True, classification=True) print(op_class_dict[key].regression) - print(op_class_dict[key].classification) + print(op_class_dict[key].root) print(op_class_dict[key].import_hash) print(op_class_dict[key].arg_types) From 53ed802f8bbe8601d746950457f82f21f7f0fbcf Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 26 Jan 2017 10:53:23 -0500 Subject: [PATCH 046/154] reformat classifier dict fix import conflicts --- tpot/base.py | 3 +- tpot/config_classifier.py | 247 ++++++++++++++++++++++++-------------- tpot/config_selector.py | 2 +- tpot/operator_utils.py | 35 ++++-- 4 files changed, 178 insertions(+), 109 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 8abe4620..c043aa4b 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -33,8 +33,6 @@ from tqdm import tqdm from sklearn.base import BaseEstimator -"""from sklearn.base import ClassifierMixin -from sklearn.base import RegressorMixin""" from sklearn.model_selection import cross_val_score from sklearn.pipeline import make_pipeline, make_union from sklearn.preprocessing import FunctionTransformer @@ -53,6 +51,7 @@ from .gp_deap import eaMuPlusLambda, mutNodeReplacement + #Create another param for init method: string or dict #If string: import lite vs actual #Lite will use a subset of TPOT normal - models that are simple to learn; nothing expensive diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py index 90368310..f312cbc6 100644 --- a/tpot/config_classifier.py +++ b/tpot/config_classifier.py @@ -31,119 +31,180 @@ classifier_config_dict = { - 'GaussianNB': { - 'source': 'sklearn.naive_bayes', - 'dependencies': None, - 'params': None - }, - - 'BernoulliNB': { - 'source': 'sklearn.naive_bayes', - 'dependencies': None, - 'params':{ - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False] - } + # Classifiers + 'sklearn.naive_bayes.GaussianNB': { }, - 'MultinomialNB': { - 'source': 'sklearn.naive_bayes', - 'dependencies': None, - 'params':{ - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False] - } + 'sklearn.naive_bayes.BernoulliNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] }, - 'DecisionTreeClassifier': { - 'source': 'sklearn.tree', - 'dependencies': None, - 'params':{ - 'criterion': ["gini", "entropy"], - 'max_depth': range(1, 11), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21) - } + 'sklearn.naive_bayes.MultinomialNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] }, - 'ExtraTreesClassifier': { - 'source': 'sklearn.ensemble', - 'dependencies': None, - 'params':{ - 'criterion': ["gini", "entropy"], - 'max_features': np.arange(0, 1.01, 0.05), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'bootstrap': [True, False] - } + 'sklearn.tree.DecisionTreeClassifier': { + 'criterion': ["gini", "entropy"], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21) }, - 'RandomForestClassifier': { - 'source': 'sklearn.ensemble', - 'dependencies': None, - 'params':{ - 'criterion': ["gini", "entropy"], - 'max_features': np.arange(0, 1.01, 0.05), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'bootstrap': [True, False] - } + 'sklearn.ensemble.ExtraTreesClassifier': { + 'criterion': ["gini", "entropy"], + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] }, - 'GradientBoostingClassifier': { - 'source': 'sklearn.ensemble', - 'dependencies': None, - 'params':{ - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'max_depth': range(1, 11), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'subsample': np.arange(0.05, 1.01, 0.05), - 'max_features': np.arange(0, 1.01, 0.05) - } + 'sklearn.ensemble.RandomForestClassifier': { + 'criterion': ["gini", "entropy"], + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] }, - 'KNeighborsClassifier': { - 'source': 'sklearn.neighbors', - 'dependencies': None, - 'params':{ - 'n_neighbors': range(1, 101), - 'weights': ["uniform", "distance"], - 'p': [1, 2] - } + 'sklearn.ensemble.GradientBoostingClassifier': { + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'subsample': np.arange(0.05, 1.01, 0.05), + 'max_features': np.arange(0, 1.01, 0.05) }, - 'LinearSVC': { - 'source': 'sklearn.svm', - 'dependencies': None, - 'params':{ - 'penalty': ["l1", "l2"], - 'loss': ["hinge", "squared_hinge"], - 'dual': [True, False], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] - } + 'sklearn.neighbors.KNeighborsClassifier': { + 'n_neighbors': range(1, 101), + 'weights': ["uniform", "distance"], + 'p': [1, 2] + }, + + 'sklearn.svm.LinearSVC': { + 'penalty': ["l1", "l2"], + 'loss': ["hinge", "squared_hinge"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] + }, + + 'sklearn.linear_model.LogisticRegression': { + 'penalty': ["l1", "l2"], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'dual': [True, False] + }, + + 'xgboost.XGBClassifier': { + 'max_depth': range(1, 11), + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'subsample': np.arange(0.05, 1.01, 0.05), + 'min_child_weight': range(1, 21) + } + + # Preprocesssors + 'sklearn.preprocessing.Binarizer': { + 'threshold': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.decomposition.FastICA': { + 'tol': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.cluster.FeatureAgglomeration': { + 'linkage': ['ward', 'complete', 'average'], + 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] + }, + + 'sklearn.preprocessing.MaxAbsScaler': { + }, + + 'sklearn.preprocessing.MinMaxScaler': { + }, + + 'sklearn.preprocessing.Normalizer': { + 'norm': ['l1', 'l2', 'max'] }, - 'LogisticRegression': { - 'source': 'sklearn.linear_model', - 'dependencies': None, - 'params':{ - 'penalty': ["l1", "l2"], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], - 'dual': [True, False] + 'sklearn.kernel_approximation.Nystroem': { + 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], + 'gamma': np.arange(0.0, 1.01, 0.05), + 'n_components': range(1, 11) + }, + + 'sklearn.decomposition.PCA': { + 'svd_solver': ['randomized'], + 'iterated_power': range(1, 11) + }, + + 'sklearn.preprocessing.PolynomialFeatures': { + 'degree': [2], + 'include_bias': [False], + 'interaction_only': [False] + }, + + 'sklearn.kernel_approximation.RBFSampler': { + 'gamma': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.preprocessing.RobustScaler': { + }, + + 'sklearn.preprocessing.StandardScaler': { + }, + + 'tpot.build_in_operators.ZeroCount': { + }, + + # Selectors + 'sklearn.feature_selection.SelectFwe': { + 'alpha': np.arange(0, 0.05, 0.001), + 'score_func': { + 'sklearn.feature_selection.f_classif': None + } # read from dependencies ! need add an exception in preprocess_args + + }, + + 'sklearn.feature_selection.SelectKBest': { + 'k': range(1, 100), # need check range! + 'score_func': { + 'sklearn.feature_selection.f_classif': None } }, - 'XGBClassifier': { - 'source': 'xgboost', - 'dependencies': None, - 'params':{ - 'max_depth': range(1, 11), - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'subsample': np.arange(0.05, 1.01, 0.05), - 'min_child_weight': range(1, 21) + 'sklearn.feature_selection.SelectPercentile': { + 'percentile': range(1, 100), + 'score_func': { + 'sklearn.feature_selection.f_classif': None } + }, + + 'sklearn.feature_selection.VarianceThreshold': { + 'threshold': np.arange(0.05, 1.01, 0.05) } + + 'sklearn.feature_selection.RFE': { + 'step': np.arange(0.05, 1.01, 0.05), + 'estimator': { + 'sklearn.svm.SVC': { + 'kernel': ['linear'], + 'random_state': [42] + } + } + }, + + 'sklearn.feature_selection.SelectFromModel': { + 'threshold': np.arange(0, 1.01, 0.05), + 'estimator': { + 'sklearn.ensemble.ExtraTreesClassifier': { + 'criterion': ['gini', 'entropy'], + 'max_features': np.arange(0, 1.01, 0.05) + } + } + }, + + } diff --git a/tpot/config_selector.py b/tpot/config_selector.py index b983a7f2..842b7b7f 100644 --- a/tpot/config_selector.py +++ b/tpot/config_selector.py @@ -62,7 +62,7 @@ }, 'sklearn.feature_selection.VarianceThreshold': { - 'threshold': np.arange(0, 0.05, 0.001) + 'threshold': np.arange(0, 1.01, 0.05) } } diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index adacab28..8ed9b3c3 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -23,10 +23,8 @@ from sklearn.base import ClassifierMixin from sklearn.base import RegressorMixin -#from config_classifier import classifier_config_dict +from config_classifier import classifier_config_dict #from config_regressor import regressor_config_dict -from config_selector import selector_config_dict -#from config_preprocessor import preprocessor_config_dict try: from inspect import signature # Python 3 @@ -167,7 +165,7 @@ class TPOTOperator(Operator): """ - root = True # Whether this operator type can be the root of the tree + root = False # Whether this operator type can be the root of the tree regression = False # Whether this operator can be used in a regression problem classification = False # Whether the operator can be used for classification import_hash = None @@ -188,7 +186,7 @@ def inheritors(cls): operators: list List of all discovered operators that inherit from the base class """ - return [True, False].__subclasses__() + return cls.__subclasses__() def source_decode(sourcecode): @@ -226,7 +224,7 @@ def ARGTypeClassFactory(opname, pname, prange, BaseClass=ARGType): classname = '{}_{}'.format(opname, pname) return type(classname, (BaseClass,), {'values':prange}) -def TPOTOperatorClassFactory(opsourse, opdict, root, regression, classification, BaseClass=TPOTOperator): +def TPOTOperatorClassFactory(opsourse, opdict, root, regression=True, classification=True, BaseClass=TPOTOperator): """Dynamically create operator class Parameters ---------- @@ -234,9 +232,13 @@ def TPOTOperatorClassFactory(opsourse, opdict, root, regression, classification, operator source in config dictionary (key) opdict: dictionary operator params in config dictionary (value) + regression: bool + True if it can be used in TPOTRegressor + classification: bool + True if it can be used in TPOTClassifier BaseClass: Class inherited BaseClass - Other params: operator profile + Returns ------- @@ -251,8 +253,11 @@ def __init__(self): class_profile['regression'] = regression class_profile['classification'] = classification import_str, op_str, op_obj = source_decode(opsourse) - if not issubclass(op_obj, ClassifierMixin): - class_profile['root'] = False + # define if the operator can be the root of a pipeline + if issubclass(op_obj, ClassifierMixin) or issubclass(op_obj, RegressorMixin) + class_profile['root'] = True + optype = "Classifier or Regressor" + else: optype = "Preprocessor or Selector" @property def op_type(self): @@ -262,7 +267,7 @@ def op_type(self): return optype class_profile['type'] = op_type - sklearn_class = op_obj + class_profile['sklearn_class'] = op_obj import_hash = {} import_hash[import_str] = [op_str] arg_type_dict = {} @@ -291,9 +296,13 @@ def op_type(self): for key, val in selector_config_dict.items(): print('Config: {}'.format(key)) - op_class_dict[key]=TPOTOperatorClassFactory(key, val, root=False, - regression=True, classification=True) - print(op_class_dict[key].regression) + op_class_dict[key]=TPOTOperatorClassFactory(key, val, regression=True, classification=True) + print(op_class_dict[key].sklearn_class.__name__) print(op_class_dict[key].root) print(op_class_dict[key].import_hash) print(op_class_dict[key].arg_types) +for op in Operator.inheritors(): + print(op.sklearn_class.__name__) + +for arg in ARGType.inheritors(): + print(arg.__name__, arg.values) From af38a2bc0363fbc17c8e8f2c774a0d732b510449 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 26 Jan 2017 16:48:46 -0500 Subject: [PATCH 047/154] dym_class works --- tpot/base.py | 15 +-- tpot/build_in_operators.py | 7 ++ tpot/config_classifier.py | 5 +- tpot/export_utils.py | 2 +- tpot/operator_utils.py | 199 ++++++++++++++++++++----------------- 5 files changed, 121 insertions(+), 107 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index c043aa4b..dac75a2f 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -44,8 +44,8 @@ from ._version import __version__ from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code from .decorators import _timeout -from . import operators -from .operators import CombineDFs +from .operator_utils import operators, argument_types +from .build_in_operators import CombineDFs from .gp_types import Bool, Output_DF from .metrics import SCORERS from .gp_deap import eaMuPlusLambda, mutNodeReplacement @@ -148,9 +148,6 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, Flag indicating whether TPOT will reuse models from previous calls to fit() for faster operation - config: dictionary or string (default: classifier_config_dict) - Sci-kit learn classifiers or regressors, and respective params to include in pipelines - Returns ------- None @@ -232,14 +229,6 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, else: self.n_jobs = n_jobs - if type(config) is dict: - self.operators = config - else: - with open(config, 'r') as f: - data = f.read().replace('\n', ' ') - - self.operators = eval(data) - self._setup_pset() self._setup_toolbox() diff --git a/tpot/build_in_operators.py b/tpot/build_in_operators.py index b2bb7c85..0641aeb9 100644 --- a/tpot/build_in_operators.py +++ b/tpot/build_in_operators.py @@ -64,3 +64,10 @@ def transform(self, X, y=None): X_transformed = np.insert(X_transformed, n_features + 1, zero_col, axis=1) return X_transformed + +class CombineDFs(object): + """Operator to combine two DataFrames""" + + @property + def __name__(self): + return self.__class__.__name__ diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py index f312cbc6..9a63d2f2 100644 --- a/tpot/config_classifier.py +++ b/tpot/config_classifier.py @@ -102,7 +102,7 @@ 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], 'subsample': np.arange(0.05, 1.01, 0.05), 'min_child_weight': range(1, 21) - } + }, # Preprocesssors 'sklearn.preprocessing.Binarizer': { @@ -183,8 +183,7 @@ 'sklearn.feature_selection.VarianceThreshold': { 'threshold': np.arange(0.05, 1.01, 0.05) - } - + }, 'sklearn.feature_selection.RFE': { 'step': np.arange(0.05, 1.01, 0.05), diff --git a/tpot/export_utils.py b/tpot/export_utils.py index 710e424b..14e1da8e 100644 --- a/tpot/export_utils.py +++ b/tpot/export_utils.py @@ -19,7 +19,7 @@ """ import deap -from . import operators +from .operator_utils import operators def export_pipeline(exported_pipeline): diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 8ed9b3c3..7c52f915 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -41,37 +41,6 @@ def __name__(self): class Operator(object): """Base class for operators in TPOT""" - ''' - def export(self, *args, **kwargs): # need rework !!! - """Represent the operator as a string so that it can be exported to a - file - - Parameters - ---------- - args, kwargs - Arbitrary arguments to be passed to the operator - - Returns - ------- - export_string: str - String representation of the sklearn class with its parameters in - the format: - SklearnClassName(param1="val1", param2=val2) - - """ - operator_args = self.preprocess_args(*args, **kwargs) #??? - - arguments = [] - for key in sorted(operator_args.keys()): - val = operator_args[key] - if isinstance(val, str): - val = '\"{}\"'.format(val) - elif isinstance(val, FunctionType): - val = val.__name__ - - arguments.append("{}={}".format(key, val)) - - return "{}({})".format(self.sklearn_class.__name__, ", ".join(arguments))''' @property def __name__(self): @@ -79,42 +48,7 @@ def __name__(self): each opeartor. """ return self.__class__.sklearn_class.__name__ - ''' - def parameter_types(self): # need rework !!! - """Return tuple of argument types for calling of the operator and the - return type of the operator - Parameters - ---------- - None - - Returns - ------- - parameter_types: tuple - Tuple of the DEAP parameter types and the DEAP return type for the - operator - - """ - try: - # Python 3 - num_args = len(signature(self.preprocess_args).parameters.keys()) #??? - except NameError: - # Python 2 - - # Remove 'self' - num_args = len(getargspec(self.preprocess_args).args[1:]) #??? - - # Make sure the class has been written properly - if num_args != len(self.arg_types): - raise RuntimeError(("{}'s arg_types does not correspond to the " - "arguments defined for itself". - format(self.__name__))) - - # First argument is always a DataFrame - arg_types = [np.ndarray] + list(self.arg_types) - return_type = np.ndarray - - return (arg_types, return_type)''' @classmethod def inheritors(cls): @@ -164,6 +98,8 @@ class TPOTOperator(Operator): A template of TPOT Operator Class """ + def __init__(self): + pass root = False # Whether this operator type can be the root of the tree regression = False # Whether this operator can be used in a regression problem @@ -171,9 +107,8 @@ class TPOTOperator(Operator): import_hash = None sklearn_class = None arg_types = None + dep_op_list = {} # the estimator or score_func as params in this operators - def preprocess_args(self): - pass class ARGType(object): """Base class for parameter specifications""" @@ -211,20 +146,19 @@ def source_decode(sourcecode): op_str = tmp_path.pop() import_str = '.'.join(tmp_path) if sourcecode.startswith('tpot.'): - exec('from {} import {}'.format(import_str[4:], op_str)) + exec('from {} import {}'.format(import_str[5:], op_str)) # need update to 4: else: exec('from {} import {}'.format(import_str, op_str)) op_obj = eval(op_str) return import_str, op_str, op_obj -def ARGTypeClassFactory(opname, pname, prange, BaseClass=ARGType): +def ARGTypeClassFactory(classname, prange, BaseClass=ARGType): """ Dynamically create parameter type class """ - classname = '{}_{}'.format(opname, pname) return type(classname, (BaseClass,), {'values':prange}) -def TPOTOperatorClassFactory(opsourse, opdict, root, regression=True, classification=True, BaseClass=TPOTOperator): +def TPOTOperatorClassFactory(opsourse, opdict, regression=False, classification=False, BaseClass=TPOTOperator): """Dynamically create operator class Parameters ---------- @@ -239,42 +173,42 @@ def TPOTOperatorClassFactory(opsourse, opdict, root, regression=True, classifica BaseClass: Class inherited BaseClass - Returns ------- newclass: Class newclass for operators """ - def __init__(self): - pass + class_profile = {} - class_profile['__init__'] = __init__ class_profile['regression'] = regression class_profile['classification'] = classification + + dep_op_list = {} import_str, op_str, op_obj = source_decode(opsourse) # define if the operator can be the root of a pipeline - if issubclass(op_obj, ClassifierMixin) or issubclass(op_obj, RegressorMixin) + if issubclass(op_obj, ClassifierMixin) or issubclass(op_obj, RegressorMixin): class_profile['root'] = True optype = "Classifier or Regressor" else: optype = "Preprocessor or Selector" - @property - def op_type(self): + + def op_type(): """Returns the type of the operator, e.g: ("Classifier", "Regressor", "Selector", "Preprocessor") """ return optype + class_profile['type'] = op_type class_profile['sklearn_class'] = op_obj import_hash = {} import_hash[import_str] = [op_str] - arg_type_dict = {} + arg_types = [] for pname, prange in opdict.items(): if not isinstance(prange, dict): - classname = '{}_{}'.format(op_str, pname) - arg_type_dict[classname] = ARGTypeClassFactory(op_str, pname, prange) + classname = '{}__{}'.format(op_str, pname) + arg_types.append(ARGTypeClassFactory(classname, prange)) else: for dkey, dval in prange.items(): dep_import_str, dep_op_str, dep_op_obj = source_decode(dkey) @@ -282,27 +216,112 @@ def op_type(self): import_hash[import_str].append(dep_op_str) else: import_hash[dep_import_str] = [dep_op_str] + dep_op_list[pname]=dep_op_str if dval: for dpname, dprange in dval.items(): - classname = '{}_{}'.format(dep_op_str, dpname) - arg_type_dict[classname] = ARGTypeClassFactory(dep_op_str, dpname, dprange) - - class_profile['arg_types'] = tuple(arg_type_dict.values()) + classname = '{}__{}__{}'.format(op_str, dep_op_str, dpname) + arg_types.append(ARGTypeClassFactory(classname, dprange)) + class_profile['arg_types'] = tuple(arg_types) class_profile['import_hash'] = import_hash + class_profile['dep_op_list'] = dep_op_list + + def parameter_types(): + """Return tuple of argument types for calling of the operator and the + return type of the operator + + Parameters + ---------- + None + + Returns + ------- + parameter_types: tuple + Tuple of the DEAP parameter types and the DEAP return type for the + operator + + """ + return ([np.ndarray] + arg_types, np.ndarray) - return type(op_str, (BaseClass,),class_profile) + class_profile['parameter_types'] = parameter_types + + def export(*args): + """Represent the operator as a string so that it can be exported to a + file + + Parameters + ---------- + args + Arbitrary arguments to be passed to the operator + + Returns + ------- + export_string: str + String representation of the sklearn class with its parameters in + the format: + SklearnClassName(param1="val1", param2=val2) + + """ + + op_arguments = [] + if dep_op_list: + dep_op_arguments = {} + for arg_class, arg_value in zip(arg_types, args): + aname_split = arg_class.__name__.split('__') + if isinstance(arg_value, str): + arg_value = '\"{}\"'.format(arg_value) + if len(aname_split) == 2: # simple parameter + op_arguments.append("{}={}".format(aname_split[-1], arg_value)) + else: + if not list(dep_op_list.values()).count(aname_split[1]): + raise TypeError('Warning: the {} is not in right format!'.format(self.sklearn_class.__name__)) + else: + if aname_split[1] not in dep_op_arguments: + dep_op_arguments[aname_split[1]] = [] + dep_op_arguments[aname_split[1]].append("{}={}".format(aname_split[-1], arg_value)) + if dep_op_list: + for dep_op_pname, dep_op_str in dep_op_list.items(): + if dep_op_str == 'f_classif': + arg_value = dep_op_str + else: + arg_value = "{}({})".format(dep_op_str, ", ".join(dep_op_arguments[dep_op_str])) + op_arguments.append("{}={}".format(dep_op_pname, arg_value)) + + return "{}({})".format(op_obj.__name__, ", ".join(op_arguments)) + + class_profile['export'] = export + + + + op_classname = '{}__{}'.format('TPOT',op_str) + return type(op_classname, (BaseClass,), class_profile) + + +# for tpot +operators = Operator.inheritors() +argument_types = ARGType.inheritors() + + +""" +Test op_class_dict={} -for key, val in selector_config_dict.items(): +for key, val in classifier_config_dict.items(): print('Config: {}'.format(key)) - op_class_dict[key]=TPOTOperatorClassFactory(key, val, regression=True, classification=True) + op_class_dict[key]=TPOTOperatorClassFactory(key, val, classification=True) print(op_class_dict[key].sklearn_class.__name__) - print(op_class_dict[key].root) print(op_class_dict[key].import_hash) print(op_class_dict[key].arg_types) +a = op_class_dict['sklearn.naive_bayes.MultinomialNB'] + + + + + for op in Operator.inheritors(): print(op.sklearn_class.__name__) for arg in ARGType.inheritors(): print(arg.__name__, arg.values) + +""" From 76c13426e947b0cb6dac5258512b0115e55e1159 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 26 Jan 2017 17:04:58 -0500 Subject: [PATCH 048/154] better export func --- tpot/operator_utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 7c52f915..091fb30b 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -26,11 +26,6 @@ from config_classifier import classifier_config_dict #from config_regressor import regressor_config_dict -try: - from inspect import signature # Python 3 -except ImportError: - from inspect import getargspec # Python 2 - class CombineDFs(object): """Operator to combine two DataFrames""" @@ -280,13 +275,14 @@ def export(*args): dep_op_arguments[aname_split[1]] = [] dep_op_arguments[aname_split[1]].append("{}={}".format(aname_split[-1], arg_value)) if dep_op_list: + tmp_op_args = [] # to make sure the inital operators is the first parameter just for better persentation for dep_op_pname, dep_op_str in dep_op_list.items(): if dep_op_str == 'f_classif': arg_value = dep_op_str else: arg_value = "{}({})".format(dep_op_str, ", ".join(dep_op_arguments[dep_op_str])) - op_arguments.append("{}={}".format(dep_op_pname, arg_value)) - + tmp_op_args.append("{}={}".format(dep_op_pname, arg_value)) + op_arguments = tmp_op_args + op_arguments return "{}({})".format(op_obj.__name__, ", ".join(op_arguments)) class_profile['export'] = export @@ -313,6 +309,8 @@ def export(*args): print(op_class_dict[key].import_hash) print(op_class_dict[key].arg_types) a = op_class_dict['sklearn.naive_bayes.MultinomialNB'] +c = op_class_dict['sklearn.feature_selection.SelectFromModel'] +d = op_class_dict['sklearn.feature_selection.SelectFwe'] From 197e9bfd8c8a75f53aeb4e9768ac924d4f55c361 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 27 Jan 2017 12:58:19 -0500 Subject: [PATCH 049/154] work around class issue --- test_op_class.py | 19 +++ tpot/base.py | 106 +++++++---------- tpot/config_preprocessor.py | 2 +- tpot/export_utils.py | 78 ++++++++----- tpot/operator_utils.py | 110 ++++-------------- .../__init__.py | 0 tpot/{operators => operators_disable}/base.py | 0 .../classifiers/__init__.py | 0 .../classifiers/base.py | 0 .../classifiers/bernoulli_nb.py | 0 .../classifiers/decision_tree.py | 0 .../classifiers/extra_trees.py | 0 .../classifiers/gaussian_nb.py | 0 .../classifiers/gradient_boosting.py | 0 .../classifiers/knnc.py | 0 .../classifiers/linear_svc.py | 0 .../classifiers/logistic_regression.py | 0 .../classifiers/multinomial_nb.py | 0 .../classifiers/random_forest.py | 0 .../classifiers/xg_boost.py | 0 .../combine_dfs.py | 0 .../preprocessors/__init__.py | 0 .../preprocessors/base.py | 0 .../preprocessors/binarizer.py | 0 .../preprocessors/fast_ica.py | 0 .../preprocessors/feat_agg.py | 0 .../preprocessors/max_abs_scalar.py | 0 .../preprocessors/min_max_scalar.py | 0 .../preprocessors/normalizer.py | 0 .../preprocessors/nystroem.py | 0 .../preprocessors/pca.py | 0 .../preprocessors/polynomial_features.py | 0 .../preprocessors/rbf.py | 0 .../preprocessors/robust_scaler.py | 0 .../preprocessors/standard_scaler.py | 0 .../preprocessors/zero_count.py | 0 .../regressors/__init__.py | 0 .../regressors/base.py | 0 .../regressors/elastic_net.py | 0 .../regressors/extra_trees.py | 0 .../regressors/gradient_boosting.py | 0 .../regressors/knnr.py | 0 .../regressors/lasso_lars_cv.py | 0 .../regressors/linear_svr.py | 0 .../regressors/random_forest.py | 0 .../regressors/xg_boost_r.py | 0 .../selectors/__init__.py | 0 .../selectors/base.py | 0 .../selectors/rfe.py | 0 .../selectors/select_from_model.py | 0 .../selectors/select_from_model_r.py | 0 .../selectors/select_fwe.py | 0 .../selectors/select_kbest.py | 0 .../selectors/select_percentile.py | 0 .../selectors/variance_threshold.py | 0 tpot/tpot.py | 3 + tpot_test_config_dict.py | 16 +++ 57 files changed, 152 insertions(+), 182 deletions(-) create mode 100644 test_op_class.py rename tpot/{operators => operators_disable}/__init__.py (100%) rename tpot/{operators => operators_disable}/base.py (100%) rename tpot/{operators => operators_disable}/classifiers/__init__.py (100%) rename tpot/{operators => operators_disable}/classifiers/base.py (100%) rename tpot/{operators => operators_disable}/classifiers/bernoulli_nb.py (100%) rename tpot/{operators => operators_disable}/classifiers/decision_tree.py (100%) rename tpot/{operators => operators_disable}/classifiers/extra_trees.py (100%) rename tpot/{operators => operators_disable}/classifiers/gaussian_nb.py (100%) rename tpot/{operators => operators_disable}/classifiers/gradient_boosting.py (100%) rename tpot/{operators => operators_disable}/classifiers/knnc.py (100%) rename tpot/{operators => operators_disable}/classifiers/linear_svc.py (100%) rename tpot/{operators => operators_disable}/classifiers/logistic_regression.py (100%) rename tpot/{operators => operators_disable}/classifiers/multinomial_nb.py (100%) rename tpot/{operators => operators_disable}/classifiers/random_forest.py (100%) rename tpot/{operators => operators_disable}/classifiers/xg_boost.py (100%) rename tpot/{operators => operators_disable}/combine_dfs.py (100%) rename tpot/{operators => operators_disable}/preprocessors/__init__.py (100%) rename tpot/{operators => operators_disable}/preprocessors/base.py (100%) rename tpot/{operators => operators_disable}/preprocessors/binarizer.py (100%) rename tpot/{operators => operators_disable}/preprocessors/fast_ica.py (100%) rename tpot/{operators => operators_disable}/preprocessors/feat_agg.py (100%) rename tpot/{operators => operators_disable}/preprocessors/max_abs_scalar.py (100%) rename tpot/{operators => operators_disable}/preprocessors/min_max_scalar.py (100%) rename tpot/{operators => operators_disable}/preprocessors/normalizer.py (100%) rename tpot/{operators => operators_disable}/preprocessors/nystroem.py (100%) rename tpot/{operators => operators_disable}/preprocessors/pca.py (100%) rename tpot/{operators => operators_disable}/preprocessors/polynomial_features.py (100%) rename tpot/{operators => operators_disable}/preprocessors/rbf.py (100%) rename tpot/{operators => operators_disable}/preprocessors/robust_scaler.py (100%) rename tpot/{operators => operators_disable}/preprocessors/standard_scaler.py (100%) rename tpot/{operators => operators_disable}/preprocessors/zero_count.py (100%) rename tpot/{operators => operators_disable}/regressors/__init__.py (100%) rename tpot/{operators => operators_disable}/regressors/base.py (100%) rename tpot/{operators => operators_disable}/regressors/elastic_net.py (100%) rename tpot/{operators => operators_disable}/regressors/extra_trees.py (100%) rename tpot/{operators => operators_disable}/regressors/gradient_boosting.py (100%) rename tpot/{operators => operators_disable}/regressors/knnr.py (100%) rename tpot/{operators => operators_disable}/regressors/lasso_lars_cv.py (100%) rename tpot/{operators => operators_disable}/regressors/linear_svr.py (100%) rename tpot/{operators => operators_disable}/regressors/random_forest.py (100%) rename tpot/{operators => operators_disable}/regressors/xg_boost_r.py (100%) rename tpot/{operators => operators_disable}/selectors/__init__.py (100%) rename tpot/{operators => operators_disable}/selectors/base.py (100%) rename tpot/{operators => operators_disable}/selectors/rfe.py (100%) rename tpot/{operators => operators_disable}/selectors/select_from_model.py (100%) rename tpot/{operators => operators_disable}/selectors/select_from_model_r.py (100%) rename tpot/{operators => operators_disable}/selectors/select_fwe.py (100%) rename tpot/{operators => operators_disable}/selectors/select_kbest.py (100%) rename tpot/{operators => operators_disable}/selectors/select_percentile.py (100%) rename tpot/{operators => operators_disable}/selectors/variance_threshold.py (100%) create mode 100644 tpot_test_config_dict.py diff --git a/test_op_class.py b/test_op_class.py new file mode 100644 index 00000000..93400d11 --- /dev/null +++ b/test_op_class.py @@ -0,0 +1,19 @@ +# coding: utf-8 +from tpot.operator_utils import ARGType, TPOTOperatorClassFactory, Operator +from tpot.config_classifier import classifier_config_dict +from sklearn.base import BaseEstimator + + +class TPOTBase(BaseEstimator): + """TPOT automatically creates and optimizes machine learning pipelines using genetic programming""" + operator_dict = classifier_config_dict + ops = [] + arglist = [] + for key in sorted(operator_dict.keys()): + print('Creating: {}'.format(key)) + op_class, arg_types = TPOTOperatorClassFactory(key, operator_dict[key], classification=True) + ops.append(op_class) + arglist += arg_types + +t = TPOTBase +t() diff --git a/tpot/base.py b/tpot/base.py index dac75a2f..371471aa 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -42,6 +42,7 @@ from update_checker import update_check from ._version import __version__ +from .operator_utils import TPOTOperatorClassFactory from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code from .decorators import _timeout from .operator_utils import operators, argument_types @@ -52,11 +53,6 @@ -#Create another param for init method: string or dict -#If string: import lite vs actual -#Lite will use a subset of TPOT normal - models that are simple to learn; nothing expensive -#If actual dictionary - means user wants to specify their own models/params etc. - # hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS if sys.platform.startswith('win'): import win32api @@ -82,7 +78,7 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, scoring=None, cv=5, n_jobs=1, max_time_mins=None, max_eval_time_mins=5, random_state=None, verbosity=0, - disable_update_check=False, warm_start=False): + disable_update_check=False, warm_start=False, operator_dict_file=None): """Sets up the genetic programming algorithm for pipeline optimization. Parameters @@ -147,6 +143,11 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, warm_start: bool (default: False) Flag indicating whether TPOT will reuse models from previous calls to fit() for faster operation + operator_dict_file: a file including a python dictionary (default: None + The python dictionary need to be named as classifier_config_dict for TPOTClassifier + but regressor_config_dic for TPOTOperator + The customized python dictionary to specify the list of operators and + their arguments. Format examples: config_regressor.py and config_classifier.py Returns ------- @@ -171,11 +172,28 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.generations = generations self.max_time_mins = max_time_mins self.max_eval_time_mins = max_eval_time_mins + # set offspring_size equal to population_size by default if offspring_size: self.offspring_size = offspring_size else: self.offspring_size = population_size + # define operator dictionary + if operator_dict_file: + try: + exec(open(operator_dict_file, 'r').read()) + except: + raise TypeError('The operator dictionary file is in bad format or not available! ' + 'Please check the dictionary file') + self.operators = [] + self.arguments = [] + + for key in sorted(self.operator_dict.keys()): + print('Creating: {}'.format(key)) + op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key], classification=True) + self.operators.append(op_class) + self.arguments += arg_types + print(self.operators) # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out @@ -205,6 +223,7 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.random_state = random_state + # If the user passed a custom scoring function, store it in the sklearn SCORERS dictionary if scoring: if hasattr(scoring, '__call__'): @@ -233,50 +252,18 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self._setup_toolbox() def _setup_pset(self): - self._pset = gp.PrimitiveSetTyped('MAIN', [np.ndarray], Output_DF) - - # Rename pipeline input to "input_df" - self._pset.renameArguments(ARG0='input_matrix') - - # Add all specified operator to primitive set - # Add from imported dictionary - for key, value in self.operators.items(): - l = key.split('.') - op_str = l.pop() - op = eval(op_str) - import_hash = l.join('.') - - if key.startswith('tpot.'): - exec('from {} import {}'.format(import_hash[4:], op_str)) - else: - exec('from {} import {}'.format(import_hash, op_str)) - - input_arg_types = [] - - for arg_name, arg_vals in value.items(): - input_arg_types = input_arg_types + [type(arg_vals[0])] - # First argument is always a DataFrame - input_arg_types = [np.ndarray] + input_arg_types + # creating dynamically create operator class - # Add Terminals - for val in arg_vals: - self._pset.addTerminal(val, type(val)) - if issubclass(op, ClassifierMixin) or issubclass(op, RegressorMixin): - # We need to add rooted primitives twice so that they can - # return both an Output_DF (and thus be the root of the tree), - # and return a np.ndarray so they can exist elsewhere in the tree. - self._pset.addPrimitive(op, input_arg_types, Output_DF) - - return_type = np.ndarray - self._pset.addPrimitive(op, input_arg_types, return_type) + self._pset = gp.PrimitiveSetTyped('MAIN', [np.ndarray], Output_DF) - self._pset.addPrimitive(CombineDFs(), [np.ndarray, np.ndarray], np.ndarray) + # Rename pipeline input to "input_df" + self._pset.renameArguments(ARG0='input_matrix') # Add all operators to the primitive set - for op in operators.Operator.inheritors(): + for op in self.operators: if self._ignore_operator(op): continue @@ -284,6 +271,7 @@ def _setup_pset(self): # We need to add rooted primitives twice so that they can # return both an Output_DF (and thus be the root of the tree), # and return a np.ndarray so they can exist elsewhere in the tree. + print(op.__name__) p_types = (op.parameter_types()[0], Output_DF) self._pset.addPrimitive(op, *p_types) @@ -305,26 +293,9 @@ def _setup_pset(self): self._pset.addPrimitive(CombineDFs(), [np.ndarray, np.ndarray], np.ndarray) # Terminals - int_terminals = np.concatenate(( - np.arange(0, 51, 1), - np.arange(60, 110, 10)) - ) - - for val in int_terminals: - self._pset.addTerminal(val, int) - - float_terminals = np.concatenate(( - [1e-6, 1e-5, 1e-4, 1e-3], - np.arange(0., 1.01, 0.01), - np.arange(2., 51., 1.), - np.arange(60., 101., 10.)) - ) - - for val in float_terminals: - self._pset.addTerminal(val, float) - - self._pset.addTerminal(True, Bool) - self._pset.addTerminal(False, Bool) + for _type in self.arguments: + for val in _type.values: + self._pset.addTerminal(val, _type) def _setup_toolbox(self): creator.create('FitnessMulti', base.Fitness, weights=(-1.0, 1.0)) @@ -585,7 +556,7 @@ def export(self, output_file_name): raise ValueError('A pipeline has not yet been optimized. Please call fit() first.') with open(output_file_name, 'w') as output_file: - output_file.write(export_pipeline(self._optimized_pipeline)) + output_file.write(export_pipeline(self._optimized_pipeline, self.operators)) def _compile_to_sklearn(self, expr): """Compiles a DEAP pipeline into a sklearn pipeline @@ -599,7 +570,7 @@ def _compile_to_sklearn(self, expr): ------- sklearn_pipeline: sklearn.pipeline.Pipeline """ - sklearn_pipeline = generate_pipeline_code(expr_to_tree(expr)) + sklearn_pipeline = generate_pipeline_code(expr_to_tree(expr), self.operators) return eval(sklearn_pipeline, self.operators_context) @@ -673,6 +644,7 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): raise ValueError('Invalid pipeline -- skipping its evaluation') + print(individual_str) # Transform the tree expression into an sklearn pipeline sklearn_pipeline = self._toolbox.compile(expr=individual) @@ -712,8 +684,8 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No except Exception: # Catch-all: Do not allow one pipeline that crashes to cause TPOT # to crash. Instead, assign the crashing pipeline a poor fitness - # import traceback - # traceback.print_exc() + import traceback + traceback.print_exc() return 5000., -float('inf') finally: if not self._pbar.disable: diff --git a/tpot/config_preprocessor.py b/tpot/config_preprocessor.py index 54128e09..89b4c655 100644 --- a/tpot/config_preprocessor.py +++ b/tpot/config_preprocessor.py @@ -121,7 +121,7 @@ 'source': 'sklearn.preprocessing', 'dependencies': None, 'params': None - } + }, 'ZeroCount': { 'source': 'tpot.build_in_operators', diff --git a/tpot/export_utils.py b/tpot/export_utils.py index 14e1da8e..5d33fd61 100644 --- a/tpot/export_utils.py +++ b/tpot/export_utils.py @@ -19,16 +19,40 @@ """ import deap -from .operator_utils import operators +def get_by_name(opname, operators): + """Returns operator class instance by name -def export_pipeline(exported_pipeline): + Parameters + ---------- + opname: str + Name of the sklearn class that belongs to a TPOT operator + operators: list + List of operator classes from operator library + + Returns + ------- + ret_op_class: class + An operator class + + """ + ret_op_class = [op for op in operators if op.__name__ == opname] + if len(ret_op_class) == 0: + raise TypeError('Cannot found operator {} in operator dictionary'.format(opname)) + elif len(ret_op_class) > 1: + print('Found multiple operator {} in operator dictionary'.format(opname), + 'Please check your dictionary file') + return ret_op_class + +def export_pipeline(exported_pipeline, operators): """Generates the source code of a TPOT Pipeline Parameters ---------- exported_pipeline: deap.creator.Individual The pipeline that is being exported + operators: + List of operator classes from operator library Returns ------- @@ -40,7 +64,7 @@ def export_pipeline(exported_pipeline): pipeline_tree = expr_to_tree(exported_pipeline) # Have the exported code import all of the necessary modules and functions - pipeline_text = generate_import_code(exported_pipeline) + pipeline_text = generate_import_code(exported_pipeline, operators) # Replace the function calls with their corresponding Python code pipeline_text += pipeline_code_wrapper(generate_export_pipeline_code(pipeline_tree)) @@ -88,19 +112,21 @@ def prim_to_list(prim, args): return tree -def generate_import_code(pipeline): +def generate_import_code(pipeline, operators): """Generate all library import calls for use in TPOT.export() Parameters ---------- pipeline: List - List of operators in the current optimized pipeline + List of operators in the current optimized pipeline + operators: + List of operator class from operator library Returns ------- pipeline_text: String - The Python code that imports all required library used in the current - optimized pipeline + The Python code that imports all required library used in the current + optimized pipeline """ # operator[1] is the name of the operator @@ -115,14 +141,14 @@ def generate_import_code(pipeline): # Build dict of import requirments from list of operators import_relations = {} - for op in operators.Operator.inheritors(): + for op in operators: import_relations[op.__name__] = op.import_hash # number of classifier/regressor or CombineDFs num_op_root = 0 for op in operators_used: if op != 'CombineDFs': - tpot_op = operators.Operator.get_by_name(op) + tpot_op = get_by_name(op, operators) if tpot_op.root: num_op_root += 1 else: @@ -200,7 +226,7 @@ def pipeline_code_wrapper(pipeline_code): """.format(pipeline_code) -def generate_pipeline_code(pipeline_tree): +def generate_pipeline_code(pipeline_tree, operators): """Generate code specific to the construction of the sklearn Pipeline Parameters @@ -213,11 +239,11 @@ def generate_pipeline_code(pipeline_tree): Source code for the sklearn pipeline """ - steps = process_operator(pipeline_tree) + steps = process_operator(pipeline_tree, operators) pipeline_text = "make_pipeline(\n{STEPS}\n)".format(STEPS=_indent(",\n".join(steps), 4)) return pipeline_text -def generate_export_pipeline_code(pipeline_tree): +def generate_export_pipeline_code(pipeline_tree, operators): """Generate code specific to the construction of the sklearn Pipeline for export_pipeline Parameters @@ -230,7 +256,7 @@ def generate_export_pipeline_code(pipeline_tree): Source code for the sklearn pipeline """ - steps = process_operator(pipeline_tree) + steps = process_operator(pipeline_tree, operators) # number of steps in a pipeline num_step = len(steps) if num_step > 1: @@ -242,20 +268,20 @@ def generate_export_pipeline_code(pipeline_tree): return pipeline_text -def process_operator(operator, depth=0): +def process_operator(operator, depth=0, operators): steps = [] op_name = operator[0] if op_name == "CombineDFs": steps.append( - _combine_dfs(operator[1], operator[2]) + _combine_dfs(operator[1], operator[2], operators) ) else: input_name, args = operator[1], operator[2:] - tpot_op = operators.Operator.get_by_name(op_name) + tpot_op = get_by_name(op_name, operators) if input_name != 'input_matrix': - steps.extend(process_operator(input_name, depth + 1)) + steps.extend(process_operator(input_name, depth + 1, operators)) # If the step is an estimator and is not the last step then we must # add its guess as a synthetic feature @@ -266,7 +292,7 @@ def process_operator(operator, depth=0): ) else: steps.append(tpot_op.export(*args)) - + print(steps) return steps @@ -289,30 +315,30 @@ def _indent(text, amount): return indentation + ('\n' + indentation).join(text.split('\n')) -def _combine_dfs(left, right): +def _combine_dfs(left, right, operators): def _make_branch(branch): if branch == "input_matrix": return "FunctionTransformer(lambda X: X)" elif branch[0] == "CombineDFs": - return _combine_dfs(branch[1], branch[2]) + return _combine_dfs(branch[1], branch[2], operators) elif branch[1] == "input_matrix": # If depth of branch == 1 - tpot_op = operators.Operator.get_by_name(branch[0]) + tpot_op = get_by_name(branch[0], operators) if tpot_op.root: return """make_union(VotingClassifier([('branch', {} -)]), FunctionTransformer(lambda X: X))""".format(_indent(process_operator(branch)[0], 4)) +)]), FunctionTransformer(lambda X: X))""".format(_indent(process_operator(branch, operators)[0], 4)) else: - return process_operator(branch)[0] + return process_operator(branch, operators)[0] else: # We're going to have to make a pipeline - tpot_op = operators.Operator.get_by_name(branch[0]) + tpot_op = get_by_name(branch[0], operators) if tpot_op.root: return """make_union(VotingClassifier([('branch', {} -)]), FunctionTransformer(lambda X: X))""".format(_indent(generate_pipeline_code(branch), 4)) +)]), FunctionTransformer(lambda X: X))""".format(_indent(generate_pipeline_code(branch, operators), 4)) else: - return generate_pipeline_code(branch) + return generate_pipeline_code(branch, operators) return "make_union(\n{},\n{}\n)".\ format(_indent(_make_branch(left), 4), _indent(_make_branch(right), 4)) diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 091fb30b..6823acb7 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -19,23 +19,14 @@ """ import numpy as np -from types import FunctionType - from sklearn.base import ClassifierMixin from sklearn.base import RegressorMixin -from config_classifier import classifier_config_dict -#from config_regressor import regressor_config_dict - -class CombineDFs(object): - """Operator to combine two DataFrames""" - - @property - def __name__(self): - return self.__class__.__name__ class Operator(object): """Base class for operators in TPOT""" + def __init__(self): + pass @property def __name__(self): @@ -43,59 +34,6 @@ def __name__(self): each opeartor. """ return self.__class__.sklearn_class.__name__ - - - @classmethod - def inheritors(cls): - """Returns set of all operators defined - - Parameters - ---------- - None - - Returns - ------- - operators: set - Set of all discovered operators that inherit from the base class - - """ - operators = set() - - # Search two levels deep and report leaves in inheritance tree - for operator_type in cls.__subclasses__(): - for operator in operator_type.__subclasses__(): - operators.add(operator()) # Instantiate class and append - - return operators - - @classmethod - def get_by_name(cls, name): - """Returns operator class instance by name - - Parameters - ---------- - name: str - Name of the sklearn class that belongs to a TPOT operator - - Returns - ------- - grandchild - An instance of the TPOT operator with a matching sklearn class name - - """ - for operator_type in cls.__subclasses__(): - for operator in operator_type.__subclasses__(): - if operator.sklearn_class.__name__ == name: - return operator() - -class TPOTOperator(Operator): - """ - A template of TPOT Operator Class - - """ - def __init__(self): - pass - root = False # Whether this operator type can be the root of the tree regression = False # Whether this operator can be used in a regression problem classification = False # Whether the operator can be used for classification @@ -105,18 +43,11 @@ def __init__(self): dep_op_list = {} # the estimator or score_func as params in this operators + class ARGType(object): """Base class for parameter specifications""" - @classmethod - def inheritors(cls): - """Returns set of all parameter types defined - - Returns - ------- - operators: list - List of all discovered operators that inherit from the base class - """ - return cls.__subclasses__() + def __init__(self): + pass def source_decode(sourcecode): @@ -141,7 +72,7 @@ def source_decode(sourcecode): op_str = tmp_path.pop() import_str = '.'.join(tmp_path) if sourcecode.startswith('tpot.'): - exec('from {} import {}'.format(import_str[5:], op_str)) # need update to 4: + exec('from {} import {}'.format(import_str[4:], op_str)) else: exec('from {} import {}'.format(import_str, op_str)) op_obj = eval(op_str) @@ -153,7 +84,7 @@ def ARGTypeClassFactory(classname, prange, BaseClass=ARGType): """ return type(classname, (BaseClass,), {'values':prange}) -def TPOTOperatorClassFactory(opsourse, opdict, regression=False, classification=False, BaseClass=TPOTOperator): +def TPOTOperatorClassFactory(opsourse, opdict, regression=False, classification=False, BaseClass=Operator): """Dynamically create operator class Parameters ---------- @@ -170,8 +101,11 @@ def TPOTOperatorClassFactory(opsourse, opdict, regression=False, classification= Returns ------- - newclass: Class - newclass for operators + op_class: Class + a new class for a operator + arg_types: list + a list of parameter class + """ @@ -200,7 +134,8 @@ def op_type(): import_hash = {} import_hash[import_str] = [op_str] arg_types = [] - for pname, prange in opdict.items(): + for pname in sorted(opdict.keys()): + prange = opdict[pname] if not isinstance(prange, dict): classname = '{}__{}'.format(op_str, pname) arg_types.append(ARGTypeClassFactory(classname, prange)) @@ -213,7 +148,8 @@ def op_type(): import_hash[dep_import_str] = [dep_op_str] dep_op_list[pname]=dep_op_str if dval: - for dpname, dprange in dval.items(): + for dpname in sorted(dval.keys()): + dprange = dval[dpname] classname = '{}__{}__{}'.format(op_str, dep_op_str, dpname) arg_types.append(ARGTypeClassFactory(classname, dprange)) class_profile['arg_types'] = tuple(arg_types) @@ -267,15 +203,16 @@ def export(*args): arg_value = '\"{}\"'.format(arg_value) if len(aname_split) == 2: # simple parameter op_arguments.append("{}={}".format(aname_split[-1], arg_value)) - else: + else: # parameter of internal operator as a parameter in the operator, usually in Selector if not list(dep_op_list.values()).count(aname_split[1]): raise TypeError('Warning: the {} is not in right format!'.format(self.sklearn_class.__name__)) else: if aname_split[1] not in dep_op_arguments: dep_op_arguments[aname_split[1]] = [] dep_op_arguments[aname_split[1]].append("{}={}".format(aname_split[-1], arg_value)) + tmp_op_args = [] if dep_op_list: - tmp_op_args = [] # to make sure the inital operators is the first parameter just for better persentation + # to make sure the inital operators is the first parameter just for better persentation for dep_op_pname, dep_op_str in dep_op_list.items(): if dep_op_str == 'f_classif': arg_value = dep_op_str @@ -287,15 +224,12 @@ def export(*args): class_profile['export'] = export - - op_classname = '{}__{}'.format('TPOT',op_str) - return type(op_classname, (BaseClass,), class_profile) + op_class = type(op_classname, (BaseClass,), class_profile) + return op_class, arg_types + -# for tpot -operators = Operator.inheritors() -argument_types = ARGType.inheritors() """ diff --git a/tpot/operators/__init__.py b/tpot/operators_disable/__init__.py similarity index 100% rename from tpot/operators/__init__.py rename to tpot/operators_disable/__init__.py diff --git a/tpot/operators/base.py b/tpot/operators_disable/base.py similarity index 100% rename from tpot/operators/base.py rename to tpot/operators_disable/base.py diff --git a/tpot/operators/classifiers/__init__.py b/tpot/operators_disable/classifiers/__init__.py similarity index 100% rename from tpot/operators/classifiers/__init__.py rename to tpot/operators_disable/classifiers/__init__.py diff --git a/tpot/operators/classifiers/base.py b/tpot/operators_disable/classifiers/base.py similarity index 100% rename from tpot/operators/classifiers/base.py rename to tpot/operators_disable/classifiers/base.py diff --git a/tpot/operators/classifiers/bernoulli_nb.py b/tpot/operators_disable/classifiers/bernoulli_nb.py similarity index 100% rename from tpot/operators/classifiers/bernoulli_nb.py rename to tpot/operators_disable/classifiers/bernoulli_nb.py diff --git a/tpot/operators/classifiers/decision_tree.py b/tpot/operators_disable/classifiers/decision_tree.py similarity index 100% rename from tpot/operators/classifiers/decision_tree.py rename to tpot/operators_disable/classifiers/decision_tree.py diff --git a/tpot/operators/classifiers/extra_trees.py b/tpot/operators_disable/classifiers/extra_trees.py similarity index 100% rename from tpot/operators/classifiers/extra_trees.py rename to tpot/operators_disable/classifiers/extra_trees.py diff --git a/tpot/operators/classifiers/gaussian_nb.py b/tpot/operators_disable/classifiers/gaussian_nb.py similarity index 100% rename from tpot/operators/classifiers/gaussian_nb.py rename to tpot/operators_disable/classifiers/gaussian_nb.py diff --git a/tpot/operators/classifiers/gradient_boosting.py b/tpot/operators_disable/classifiers/gradient_boosting.py similarity index 100% rename from tpot/operators/classifiers/gradient_boosting.py rename to tpot/operators_disable/classifiers/gradient_boosting.py diff --git a/tpot/operators/classifiers/knnc.py b/tpot/operators_disable/classifiers/knnc.py similarity index 100% rename from tpot/operators/classifiers/knnc.py rename to tpot/operators_disable/classifiers/knnc.py diff --git a/tpot/operators/classifiers/linear_svc.py b/tpot/operators_disable/classifiers/linear_svc.py similarity index 100% rename from tpot/operators/classifiers/linear_svc.py rename to tpot/operators_disable/classifiers/linear_svc.py diff --git a/tpot/operators/classifiers/logistic_regression.py b/tpot/operators_disable/classifiers/logistic_regression.py similarity index 100% rename from tpot/operators/classifiers/logistic_regression.py rename to tpot/operators_disable/classifiers/logistic_regression.py diff --git a/tpot/operators/classifiers/multinomial_nb.py b/tpot/operators_disable/classifiers/multinomial_nb.py similarity index 100% rename from tpot/operators/classifiers/multinomial_nb.py rename to tpot/operators_disable/classifiers/multinomial_nb.py diff --git a/tpot/operators/classifiers/random_forest.py b/tpot/operators_disable/classifiers/random_forest.py similarity index 100% rename from tpot/operators/classifiers/random_forest.py rename to tpot/operators_disable/classifiers/random_forest.py diff --git a/tpot/operators/classifiers/xg_boost.py b/tpot/operators_disable/classifiers/xg_boost.py similarity index 100% rename from tpot/operators/classifiers/xg_boost.py rename to tpot/operators_disable/classifiers/xg_boost.py diff --git a/tpot/operators/combine_dfs.py b/tpot/operators_disable/combine_dfs.py similarity index 100% rename from tpot/operators/combine_dfs.py rename to tpot/operators_disable/combine_dfs.py diff --git a/tpot/operators/preprocessors/__init__.py b/tpot/operators_disable/preprocessors/__init__.py similarity index 100% rename from tpot/operators/preprocessors/__init__.py rename to tpot/operators_disable/preprocessors/__init__.py diff --git a/tpot/operators/preprocessors/base.py b/tpot/operators_disable/preprocessors/base.py similarity index 100% rename from tpot/operators/preprocessors/base.py rename to tpot/operators_disable/preprocessors/base.py diff --git a/tpot/operators/preprocessors/binarizer.py b/tpot/operators_disable/preprocessors/binarizer.py similarity index 100% rename from tpot/operators/preprocessors/binarizer.py rename to tpot/operators_disable/preprocessors/binarizer.py diff --git a/tpot/operators/preprocessors/fast_ica.py b/tpot/operators_disable/preprocessors/fast_ica.py similarity index 100% rename from tpot/operators/preprocessors/fast_ica.py rename to tpot/operators_disable/preprocessors/fast_ica.py diff --git a/tpot/operators/preprocessors/feat_agg.py b/tpot/operators_disable/preprocessors/feat_agg.py similarity index 100% rename from tpot/operators/preprocessors/feat_agg.py rename to tpot/operators_disable/preprocessors/feat_agg.py diff --git a/tpot/operators/preprocessors/max_abs_scalar.py b/tpot/operators_disable/preprocessors/max_abs_scalar.py similarity index 100% rename from tpot/operators/preprocessors/max_abs_scalar.py rename to tpot/operators_disable/preprocessors/max_abs_scalar.py diff --git a/tpot/operators/preprocessors/min_max_scalar.py b/tpot/operators_disable/preprocessors/min_max_scalar.py similarity index 100% rename from tpot/operators/preprocessors/min_max_scalar.py rename to tpot/operators_disable/preprocessors/min_max_scalar.py diff --git a/tpot/operators/preprocessors/normalizer.py b/tpot/operators_disable/preprocessors/normalizer.py similarity index 100% rename from tpot/operators/preprocessors/normalizer.py rename to tpot/operators_disable/preprocessors/normalizer.py diff --git a/tpot/operators/preprocessors/nystroem.py b/tpot/operators_disable/preprocessors/nystroem.py similarity index 100% rename from tpot/operators/preprocessors/nystroem.py rename to tpot/operators_disable/preprocessors/nystroem.py diff --git a/tpot/operators/preprocessors/pca.py b/tpot/operators_disable/preprocessors/pca.py similarity index 100% rename from tpot/operators/preprocessors/pca.py rename to tpot/operators_disable/preprocessors/pca.py diff --git a/tpot/operators/preprocessors/polynomial_features.py b/tpot/operators_disable/preprocessors/polynomial_features.py similarity index 100% rename from tpot/operators/preprocessors/polynomial_features.py rename to tpot/operators_disable/preprocessors/polynomial_features.py diff --git a/tpot/operators/preprocessors/rbf.py b/tpot/operators_disable/preprocessors/rbf.py similarity index 100% rename from tpot/operators/preprocessors/rbf.py rename to tpot/operators_disable/preprocessors/rbf.py diff --git a/tpot/operators/preprocessors/robust_scaler.py b/tpot/operators_disable/preprocessors/robust_scaler.py similarity index 100% rename from tpot/operators/preprocessors/robust_scaler.py rename to tpot/operators_disable/preprocessors/robust_scaler.py diff --git a/tpot/operators/preprocessors/standard_scaler.py b/tpot/operators_disable/preprocessors/standard_scaler.py similarity index 100% rename from tpot/operators/preprocessors/standard_scaler.py rename to tpot/operators_disable/preprocessors/standard_scaler.py diff --git a/tpot/operators/preprocessors/zero_count.py b/tpot/operators_disable/preprocessors/zero_count.py similarity index 100% rename from tpot/operators/preprocessors/zero_count.py rename to tpot/operators_disable/preprocessors/zero_count.py diff --git a/tpot/operators/regressors/__init__.py b/tpot/operators_disable/regressors/__init__.py similarity index 100% rename from tpot/operators/regressors/__init__.py rename to tpot/operators_disable/regressors/__init__.py diff --git a/tpot/operators/regressors/base.py b/tpot/operators_disable/regressors/base.py similarity index 100% rename from tpot/operators/regressors/base.py rename to tpot/operators_disable/regressors/base.py diff --git a/tpot/operators/regressors/elastic_net.py b/tpot/operators_disable/regressors/elastic_net.py similarity index 100% rename from tpot/operators/regressors/elastic_net.py rename to tpot/operators_disable/regressors/elastic_net.py diff --git a/tpot/operators/regressors/extra_trees.py b/tpot/operators_disable/regressors/extra_trees.py similarity index 100% rename from tpot/operators/regressors/extra_trees.py rename to tpot/operators_disable/regressors/extra_trees.py diff --git a/tpot/operators/regressors/gradient_boosting.py b/tpot/operators_disable/regressors/gradient_boosting.py similarity index 100% rename from tpot/operators/regressors/gradient_boosting.py rename to tpot/operators_disable/regressors/gradient_boosting.py diff --git a/tpot/operators/regressors/knnr.py b/tpot/operators_disable/regressors/knnr.py similarity index 100% rename from tpot/operators/regressors/knnr.py rename to tpot/operators_disable/regressors/knnr.py diff --git a/tpot/operators/regressors/lasso_lars_cv.py b/tpot/operators_disable/regressors/lasso_lars_cv.py similarity index 100% rename from tpot/operators/regressors/lasso_lars_cv.py rename to tpot/operators_disable/regressors/lasso_lars_cv.py diff --git a/tpot/operators/regressors/linear_svr.py b/tpot/operators_disable/regressors/linear_svr.py similarity index 100% rename from tpot/operators/regressors/linear_svr.py rename to tpot/operators_disable/regressors/linear_svr.py diff --git a/tpot/operators/regressors/random_forest.py b/tpot/operators_disable/regressors/random_forest.py similarity index 100% rename from tpot/operators/regressors/random_forest.py rename to tpot/operators_disable/regressors/random_forest.py diff --git a/tpot/operators/regressors/xg_boost_r.py b/tpot/operators_disable/regressors/xg_boost_r.py similarity index 100% rename from tpot/operators/regressors/xg_boost_r.py rename to tpot/operators_disable/regressors/xg_boost_r.py diff --git a/tpot/operators/selectors/__init__.py b/tpot/operators_disable/selectors/__init__.py similarity index 100% rename from tpot/operators/selectors/__init__.py rename to tpot/operators_disable/selectors/__init__.py diff --git a/tpot/operators/selectors/base.py b/tpot/operators_disable/selectors/base.py similarity index 100% rename from tpot/operators/selectors/base.py rename to tpot/operators_disable/selectors/base.py diff --git a/tpot/operators/selectors/rfe.py b/tpot/operators_disable/selectors/rfe.py similarity index 100% rename from tpot/operators/selectors/rfe.py rename to tpot/operators_disable/selectors/rfe.py diff --git a/tpot/operators/selectors/select_from_model.py b/tpot/operators_disable/selectors/select_from_model.py similarity index 100% rename from tpot/operators/selectors/select_from_model.py rename to tpot/operators_disable/selectors/select_from_model.py diff --git a/tpot/operators/selectors/select_from_model_r.py b/tpot/operators_disable/selectors/select_from_model_r.py similarity index 100% rename from tpot/operators/selectors/select_from_model_r.py rename to tpot/operators_disable/selectors/select_from_model_r.py diff --git a/tpot/operators/selectors/select_fwe.py b/tpot/operators_disable/selectors/select_fwe.py similarity index 100% rename from tpot/operators/selectors/select_fwe.py rename to tpot/operators_disable/selectors/select_fwe.py diff --git a/tpot/operators/selectors/select_kbest.py b/tpot/operators_disable/selectors/select_kbest.py similarity index 100% rename from tpot/operators/selectors/select_kbest.py rename to tpot/operators_disable/selectors/select_kbest.py diff --git a/tpot/operators/selectors/select_percentile.py b/tpot/operators_disable/selectors/select_percentile.py similarity index 100% rename from tpot/operators/selectors/select_percentile.py rename to tpot/operators_disable/selectors/select_percentile.py diff --git a/tpot/operators/selectors/variance_threshold.py b/tpot/operators_disable/selectors/variance_threshold.py similarity index 100% rename from tpot/operators/selectors/variance_threshold.py rename to tpot/operators_disable/selectors/variance_threshold.py diff --git a/tpot/tpot.py b/tpot/tpot.py index b5e030cc..4215151a 100644 --- a/tpot/tpot.py +++ b/tpot/tpot.py @@ -19,12 +19,15 @@ """ from .base import TPOTBase +from .config_classifier import classifier_config_dict +#from config_regressor import regressor_config_dic class TPOTClassifier(TPOTBase): """TPOT estimator for classification problems""" scoring_function = 'balanced_accuracy' # Classification scoring + operator_dict = classifier_config_dict def _ignore_operator(self, op): """Filter that describes which operators are not used diff --git a/tpot_test_config_dict.py b/tpot_test_config_dict.py new file mode 100644 index 00000000..c90dbd6e --- /dev/null +++ b/tpot_test_config_dict.py @@ -0,0 +1,16 @@ +from tpot import TPOTClassifier +from sklearn.datasets import make_classification +from sklearn.model_selection import train_test_split +import time + +X, y = make_classification(n_samples=100, n_features=50, + n_informative=2, n_redundant=10, random_state=42) +X_train, X_test, y_train, y_test = train_test_split(X, y, + train_size=0.75, test_size=0.25) + + +tpot = TPOTClassifier(generations=2, population_size=2, verbosity=2, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime usages:',time.time()-time_start) From ea26d66fe2f2464857bc2ffa6cf24c1438a91837 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 27 Jan 2017 15:12:36 -0500 Subject: [PATCH 050/154] new decorator works --- test_op_class.py | 26 +++++++++++++++++++++++ tests.py | 4 ++-- tpot/base.py | 24 +++++++++------------- tpot/decorators.py | 43 +++++++++++++++++++++++++++++++++++++++ tpot/export_utils.py | 17 ++++++++-------- tpot/operator_utils.py | 32 +---------------------------- tpot_export_class_conf.py | 22 ++++++++++++++++++++ tpot_test_config_dict.py | 7 ++++--- 8 files changed, 116 insertions(+), 59 deletions(-) create mode 100644 tpot_export_class_conf.py diff --git a/test_op_class.py b/test_op_class.py index 93400d11..2f9909b1 100644 --- a/test_op_class.py +++ b/test_op_class.py @@ -17,3 +17,29 @@ class TPOTBase(BaseEstimator): t = TPOTBase t() + +from sklearn.pipeline import make_pipeline, make_union +from sklearn.preprocessing import FunctionTransformer +from sklearn.ensemble import VotingClassifier +from sklearn.svm import LinearSVC +from sklearn.cluster import FeatureAgglomeration +from sklearn.ensemble import RandomForestClassifier +from sklearn.feature_selection import VarianceThreshold, SelectFromModel +from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer, MaxAbsScaler +from sklearn.naive_bayes import MultinomialNB +from sklearn.linear_model import LogisticRegression +from sklearn.datasets import make_classification +from sklearn.model_selection import train_test_split +X, y = make_classification(n_samples=200, n_features=50, + n_informative=10, n_redundant=10, random_state=42) + +clr2 = make_pipeline( + MinMaxScaler(), + Normalizer(norm="l1"), + StandardScaler(), + MaxAbsScaler(), + SelectFromModel(estimator=ExtraTreesClassifier(criterion="entropy", max_features=0.1), threshold=0.9500000000000001), + LogisticRegression(C=0.0001, dual=True, penalty="l2") +) + +clr2.fit(X,y) diff --git a/tests.py b/tests.py index e35769da..8fdbbb30 100644 --- a/tests.py +++ b/tests.py @@ -11,8 +11,8 @@ from tpot.gp_types import Output_DF from tpot.gp_deap import mutNodeReplacement -from tpot.operators import Operator -from tpot.operators.selectors import TPOTSelectKBest +from tpot.operator_utils import Operator, +#from tpot.operators.selectors import TPOTSelectKBest import numpy as np import inspect diff --git a/tpot/base.py b/tpot/base.py index 371471aa..ed8f1b79 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -44,7 +44,7 @@ from ._version import __version__ from .operator_utils import TPOTOperatorClassFactory from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code -from .decorators import _timeout +from .decorators import _timeout, _pre_test from .operator_utils import operators, argument_types from .build_in_operators import CombineDFs from .gp_types import Bool, Output_DF @@ -189,11 +189,10 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.arguments = [] for key in sorted(self.operator_dict.keys()): - print('Creating: {}'.format(key)) op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key], classification=True) self.operators.append(op_class) self.arguments += arg_types - print(self.operators) + # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out @@ -216,6 +215,9 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, 'FunctionTransformer': FunctionTransformer } + if self.verbosity > 1: + print('{} operators are imported.'.format(len(self.operators))) + self._pbar = None # a dictionary of individual which has already evaluated in previous generation. @@ -271,7 +273,6 @@ def _setup_pset(self): # We need to add rooted primitives twice so that they can # return both an Output_DF (and thus be the root of the tree), # and return a np.ndarray so they can exist elsewhere in the tree. - print(op.__name__) p_types = (op.parameter_types()[0], Output_DF) self._pset.addPrimitive(op, *p_types) @@ -307,9 +308,9 @@ def _setup_toolbox(self): self._toolbox.register('population', tools.initRepeat, list, self._toolbox.individual) self._toolbox.register('compile', self._compile_to_sklearn) self._toolbox.register('select', tools.selNSGA2) - self._toolbox.register('mate', gp.cxOnePoint) + self._toolbox.register('mate', _pre_test(gp.cxOnePoint)) self._toolbox.register('expr_mut', self._gen_grow_safe, min_=1, max_=4) - self._toolbox.register('mutate', self._random_mutation_operator) + self._toolbox.register('mutate', _pre_test(self._random_mutation_operator)) def fit(self, features, classes, sample_weight=None): """Fits a machine learning pipeline that maximizes classification score @@ -571,7 +572,6 @@ def _compile_to_sklearn(self, expr): sklearn_pipeline: sklearn.pipeline.Pipeline """ sklearn_pipeline = generate_pipeline_code(expr_to_tree(expr), self.operators) - return eval(sklearn_pipeline, self.operators_context) def _set_param_recursive(self, pipeline_steps, parameter, value, sample_weight = None): @@ -644,11 +644,9 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): raise ValueError('Invalid pipeline -- skipping its evaluation') - print(individual_str) # Transform the tree expression into an sklearn pipeline sklearn_pipeline = self._toolbox.compile(expr=individual) - # Fix random state when the operator allows and build sample weight dictionary sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) @@ -676,16 +674,13 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No cv_scores = cross_val_score(self, sklearn_pipeline, features, classes, cv=self.cv, scoring=self.scoring_function, n_jobs=self.n_jobs, fit_params=sample_weight_dict) - try: resulting_score = np.mean(cv_scores) - except TypeError: - raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') except Exception: # Catch-all: Do not allow one pipeline that crashes to cause TPOT # to crash. Instead, assign the crashing pipeline a poor fitness - import traceback - traceback.print_exc() + #import traceback + #traceback.print_exc() return 5000., -float('inf') finally: if not self._pbar.disable: @@ -755,6 +750,7 @@ def condition(height, depth, type_): return self._generate(pset, min_, max_, condition, type_) # Generate function stolen straight from deap.gp.generate + @_pre_test def _generate(self, pset, min_, max_, condition, type_=None): """Generate a Tree as a list of list. The tree is build from the root to the leaves, and it stop growing when the condition is fulfilled. diff --git a/tpot/decorators.py b/tpot/decorators.py index 75b659e5..596fda3d 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -21,6 +21,12 @@ from functools import wraps import sys +import warnings +from sklearn.datasets import make_classification +from .export_utils import expr_to_tree, generate_pipeline_code +# generate a small data set for a new pipeline, in order to check if the pipeline +# has unsuppported combinations in params +pretest_X, pretest_y = make_classification(n_samples=50, n_features=10, random_state=42) def _timeout(func): """Runs a function with time limit @@ -124,3 +130,40 @@ def limitedTime(self, *args, **kw): tmp_it.stop() # return func return limitedTime + +def _pre_test(func): + """Decorator that wraps functions to check if the pipeline works with a pretest data set + If not, then rerun the func until it generates a good pipeline + + Parameters + ---------- + func: function + The function being decorated + + Returns + ------- + wrapped_func: function + A wrapper function around the func parameter + """ + @wraps(func) + def check_pipeline(self, *args, **kwargs): + bad_pipeline = True + num_test = 0 # number of tests + """with warnings.catch_warnings(): + warnings.simplefilter('ignore')""" + while bad_pipeline and num_test < 10: # a pool for workable pipeline + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + expr = func(self, *args, **kwargs) + #debug use + #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) + sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(expr), self.operators), self.operators_context) + sklearn_pipeline.fit(pretest_X, pretest_y) + bad_pipeline = False + except: + pass + finally: + num_test += 1 + return expr + return check_pipeline diff --git a/tpot/export_utils.py b/tpot/export_utils.py index 5d33fd61..bf996fb8 100644 --- a/tpot/export_utils.py +++ b/tpot/export_utils.py @@ -36,12 +36,13 @@ def get_by_name(opname, operators): An operator class """ - ret_op_class = [op for op in operators if op.__name__ == opname] - if len(ret_op_class) == 0: + ret_op_classes = [op for op in operators if op.__name__ == opname] + if len(ret_op_classes) == 0: raise TypeError('Cannot found operator {} in operator dictionary'.format(opname)) - elif len(ret_op_class) > 1: + elif len(ret_op_classes) > 1: print('Found multiple operator {} in operator dictionary'.format(opname), - 'Please check your dictionary file') + 'Please check your dictionary file.') + ret_op_class = ret_op_classes[0] return ret_op_class def export_pipeline(exported_pipeline, operators): @@ -67,7 +68,7 @@ def export_pipeline(exported_pipeline, operators): pipeline_text = generate_import_code(exported_pipeline, operators) # Replace the function calls with their corresponding Python code - pipeline_text += pipeline_code_wrapper(generate_export_pipeline_code(pipeline_tree)) + pipeline_text += pipeline_code_wrapper(generate_export_pipeline_code(pipeline_tree, operators)) return pipeline_text @@ -263,12 +264,11 @@ def generate_export_pipeline_code(pipeline_tree, operators): pipeline_text = "make_pipeline(\n{STEPS}\n)".format(STEPS=_indent(",\n".join(steps), 4)) else: # only one operator (root = True) pipeline_text = "{STEPS}".format(STEPS=_indent(",\n".join(steps), 0)) - print(pipeline_text) return pipeline_text -def process_operator(operator, depth=0, operators): +def process_operator(operator, operators, depth=0): steps = [] op_name = operator[0] @@ -281,7 +281,7 @@ def process_operator(operator, depth=0, operators): tpot_op = get_by_name(op_name, operators) if input_name != 'input_matrix': - steps.extend(process_operator(input_name, depth + 1, operators)) + steps.extend(process_operator(input_name, operators, depth + 1)) # If the step is an estimator and is not the last step then we must # add its guess as a synthetic feature @@ -292,7 +292,6 @@ def process_operator(operator, depth=0, operators): ) else: steps.append(tpot_op.export(*args)) - print(steps) return steps diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 6823acb7..e191782c 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -224,36 +224,6 @@ def export(*args): class_profile['export'] = export - op_classname = '{}__{}'.format('TPOT',op_str) + op_classname = '{}_{}'.format('TPOT',op_str) op_class = type(op_classname, (BaseClass,), class_profile) return op_class, arg_types - - - - - -""" -Test -op_class_dict={} - -for key, val in classifier_config_dict.items(): - print('Config: {}'.format(key)) - op_class_dict[key]=TPOTOperatorClassFactory(key, val, classification=True) - print(op_class_dict[key].sklearn_class.__name__) - print(op_class_dict[key].import_hash) - print(op_class_dict[key].arg_types) -a = op_class_dict['sklearn.naive_bayes.MultinomialNB'] -c = op_class_dict['sklearn.feature_selection.SelectFromModel'] -d = op_class_dict['sklearn.feature_selection.SelectFwe'] - - - - - -for op in Operator.inheritors(): - print(op.sklearn_class.__name__) - -for arg in ARGType.inheritors(): - print(arg.__name__, arg.values) - -""" diff --git a/tpot_export_class_conf.py b/tpot_export_class_conf.py new file mode 100644 index 00000000..2cd39c4b --- /dev/null +++ b/tpot_export_class_conf.py @@ -0,0 +1,22 @@ +import numpy as np + +from sklearn.ensemble import VotingClassifier +from sklearn.model_selection import train_test_split +from sklearn.naive_bayes import BernoulliNB +from sklearn.neighbors import KNeighborsClassifier +from sklearn.pipeline import make_pipeline, make_union +from sklearn.preprocessing import FunctionTransformer + +# NOTE: Make sure that the class is labeled 'class' in the data file +tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) +features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) +training_features, testing_features, training_classes, testing_classes = \ + train_test_split(features, tpot_data['class'], random_state=42) + +exported_pipeline = make_pipeline( + make_union(VotingClassifier([("est", BernoulliNB(alpha=10.0, fit_prior=False))]), FunctionTransformer(lambda X: X)), + KNeighborsClassifier(n_neighbors=10, p=1, weights="uniform") +) + +exported_pipeline.fit(training_features, training_classes) +results = exported_pipeline.predict(testing_features) diff --git a/tpot_test_config_dict.py b/tpot_test_config_dict.py index c90dbd6e..3bae715f 100644 --- a/tpot_test_config_dict.py +++ b/tpot_test_config_dict.py @@ -3,14 +3,15 @@ from sklearn.model_selection import train_test_split import time -X, y = make_classification(n_samples=100, n_features=50, - n_informative=2, n_redundant=10, random_state=42) +X, y = make_classification(n_samples=200, n_features=50, + n_informative=10, n_redundant=10, random_state=42) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, test_size=0.25) -tpot = TPOTClassifier(generations=2, population_size=2, verbosity=2, random_state = 42) +tpot = TPOTClassifier(generations=4, population_size=20, verbosity=2, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) +tpot.export('tpot_export_class_conf.py') print('\nTime usages:',time.time()-time_start) From b0290a6db2966588f451de59bbd91ad3a188f893 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 27 Jan 2017 17:19:45 -0500 Subject: [PATCH 051/154] regressor works --- test_op_class.py | 45 ---- tests.py | 38 +-- tpot/base.py | 3 +- tpot/config_preprocessor.py | 132 ---------- tpot/config_regressor.py | 227 +++++++++++------- tpot/config_selector.py | 100 -------- tpot/decorators.py | 3 +- tpot/operator_utils.py | 2 +- tpot/operators_disable/__init__.py | 26 -- tpot/operators_disable/base.py | 155 ------------ .../operators_disable/classifiers/__init__.py | 35 --- tpot/operators_disable/classifiers/base.py | 29 --- .../classifiers/bernoulli_nb.py | 48 ---- .../classifiers/decision_tree.py | 41 ---- .../classifiers/extra_trees.py | 55 ----- .../classifiers/gaussian_nb.py | 41 ---- .../classifiers/gradient_boosting.py | 51 ---- tpot/operators_disable/classifiers/knnc.py | 52 ---- .../classifiers/linear_svc.py | 59 ----- .../classifiers/logistic_regression.py | 61 ----- .../classifiers/multinomial_nb.py | 47 ---- .../classifiers/random_forest.py | 45 ---- .../operators_disable/classifiers/xg_boost.py | 58 ----- tpot/operators_disable/combine_dfs.py | 27 --- .../preprocessors/__init__.py | 34 --- tpot/operators_disable/preprocessors/base.py | 29 --- .../preprocessors/binarizer.py | 44 ---- .../preprocessors/fast_ica.py | 46 ---- .../preprocessors/feat_agg.py | 58 ----- .../preprocessors/max_abs_scalar.py | 42 ---- .../preprocessors/min_max_scalar.py | 42 ---- .../preprocessors/normalizer.py | 47 ---- .../preprocessors/nystroem.py | 59 ----- tpot/operators_disable/preprocessors/pca.py | 47 ---- .../preprocessors/polynomial_features.py | 45 ---- tpot/operators_disable/preprocessors/rbf.py | 44 ---- .../preprocessors/robust_scaler.py | 43 ---- .../preprocessors/standard_scaler.py | 43 ---- .../preprocessors/zero_count.py | 85 ------- tpot/operators_disable/regressors/__init__.py | 32 --- tpot/operators_disable/regressors/base.py | 29 --- .../regressors/elastic_net.py | 50 ---- .../regressors/extra_trees.py | 50 ---- .../regressors/gradient_boosting.py | 51 ---- tpot/operators_disable/regressors/knnr.py | 52 ---- .../regressors/lasso_lars_cv.py | 45 ---- .../regressors/linear_svr.py | 50 ---- .../regressors/random_forest.py | 45 ---- .../regressors/xg_boost_r.py | 58 ----- tpot/operators_disable/selectors/__init__.py | 30 --- tpot/operators_disable/selectors/base.py | 29 --- tpot/operators_disable/selectors/rfe.py | 49 ---- .../selectors/select_from_model.py | 67 ------ .../selectors/select_from_model_r.py | 59 ----- .../operators_disable/selectors/select_fwe.py | 47 ---- .../selectors/select_kbest.py | 47 ---- .../selectors/select_percentile.py | 48 ---- .../selectors/variance_threshold.py | 44 ---- tpot/tpot.py | 9 +- tpot_export_class_conf.py | 10 +- tpot_export_reg_conf.py | 15 ++ tpot_test_config_dict.py | 19 +- 62 files changed, 206 insertions(+), 2817 deletions(-) delete mode 100644 test_op_class.py delete mode 100644 tpot/config_preprocessor.py delete mode 100644 tpot/config_selector.py delete mode 100644 tpot/operators_disable/__init__.py delete mode 100644 tpot/operators_disable/base.py delete mode 100644 tpot/operators_disable/classifiers/__init__.py delete mode 100644 tpot/operators_disable/classifiers/base.py delete mode 100644 tpot/operators_disable/classifiers/bernoulli_nb.py delete mode 100644 tpot/operators_disable/classifiers/decision_tree.py delete mode 100644 tpot/operators_disable/classifiers/extra_trees.py delete mode 100644 tpot/operators_disable/classifiers/gaussian_nb.py delete mode 100644 tpot/operators_disable/classifiers/gradient_boosting.py delete mode 100644 tpot/operators_disable/classifiers/knnc.py delete mode 100644 tpot/operators_disable/classifiers/linear_svc.py delete mode 100644 tpot/operators_disable/classifiers/logistic_regression.py delete mode 100644 tpot/operators_disable/classifiers/multinomial_nb.py delete mode 100644 tpot/operators_disable/classifiers/random_forest.py delete mode 100644 tpot/operators_disable/classifiers/xg_boost.py delete mode 100644 tpot/operators_disable/combine_dfs.py delete mode 100644 tpot/operators_disable/preprocessors/__init__.py delete mode 100644 tpot/operators_disable/preprocessors/base.py delete mode 100644 tpot/operators_disable/preprocessors/binarizer.py delete mode 100644 tpot/operators_disable/preprocessors/fast_ica.py delete mode 100644 tpot/operators_disable/preprocessors/feat_agg.py delete mode 100644 tpot/operators_disable/preprocessors/max_abs_scalar.py delete mode 100644 tpot/operators_disable/preprocessors/min_max_scalar.py delete mode 100644 tpot/operators_disable/preprocessors/normalizer.py delete mode 100644 tpot/operators_disable/preprocessors/nystroem.py delete mode 100644 tpot/operators_disable/preprocessors/pca.py delete mode 100644 tpot/operators_disable/preprocessors/polynomial_features.py delete mode 100644 tpot/operators_disable/preprocessors/rbf.py delete mode 100644 tpot/operators_disable/preprocessors/robust_scaler.py delete mode 100644 tpot/operators_disable/preprocessors/standard_scaler.py delete mode 100644 tpot/operators_disable/preprocessors/zero_count.py delete mode 100644 tpot/operators_disable/regressors/__init__.py delete mode 100644 tpot/operators_disable/regressors/base.py delete mode 100644 tpot/operators_disable/regressors/elastic_net.py delete mode 100644 tpot/operators_disable/regressors/extra_trees.py delete mode 100644 tpot/operators_disable/regressors/gradient_boosting.py delete mode 100644 tpot/operators_disable/regressors/knnr.py delete mode 100644 tpot/operators_disable/regressors/lasso_lars_cv.py delete mode 100644 tpot/operators_disable/regressors/linear_svr.py delete mode 100644 tpot/operators_disable/regressors/random_forest.py delete mode 100644 tpot/operators_disable/regressors/xg_boost_r.py delete mode 100644 tpot/operators_disable/selectors/__init__.py delete mode 100644 tpot/operators_disable/selectors/base.py delete mode 100644 tpot/operators_disable/selectors/rfe.py delete mode 100644 tpot/operators_disable/selectors/select_from_model.py delete mode 100644 tpot/operators_disable/selectors/select_from_model_r.py delete mode 100644 tpot/operators_disable/selectors/select_fwe.py delete mode 100644 tpot/operators_disable/selectors/select_kbest.py delete mode 100644 tpot/operators_disable/selectors/select_percentile.py delete mode 100644 tpot/operators_disable/selectors/variance_threshold.py create mode 100644 tpot_export_reg_conf.py diff --git a/test_op_class.py b/test_op_class.py deleted file mode 100644 index 2f9909b1..00000000 --- a/test_op_class.py +++ /dev/null @@ -1,45 +0,0 @@ -# coding: utf-8 -from tpot.operator_utils import ARGType, TPOTOperatorClassFactory, Operator -from tpot.config_classifier import classifier_config_dict -from sklearn.base import BaseEstimator - - -class TPOTBase(BaseEstimator): - """TPOT automatically creates and optimizes machine learning pipelines using genetic programming""" - operator_dict = classifier_config_dict - ops = [] - arglist = [] - for key in sorted(operator_dict.keys()): - print('Creating: {}'.format(key)) - op_class, arg_types = TPOTOperatorClassFactory(key, operator_dict[key], classification=True) - ops.append(op_class) - arglist += arg_types - -t = TPOTBase -t() - -from sklearn.pipeline import make_pipeline, make_union -from sklearn.preprocessing import FunctionTransformer -from sklearn.ensemble import VotingClassifier -from sklearn.svm import LinearSVC -from sklearn.cluster import FeatureAgglomeration -from sklearn.ensemble import RandomForestClassifier -from sklearn.feature_selection import VarianceThreshold, SelectFromModel -from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer, MaxAbsScaler -from sklearn.naive_bayes import MultinomialNB -from sklearn.linear_model import LogisticRegression -from sklearn.datasets import make_classification -from sklearn.model_selection import train_test_split -X, y = make_classification(n_samples=200, n_features=50, - n_informative=10, n_redundant=10, random_state=42) - -clr2 = make_pipeline( - MinMaxScaler(), - Normalizer(norm="l1"), - StandardScaler(), - MaxAbsScaler(), - SelectFromModel(estimator=ExtraTreesClassifier(criterion="entropy", max_features=0.1), threshold=0.9500000000000001), - LogisticRegression(C=0.0001, dual=True, penalty="l2") -) - -clr2.fit(X,y) diff --git a/tests.py b/tests.py index 8fdbbb30..57ecdf36 100644 --- a/tests.py +++ b/tests.py @@ -11,8 +11,9 @@ from tpot.gp_types import Output_DF from tpot.gp_deap import mutNodeReplacement -from tpot.operator_utils import Operator, -#from tpot.operators.selectors import TPOTSelectKBest +from tpot.operator_utils import Operator, TPOTOperatorClassFactory +from tpot.config_classifier import classifier_config_dict + import numpy as np import inspect @@ -21,6 +22,7 @@ from sklearn.datasets import load_digits, load_boston from sklearn.model_selection import train_test_split +from sklearn.ensemble import RandomForestClassifier from deap import creator from tqdm import tqdm @@ -38,6 +40,10 @@ np.random.seed(42) random.seed(42) +test_operator_key = 'sklearn.feature_selection.SelectKBest' +TPOTSelectKBest = TPOTOperatorClassFactory(test_operator_key, + classifier_config_dict[test_operator_key]) + def test_init_custom_parameters(): """Assert that the TPOT instantiator stores the TPOT variables properly""" @@ -130,23 +136,23 @@ def test_score(): def test_score_2(): """Assert that the TPOTClassifier score function outputs a known score for a fixed pipeline""" - tpot_obj = TPOTClassifier() - tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 0.986318199045 # Assumes use of the TPOT balanced_accuracy function +tpot_obj = TPOTClassifier() +tpot_obj._pbar = tqdm(total=1, disable=True) +known_score = 0.986318199045 # Assumes use of the TPOT balanced_accuracy function - # Reify pipeline with known score - tpot_obj._optimized_pipeline = creator.Individual.\ - from_string('RandomForestClassifier(input_matrix)', tpot_obj._pset) - tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) - tpot_obj._fitted_pipeline.fit(training_features, training_classes) +# Reify pipeline with known score +tpot_obj._optimized_pipeline = creator.Individual.\ + from_string("RandomForestClassifier(True, \'gini\', \'auto\', 1, 2, input_matrix)", tpot_obj._pset) +tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) +tpot_obj._fitted_pipeline.fit(training_features, training_classes) - # Get score from TPOT - score = tpot_obj.score(testing_features, testing_classes) - # http://stackoverflow.com/questions/5595425/ - def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): - return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) +# Get score from TPOT +score = tpot_obj.score(testing_features, testing_classes) +# http://stackoverflow.com/questions/5595425/ +def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) - assert isclose(known_score, score) +assert isclose(known_score, score) def test_score_3(): diff --git a/tpot/base.py b/tpot/base.py index ed8f1b79..8e4ea00c 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -189,7 +189,8 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.arguments = [] for key in sorted(self.operator_dict.keys()): - op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key], classification=True) + op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key], + classification = self.classification, regression = self.regression) self.operators.append(op_class) self.arguments += arg_types diff --git a/tpot/config_preprocessor.py b/tpot/config_preprocessor.py deleted file mode 100644 index 89b4c655..00000000 --- a/tpot/config_preprocessor.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -""" -dictionary format (json-like format): -key: - operator name -value: - source: module source (e.g sklearn.tree) - dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency - params: a dictionary of parameter names (keys) and parameter ranges (values); None for no params -""" -import numpy as np - -preprocessor_config_dict = { - - 'Binarizer': { - 'source': 'sklearn.preprocessing', - 'dependencies': None, - 'params':{ - 'threshold': np.arange(0.0, 1.01, 0.05) - } - }, - - 'FastICA': { - 'source': 'sklearn.decomposition', - 'dependencies': None, - 'params':{ - 'tol': np.arange(0.0, 1.01, 0.05) - } - }, - - 'FeatureAgglomeration': { - 'source': 'sklearn.cluster', - 'dependencies': None, - 'params':{ - 'linkage': ['ward', 'complete', 'average'], - 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] - } - }, - - 'MaxAbsScaler': { - 'source': 'sklearn.preprocessing', - 'dependencies': None, - 'params': None - }, - - 'MinMaxScaler': { - 'source': 'sklearn.preprocessing', - 'dependencies': None, - 'params': None - }, - - 'Normalizer': { - 'source': 'sklearn.preprocessing', - 'dependencies': None, - 'params': { - 'norm': ['l1', 'l2', 'max'] - } - }, - - 'Nystroem': { - 'source': 'sklearn.kernel_approximation', - 'dependencies': None, - 'params': { - 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], - 'gamma': np.arange(0.0, 1.01, 0.05), - 'n_components': range(1, 11) - } - }, - - 'PCA': { - 'source': 'sklearn.decomposition', - 'dependencies': None, - 'params': { - 'svd_solver': ['randomized'], - 'iterated_power': range(1, 11) - } - }, - - 'PolynomialFeatures': { - 'source': 'sklearn.preprocessing', - 'dependencies': None, - 'params': { - 'degree': [2], - 'include_bias': [False], - 'interaction_only': [False] - } - }, - - 'RBFSampler': { - 'source': 'sklearn.kernel_approximation', - 'dependencies': None, - 'params': { - 'gamma': np.arange(0.0, 1.01, 0.05) - } - }, - - 'RobustScaler': { - 'source': 'sklearn.preprocessing', - 'dependencies': None, - 'params': None - }, - - 'StandardScaler': { - 'source': 'sklearn.preprocessing', - 'dependencies': None, - 'params': None - }, - - 'ZeroCount': { - 'source': 'tpot.build_in_operators', - 'dependencies': None, - 'params': None - } - -} diff --git a/tpot/config_regressor.py b/tpot/config_regressor.py index 14790310..534a7355 100644 --- a/tpot/config_regressor.py +++ b/tpot/config_regressor.py @@ -30,119 +30,168 @@ regressor_config_dict = { - 'ElasticNetCV': { - 'source': 'sklearn.linear_model', - 'dependencies': None, - 'params':{ - 'l1_ratio': np.arange(0.0, 1.01, 0.05), - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1] - } + 'sklearn.linear_model.ElasticNetCV': { + 'l1_ratio': np.arange(0.0, 1.01, 0.05), + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1] }, - 'ExtraTreesRegressor': { - 'source': 'sklearn.ensemble', - 'dependencies': None, - 'params':{ - 'max_features': np.arange(0, 1.01, 0.05), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'bootstrap': [True, False] - } + 'sklearn.ensemble.ExtraTreesRegressor': { + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] }, - 'GradientBoostingRegressor': { - 'source': 'sklearn.ensemble', - 'dependencies': None, - 'params':{ - 'loss': ["ls", "lad", "huber", "quantile"], - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'max_depth': range(1, 11), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'subsample': np.arange(0.05, 1.01, 0.05), - 'max_features': np.arange(0, 1.01, 0.05), - 'alpha': [0.75, 0.8, 0.85, 0.9, 0.95, 0.99] - } + 'sklearn.ensemble.GradientBoostingRegressor': { + 'loss': ["ls", "lad", "huber", "quantile"], + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'subsample': np.arange(0.05, 1.01, 0.05), + 'max_features': np.arange(0, 1.01, 0.05), + 'alpha': [0.75, 0.8, 0.85, 0.9, 0.95, 0.99] }, - 'AdaBoostRegressor': { - 'source': 'sklearn.ensemble', - 'dependencies': None, - 'params':{ - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'loss': ["linear", "square", "exponential"], - 'max_depth': range(1, 11) - } + 'sklearn.ensemble.AdaBoostRegressor': { + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'loss': ["linear", "square", "exponential"], + 'max_depth': range(1, 11) }, - 'DecisionTreeRegressor': { - 'source': 'sklearn.tree', - 'dependencies': None, - 'params':{ - 'max_depth': range(1, 11), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21) - } + 'sklearn.tree.DecisionTreeRegressor': { + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21) }, - 'KNeighborsRegressor': { - 'source': 'sklearn.neighbors', - 'dependencies': None, - 'params':{ - 'n_neighbors': range(1, 101), - 'weights': ["uniform", "distance"], - 'p': [1, 2] - } + 'sklearn.neighbors.KNeighborsRegressor': { + 'n_neighbors': range(1, 101), + 'weights': ["uniform", "distance"], + 'p': [1, 2] }, - 'LassoLarsCV': { - 'source': 'sklearn.linear_model', - 'dependencies': None, - 'params':{ - 'normalize': [True, False] - } + 'sklearn.linear_model.LassoLarsCV': { + 'normalize': [True, False] + }, + + 'sklearn.svm.LinearSVR': { + 'loss': ["epsilon_insensitive", "squared_epsilon_insensitive"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'epsilon': [1e-4, 1e-3, 1e-2, 1e-1, 1.] + }, + + 'sklearn.ensemble.RandomForestRegressor': { + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] + }, + + 'sklearn.linear_model.RidgeCV': { + }, + + + 'xgboost.XGBRegressor': { + 'max_depth': range(1, 11), + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'subsample': np.arange(0.05, 1.01, 0.05), + 'min_child_weight': range(1, 21) + }, + + # Preprocesssors + 'sklearn.preprocessing.Binarizer': { + 'threshold': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.decomposition.FastICA': { + 'tol': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.cluster.FeatureAgglomeration': { + 'linkage': ['ward', 'complete', 'average'], + 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] + }, + + 'sklearn.preprocessing.MaxAbsScaler': { }, - 'LinearSVR': { - 'source': 'sklearn.svm', - 'dependencies': None, - 'params':{ - 'loss': ["epsilon_insensitive", "squared_epsilon_insensitive"], - 'dual': [True, False], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], - 'epsilon': [1e-4, 1e-3, 1e-2, 1e-1, 1.] + 'sklearn.preprocessing.MinMaxScaler': { + }, + + 'sklearn.preprocessing.Normalizer': { + 'norm': ['l1', 'l2', 'max'] + }, + + 'sklearn.kernel_approximation.Nystroem': { + 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], + 'gamma': np.arange(0.0, 1.01, 0.05), + 'n_components': range(1, 11) + }, + + 'sklearn.decomposition.PCA': { + 'svd_solver': ['randomized'], + 'iterated_power': range(1, 11) + }, + + 'sklearn.preprocessing.PolynomialFeatures': { + 'degree': [2], + 'include_bias': [False], + 'interaction_only': [False] + }, + + 'sklearn.kernel_approximation.RBFSampler': { + 'gamma': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.preprocessing.RobustScaler': { + }, + + 'sklearn.preprocessing.StandardScaler': { + }, + + 'tpot.build_in_operators.ZeroCount': { + }, + + # Selectors + 'sklearn.feature_selection.SelectFwe': { + 'alpha': np.arange(0, 0.05, 0.001), + 'score_func': { + 'sklearn.feature_selection.f_classif': None + } # read from dependencies ! need add an exception in preprocess_args + + }, + + 'sklearn.feature_selection.SelectKBest': { + 'k': range(1, 100), # need check range! + 'score_func': { + 'sklearn.feature_selection.f_classif': None } }, - 'RandomForestRegressor': { - 'source': 'sklearn.ensemble', - 'dependencies': None, - 'params':{ - 'max_features': np.arange(0, 1.01, 0.05), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'bootstrap': [True, False] + 'sklearn.feature_selection.SelectPercentile': { + 'percentile': range(1, 100), + 'score_func': { + 'sklearn.feature_selection.f_classif': None } }, - 'RidgeCV': { - 'source': 'sklearn.linear_model', - 'dependencies': None, - 'params': None + 'sklearn.feature_selection.VarianceThreshold': { + 'threshold': np.arange(0.05, 1.01, 0.05) }, + 'sklearn.feature_selection.SelectFromModel': { + 'threshold': np.arange(0, 1.01, 0.05), + 'estimator': { + 'sklearn.ensemble.ExtraTreesRegressor': { + 'max_features': np.arange(0, 1.01, 0.05) + } + } - 'XGBRegressor': { - 'source': 'xgboost', - 'dependencies': None, - 'params':{ - 'max_depth': range(1, 11), - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'subsample': np.arange(0.05, 1.01, 0.05), - 'min_child_weight': range(1, 21) - } } + } diff --git a/tpot/config_selector.py b/tpot/config_selector.py deleted file mode 100644 index 842b7b7f..00000000 --- a/tpot/config_selector.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -""" -dictionary format (json-like format): -key: - unique operator name -value: - source: module source (e.g sklearn.tree) - dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency - params: a dictionary of parameter names (keys) and parameter ranges (values); None for no params -""" -import numpy as np - -selector_config_dict = { - - 'sklearn.feature_selection.SelectFromModel': { - 'threshold': np.arange(0, 1.01, 0.05), - 'estimator': { - 'sklearn.ensemble.ExtraTreesRegressor': { - 'max_features': np.arange(0, 1.01, 0.05) - } - } - - }, - - 'sklearn.feature_selection.SelectFwe': { - 'alpha': np.arange(0, 0.05, 0.001), - 'score_func': { - 'sklearn.feature_selection.f_classif': None - } # read from dependencies ! need add an exception in preprocess_args - - }, - - 'sklearn.feature_selection.SelectKBest': { - 'k': range(1, 100), # need check range! - 'score_func': { - 'sklearn.feature_selection.f_classif': None - } - }, - - 'sklearn.feature_selection.SelectPercentile': { - 'percentile': range(1, 100), - 'score_func': { - 'sklearn.feature_selection.f_classif': None - } - }, - - 'sklearn.feature_selection.VarianceThreshold': { - 'threshold': np.arange(0, 1.01, 0.05) - } - -} - -"""'TPOTRFE': { - 'source': 'sklearn.feature_selection.RFE', - 'dependencies': { - 'sklearn.svm.SVC': { - 'kernel': ['linear'], - 'random_state': [42] - }, - 'regression': False - }, - 'params':{ - 'step': np.arange(0.1, 1.01, 0.05), - 'estimator': 'SVC(kernel=\'linear\', random_state=42)' # read from dependencies ! need add an exception in preprocess_args - } -},""" - - -""" 'TPOTSelectFromModel': { - 'source': 'sklearn.feature_selection.SelectFromModel', - 'dependencies': { - 'sklearn.ensemble.ExtraTreesClassifier': { - 'criterion': ['gini', 'entropy'], - 'max_features': np.arange(0, 1.01, 0.05) - }, - 'regression': False - }, - 'params':{ - 'threshold': np.arange(0, 1.01, 0.05), - 'estimator': 'ExtraTreesClassifier(criterion=criterion_selection, max_features=max_features)' # read from dependencies ! need add an exception in preprocess_args - } - }, -""" diff --git a/tpot/decorators.py b/tpot/decorators.py index 596fda3d..0b515f09 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -156,8 +156,7 @@ def check_pipeline(self, *args, **kwargs): with warnings.catch_warnings(): warnings.simplefilter('ignore') expr = func(self, *args, **kwargs) - #debug use - #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) + #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) # debug sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(expr), self.operators), self.operators_context) sklearn_pipeline.fit(pretest_X, pretest_y) bad_pipeline = False diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index e191782c..f03db116 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -224,6 +224,6 @@ def export(*args): class_profile['export'] = export - op_classname = '{}_{}'.format('TPOT',op_str) + op_classname = '_{}'.format(op_str) op_class = type(op_classname, (BaseClass,), class_profile) return op_class, arg_types diff --git a/tpot/operators_disable/__init__.py b/tpot/operators_disable/__init__.py deleted file mode 100644 index 14ec6b0a..00000000 --- a/tpot/operators_disable/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -You should have received a copy of the GNU General Public License along with -the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import * -from .classifiers import * -from .preprocessors import * -from .selectors import * -from .regressors import * -from .combine_dfs import CombineDFs diff --git a/tpot/operators_disable/base.py b/tpot/operators_disable/base.py deleted file mode 100644 index e0442b15..00000000 --- a/tpot/operators_disable/base.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -import numpy as np -from types import FunctionType - -try: - from inspect import signature # Python 3 -except ImportError: - from inspect import getargspec # Python 2 - - -class Operator(object): - """Base class for operators in TPOT""" - - def export(self, *args, **kwargs): - """Represent the operator as a string so that it can be exported to a - file - - Parameters - ---------- - args, kwargs - Arbitrary arguments to be passed to the operator - - Returns - ------- - export_string: str - String representation of the sklearn class with its parameters in - the format: - SklearnClassName(param1="val1", param2=val2) - - """ - operator_args = self.preprocess_args(*args, **kwargs) - - arguments = [] - for key in sorted(operator_args.keys()): - val = operator_args[key] - if isinstance(val, str): - val = '\"{}\"'.format(val) - elif isinstance(val, FunctionType): - val = val.__name__ - - arguments.append("{}={}".format(key, val)) - - return "{}({})".format(self.sklearn_class.__name__, ", ".join(arguments)) - - @property - def __name__(self): - """Necessary for deap so that it can generate a string identifier for - each opeartor. - """ - return self.__class__.sklearn_class.__name__ - - @property - def type(self): - """Returns the type of the operator, e.g: - ("Classifier", "Regressor", "Selector", "Preprocessor") - """ - return self.__class__.__bases__[0].__name__ - - def parameter_types(self): - """Return tuple of argument types for calling of the operator and the - return type of the operator - - Parameters - ---------- - None - - Returns - ------- - parameter_types: tuple - Tuple of the DEAP parameter types and the DEAP return type for the - operator - - """ - try: - # Python 3 - num_args = len(signature(self.preprocess_args).parameters.keys()) - except NameError: - # Python 2 - - # Remove 'self' - num_args = len(getargspec(self.preprocess_args).args[1:]) - - # Make sure the class has been written properly - if num_args != len(self.arg_types): - raise RuntimeError(("{}'s arg_types does not correspond to the " - "arguments defined for itself". - format(self.__name__))) - - # First argument is always a DataFrame - arg_types = [np.ndarray] + list(self.arg_types) - return_type = np.ndarray - - return (arg_types, return_type) - - @classmethod - def inheritors(cls): - """Returns set of all operators defined - - Parameters - ---------- - None - - Returns - ------- - operators: set - Set of all discovered operators that inherit from the base class - - """ - operators = set() - - # Search two levels deep and report leaves in inheritance tree - for operator_type in cls.__subclasses__(): - for operator in operator_type.__subclasses__(): - operators.add(operator()) # Instantiate class and append - - return operators - - @classmethod - def get_by_name(cls, name): - """Returns operator class instance by name - - Parameters - ---------- - name: str - Name of the sklearn class that belongs to a TPOT operator - - Returns - ------- - grandchild - An instance of the TPOT operator with a matching sklearn class name - - """ - for operator_type in cls.__subclasses__(): - for operator in operator_type.__subclasses__(): - if operator.sklearn_class.__name__ == name: - return operator() diff --git a/tpot/operators_disable/classifiers/__init__.py b/tpot/operators_disable/classifiers/__init__.py deleted file mode 100644 index 51ac33e9..00000000 --- a/tpot/operators_disable/classifiers/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import * -from .decision_tree import * -from .random_forest import * -from .bernoulli_nb import * -from .gaussian_nb import * -from .multinomial_nb import * -from .extra_trees import * -from .linear_svc import * -from .logistic_regression import * -from .knnc import * -from .gradient_boosting import * -try: - from .xg_boost import * -except ImportError: - pass diff --git a/tpot/operators_disable/classifiers/base.py b/tpot/operators_disable/classifiers/base.py deleted file mode 100644 index 5b0bcb49..00000000 --- a/tpot/operators_disable/classifiers/base.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from tpot.operators import Operator - - -class Classifier(Operator): - """Parent class for classifiers in TPOT""" - - root = True # Whether this operator type can be the root of the tree - regression = False # Whether this operator can be used in a regression problem - classification = True # Whether the operator can be used for classification diff --git a/tpot/operators_disable/classifiers/bernoulli_nb.py b/tpot/operators_disable/classifiers/bernoulli_nb.py deleted file mode 100644 index f198a116..00000000 --- a/tpot/operators_disable/classifiers/bernoulli_nb.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from sklearn.naive_bayes import BernoulliNB - - -class TPOTBernoulliNB(Classifier): - """Fits a Bernoulli Naive Bayes Classifier - - Parameters - ---------- - alpha: float - Additive (Laplace/Lidstone) smoothing parameter (0 for no smoothing). - binarize: float - Threshold for binarizing (mapping to booleans) of sample features. - - """ - import_hash = {'sklearn.naive_bayes': ['BernoulliNB']} - sklearn_class = BernoulliNB - arg_types = (float, float) - - def __init__(self): - pass - - def preprocess_args(self, alpha, binarize): - return { - 'alpha': alpha, - 'binarize': binarize, - 'fit_prior': True - } diff --git a/tpot/operators_disable/classifiers/decision_tree.py b/tpot/operators_disable/classifiers/decision_tree.py deleted file mode 100644 index 96f2d0cf..00000000 --- a/tpot/operators_disable/classifiers/decision_tree.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from sklearn.tree import DecisionTreeClassifier - - -class TPOTDecisionTreeClassifier(Classifier): - """Fits a decision tree classifier - - Parameters - ---------- - None - - """ - import_hash = {'sklearn.tree': ['DecisionTreeClassifier']} - sklearn_class = DecisionTreeClassifier - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - return {} diff --git a/tpot/operators_disable/classifiers/extra_trees.py b/tpot/operators_disable/classifiers/extra_trees.py deleted file mode 100644 index 75e11f0f..00000000 --- a/tpot/operators_disable/classifiers/extra_trees.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from sklearn.ensemble import ExtraTreesClassifier - - -class TPOTExtraTreesClassifier(Classifier): - """Fits an Extra Trees Classifier - - Parameters - ---------- - criterion: int - Integer that is used to select from the list of valid criteria, - either 'gini', or 'entropy' - max_features: float - The number of features to consider when looking for the best split - - """ - import_hash = {'sklearn.ensemble': ['ExtraTreesClassifier']} - sklearn_class = ExtraTreesClassifier - arg_types = (int, float) - - def __init__(self): - pass - - def preprocess_args(self, criterion, max_features): - # Select criterion string from list of valid parameters - criterion_values = ['gini', 'entropy'] - criterion_selection = criterion_values[criterion % len(criterion_values)] - - max_features = min(1., max(0., max_features)) - - return { - 'criterion': criterion_selection, - 'max_features': max_features, - 'n_estimators': 500 - } diff --git a/tpot/operators_disable/classifiers/gaussian_nb.py b/tpot/operators_disable/classifiers/gaussian_nb.py deleted file mode 100644 index bba84189..00000000 --- a/tpot/operators_disable/classifiers/gaussian_nb.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from sklearn.naive_bayes import GaussianNB - - -class TPOTGaussianNB(Classifier): - """Fits a Gaussian Naive Bayes Classifier - - Parameters - ---------- - None - - """ - import_hash = {'sklearn.naive_bayes': ['GaussianNB']} - sklearn_class = GaussianNB - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - return {} diff --git a/tpot/operators_disable/classifiers/gradient_boosting.py b/tpot/operators_disable/classifiers/gradient_boosting.py deleted file mode 100644 index 39a5070c..00000000 --- a/tpot/operators_disable/classifiers/gradient_boosting.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from sklearn.ensemble import GradientBoostingClassifier - - -class TPOTGradientBoosting(Classifier): - """Fits a Gradient Boosting classifier - - Parameters - ---------- - learning_rate: float - Shrinks the contribution of each tree by learning_rate - max_features: float - Maximum number of features to use (proportion of total features) - - """ - import_hash = {'sklearn.ensemble': ['GradientBoostingClassifier']} - sklearn_class = GradientBoostingClassifier - arg_types = (float, float) - - def __init__(self): - pass - - def preprocess_args(self, learning_rate, max_features): - learning_rate = min(1., max(learning_rate, 0.0001)) - max_features = min(1., max(0., learning_rate)) - - return { - 'learning_rate': learning_rate, - 'max_features': max_features, - 'n_estimators': 500 - } diff --git a/tpot/operators_disable/classifiers/knnc.py b/tpot/operators_disable/classifiers/knnc.py deleted file mode 100644 index 130931bd..00000000 --- a/tpot/operators_disable/classifiers/knnc.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from sklearn.neighbors import KNeighborsClassifier - - -class TPOTKNeighborsClassifier(Classifier): - """Fits a k-nearest neighbor classifier - - Parameters - ---------- - n_neighbors: int - Number of neighbors to use by default for k_neighbors queries; must be a positive value - weights: int - Selects a value from the list: ['uniform', 'distance'] - - """ - import_hash = {'sklearn.neighbors': ['KNeighborsClassifier']} - sklearn_class = KNeighborsClassifier - arg_types = (int, int) - - def __init__(self): - pass - - def preprocess_args(self, n_neighbors, weights): - n_neighbors = max(min(5, n_neighbors), 2) - - weights_values = ['uniform', 'distance'] - weights_selection = weights_values[weights % len(weights_values)] - - return { - 'n_neighbors': n_neighbors, - 'weights': weights_selection - } diff --git a/tpot/operators_disable/classifiers/linear_svc.py b/tpot/operators_disable/classifiers/linear_svc.py deleted file mode 100644 index 76e80d9b..00000000 --- a/tpot/operators_disable/classifiers/linear_svc.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from ...gp_types import Bool -from .base import Classifier -from sklearn.svm import LinearSVC - - -class TPOTLinearSVC(Classifier): - """Fits a Linear Support Vector Classifier - - Parameters - ---------- - C: float - Penalty parameter C of the error term. - penalty: int - Integer used to specify the norm used in the penalization (l1 or l2) - dual: bool - Select the algorithm to either solve the dual or primal optimization problem. - - """ - import_hash = {'sklearn.svm': ['LinearSVC']} - sklearn_class = LinearSVC - arg_types = (float, int, Bool) - - def __init__(self): - pass - - def preprocess_args(self, C, penalty, dual): - penalty_values = ['l1', 'l2'] - penalty_selection = penalty_values[penalty % len(penalty_values)] - - C = min(25., max(0.0001, C)) - - if penalty_selection == 'l1': - dual = False - - return { - 'C': C, - 'penalty': penalty_selection, - 'dual': dual - } diff --git a/tpot/operators_disable/classifiers/logistic_regression.py b/tpot/operators_disable/classifiers/logistic_regression.py deleted file mode 100644 index 6c707f30..00000000 --- a/tpot/operators_disable/classifiers/logistic_regression.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from ...gp_types import Bool -from .base import Classifier -from sklearn.linear_model import LogisticRegression - - -class TPOTLogisticRegression(Classifier): - """Fits a logistic regression classifier - - Parameters - ---------- - C: float - Inverse of regularization strength; must be a positive value. Like in support vector machines, smaller values specify stronger regularization. - penalty: int - Integer used to specify the norm used in the penalization (l1 or l2) - solver: sag (Note: sciket-learn version > 0.17) for l1 and liblinear for l2 - Algorithm to use in the optimization problem. - SAG = Stochastic Average Gradient descent solver. - Note that 'sag' fast convergence is only guaranteed on features with approximately the same scale. - 'newton-cg', 'lbfgs' and 'sag' only handle L2 penalty. - """ - import_hash = {'sklearn.linear_model': ['LogisticRegression']} - sklearn_class = LogisticRegression - arg_types = (float, int) - - def __init__(self): - pass - - def preprocess_args(self, C, penalty): - C = min(50., max(0.0001, C)) - - penalty_values = ['l1', 'l2'] - penalty_selection = penalty_values[penalty % len(penalty_values)] - if penalty_selection == 'l1': - solver = 'liblinear' - else: - solver = 'sag' - return { - 'C': C, - 'penalty': penalty_selection, - 'solver': solver - } diff --git a/tpot/operators_disable/classifiers/multinomial_nb.py b/tpot/operators_disable/classifiers/multinomial_nb.py deleted file mode 100644 index 2bfa1930..00000000 --- a/tpot/operators_disable/classifiers/multinomial_nb.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from sklearn.naive_bayes import MultinomialNB - - -class TPOTMultinomialNB(Classifier): - - """Fits a Multinomial Naive Bayes Classifier - - Parameters - ---------- - alpha: float - Additive (Laplace/Lidstone) smoothing parameter (0 for no smoothing). - """ - - import_hash = {'sklearn.naive_bayes': ['MultinomialNB']} - sklearn_class = MultinomialNB - arg_types = (float, ) - - def __init__(self): - pass - - def preprocess_args(self, alpha): - """Preprocess the arguments in case they need to be limited to a certain value range""" - return { - 'alpha': alpha, - 'fit_prior': True - } diff --git a/tpot/operators_disable/classifiers/random_forest.py b/tpot/operators_disable/classifiers/random_forest.py deleted file mode 100644 index 439ae49a..00000000 --- a/tpot/operators_disable/classifiers/random_forest.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from sklearn.ensemble import RandomForestClassifier - - -class TPOTRandomForestClassifier(Classifier): - - """Fits a random forest classifier. - - Parameters - ---------- - None - """ - - import_hash = {'sklearn.ensemble': ['RandomForestClassifier']} - sklearn_class = RandomForestClassifier - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - """Preprocess the arguments in case they need to be limited to a certain value range""" - return { - 'n_estimators': 500 - } diff --git a/tpot/operators_disable/classifiers/xg_boost.py b/tpot/operators_disable/classifiers/xg_boost.py deleted file mode 100644 index 99f14b4f..00000000 --- a/tpot/operators_disable/classifiers/xg_boost.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Classifier -from xgboost import XGBClassifier - - -class TPOTXGBClassifier(Classifier): - """Fits an XGBoost Classifier - - Parameters - ---------- - max_depth: int - Maximum tree depth for base learners - min_child_weight: int - Minimum sum of instance weight(hessian) needed in a child - learning_rate: float - Shrinks the contribution of each tree by learning_rate - subsample: float - Subsample ratio of the training instance - """ - import_hash = {'xgboost': ['XGBClassifier']} - sklearn_class = XGBClassifier - arg_types = (int, int, float, float) - - def __init__(self): - pass - - def preprocess_args(self, max_depth, min_child_weight, learning_rate, subsample): - max_depth = min(10, max(max_depth, 1)) - min_child_weight = min(20, max(min_child_weight, 1)) - learning_rate = min(1., max(learning_rate, 0.0001)) - subsample = min(1., max(subsample, 0.05)) - - return { - 'max_depth': max_depth, - 'min_child_weight': min_child_weight, - 'learning_rate': learning_rate, - 'subsample': subsample, - 'n_estimators': 500 - } diff --git a/tpot/operators_disable/combine_dfs.py b/tpot/operators_disable/combine_dfs.py deleted file mode 100644 index 090119fb..00000000 --- a/tpot/operators_disable/combine_dfs.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - - -class CombineDFs(object): - """Operator to combine two DataFrames""" - - @property - def __name__(self): - return self.__class__.__name__ diff --git a/tpot/operators_disable/preprocessors/__init__.py b/tpot/operators_disable/preprocessors/__init__.py deleted file mode 100644 index 65471211..00000000 --- a/tpot/operators_disable/preprocessors/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import * -from .pca import * -from .nystroem import * -from .feat_agg import * -from .fast_ica import * -from .rbf import * -from .binarizer import * -from .normalizer import * -from .min_max_scalar import * -from .max_abs_scalar import * -from .polynomial_features import * -from .robust_scaler import * -from .standard_scaler import * -from .zero_count import * diff --git a/tpot/operators_disable/preprocessors/base.py b/tpot/operators_disable/preprocessors/base.py deleted file mode 100644 index 7b832364..00000000 --- a/tpot/operators_disable/preprocessors/base.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from tpot.operators import Operator - - -class Preprocessor(Operator): - """Parent class for Feature Preprocessors in TPOT""" - - root = False # Whether this operator type can be the root of the tree - regression = True # Whether this operator can be used in a regression problem - classification = True # Whether the operator can be used for classification diff --git a/tpot/operators_disable/preprocessors/binarizer.py b/tpot/operators_disable/preprocessors/binarizer.py deleted file mode 100644 index f6eeb42e..00000000 --- a/tpot/operators_disable/preprocessors/binarizer.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.preprocessing import Binarizer - - -class TPOTBinarizer(Preprocessor): - """Uses scikit-learn's Binarizer to transform the feature set - - Parameters - ---------- - threshold: float - Feature values below or equal to this value are replaced by 0, above it by 1 - - """ - import_hash = {'sklearn.preprocessing': ['Binarizer']} - sklearn_class = Binarizer - arg_types = (float, ) - - def __init__(self): - pass - - def preprocess_args(self, threshold): - return { - 'threshold': threshold - } diff --git a/tpot/operators_disable/preprocessors/fast_ica.py b/tpot/operators_disable/preprocessors/fast_ica.py deleted file mode 100644 index f843c862..00000000 --- a/tpot/operators_disable/preprocessors/fast_ica.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.decomposition import FastICA - - -class TPOTFastICA(Preprocessor): - """Uses scikit-learn's FastICA to transform the feature set - - Parameters - ---------- - tol: float - Tolerance on update at each iteration. - - """ - import_hash = {'sklearn.decomposition': ['FastICA']} - sklearn_class = FastICA - arg_types = (float, ) - - def __init__(self): - pass - - def preprocess_args(self, tol): - tol = max(tol, 0.0001) - - return { - 'tol': tol - } diff --git a/tpot/operators_disable/preprocessors/feat_agg.py b/tpot/operators_disable/preprocessors/feat_agg.py deleted file mode 100644 index 03a6aa56..00000000 --- a/tpot/operators_disable/preprocessors/feat_agg.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.cluster import FeatureAgglomeration - - -class TPOTFeatureAgglomeration(Preprocessor): - """Uses scikit-learn's Nystroem to transform the feature set - - Parameters - ---------- - affinity: int - Metric used to compute the linkage. Can be "euclidean", "l1", "l2", - "manhattan", "cosine", or "precomputed". If linkage is "ward", only - "euclidean" is accepted. - Input integer is used to select one of the above strings. - linkage: int - Can be one of the following values: - "ward", "complete", "average" - Input integer is used to select one of the above strings. - - """ - import_hash = {'sklearn.cluster': ['FeatureAgglomeration']} - sklearn_class = FeatureAgglomeration - arg_types = (int, int) - - def __init__(self): - pass - - def preprocess_args(self, affinity, linkage): - linkage_types = ['ward', 'complete', 'average'] - linkage_name = linkage_types[linkage % len(linkage_types)] - - affinity_types = ['euclidean', 'l1', 'l2', 'manhattan', 'precomputed'] - affinity_name = 'euclidean' if linkage_name == 'ward' else affinity_types[affinity % len(affinity_types)] - - return { - 'affinity': affinity_name, - 'linkage': linkage_name - } diff --git a/tpot/operators_disable/preprocessors/max_abs_scalar.py b/tpot/operators_disable/preprocessors/max_abs_scalar.py deleted file mode 100644 index c07f9076..00000000 --- a/tpot/operators_disable/preprocessors/max_abs_scalar.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.preprocessing import MaxAbsScaler - -class TPOTMaxAbsScaler(Preprocessor): - - """Uses scikit-learn's MaxAbsScaler to transform all of the features by scaling them to [0, 1] relative to the feature's maximum value. - - Parameters - ---------- - None - """ - - import_hash = {'sklearn.preprocessing': ['MaxAbsScaler']} - sklearn_class = MaxAbsScaler - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - """Preprocess the arguments in case they need to be limited to a certain value range""" - return { } diff --git a/tpot/operators_disable/preprocessors/min_max_scalar.py b/tpot/operators_disable/preprocessors/min_max_scalar.py deleted file mode 100644 index 56b741e7..00000000 --- a/tpot/operators_disable/preprocessors/min_max_scalar.py +++ /dev/null @@ -1,42 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.preprocessing import MinMaxScaler - -class TPOTMaxAbsScaler(Preprocessor): - - """Uses scikit-learn's MinMaxScaler to transform all of the features by scaling them to the range [0, 1]. - - Parameters - ---------- - None - """ - - import_hash = {'sklearn.preprocessing': ['MinMaxScaler']} - sklearn_class = MinMaxScaler - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - """Preprocess the arguments in case they need to be limited to a certain value range""" - return { } diff --git a/tpot/operators_disable/preprocessors/normalizer.py b/tpot/operators_disable/preprocessors/normalizer.py deleted file mode 100644 index 894d10d0..00000000 --- a/tpot/operators_disable/preprocessors/normalizer.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.preprocessing import Normalizer - - -class TPOTNormalizer(Preprocessor): - """Uses scikit-learn's Normalizer to normalize samples individually to unit norm - - Parameters - ---------- - norm: 'l1', 'l2', or 'max' - The norm to use to normalize each non zero sample. - - """ - import_hash = {'sklearn.preprocessing': ['Normalizer']} - sklearn_class = Normalizer - arg_types = (int, ) - - def __init__(self): - pass - - def preprocess_args(self, norm): - norm_types = ['l1', 'l2', 'max'] - norm = norm_types[norm % len(norm_types)] - - return { - 'norm': norm - } diff --git a/tpot/operators_disable/preprocessors/nystroem.py b/tpot/operators_disable/preprocessors/nystroem.py deleted file mode 100644 index eec93af6..00000000 --- a/tpot/operators_disable/preprocessors/nystroem.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.kernel_approximation import Nystroem - - -class TPOTNystroem(Preprocessor): - """Uses scikit-learn's Nystroem to transform the feature set - - Parameters - ---------- - kernel: int - Kernel type is selected from scikit-learn's provided types: - 'sigmoid', 'polynomial', 'additive_chi2', 'poly', 'laplacian', 'cosine', 'linear', 'rbf', 'chi2' - - Input integer is used to select one of the above strings. - gamma: float - Gamma parameter for the kernels. - n_components: int - The number of components to keep - - """ - import_hash = {'sklearn.kernel_approximation': ['Nystroem']} - sklearn_class = Nystroem - arg_types = (int, float, int) - - def __init__(self): - pass - - def preprocess_args(self, kernel, gamma, n_components): - # Pulled from sklearn.metrics.pairwise.PAIRWISE_KERNEL_FUNCTIONS - kernel_types = ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'] - kernel_name = kernel_types[kernel % len(kernel_types)] - - n_components = max(1, n_components) - - return { - 'kernel': kernel_name, - 'gamma': gamma, - 'n_components': n_components - } diff --git a/tpot/operators_disable/preprocessors/pca.py b/tpot/operators_disable/preprocessors/pca.py deleted file mode 100644 index 76e85c10..00000000 --- a/tpot/operators_disable/preprocessors/pca.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.decomposition import PCA - -class TPOTRandomizedPCA(Preprocessor): - - """Uses scikit-learn's randomized PCA to transform the feature set - - Parameters - ---------- - iterated_power: int - Number of iterations for the power method. [1, 10] - """ - - import_hash = {'sklearn.decomposition': ['PCA']} - sklearn_class = PCA - arg_types = (int, ) - - def __init__(self): - pass - - def preprocess_args(self, iterated_power): - iterated_power = min(10, max(1, iterated_power)) - - return { - 'svd_solver': 'randomized', - 'iterated_power': iterated_power - } diff --git a/tpot/operators_disable/preprocessors/polynomial_features.py b/tpot/operators_disable/preprocessors/polynomial_features.py deleted file mode 100644 index 9ba47da1..00000000 --- a/tpot/operators_disable/preprocessors/polynomial_features.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.preprocessing import PolynomialFeatures - - -class TPOTPolynomialFeatures(Preprocessor): - """Uses scikit-learn's PolynomialFeatures to transform the feature set - - Parameters - ---------- - None - - """ - import_hash = {'sklearn.preprocessing': ['PolynomialFeatures']} - sklearn_class = PolynomialFeatures - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - return { - 'degree': 2, - 'include_bias': False, - 'interaction_only': False - } diff --git a/tpot/operators_disable/preprocessors/rbf.py b/tpot/operators_disable/preprocessors/rbf.py deleted file mode 100644 index 7354781a..00000000 --- a/tpot/operators_disable/preprocessors/rbf.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.kernel_approximation import RBFSampler - - -class TPOTRBFSampler(Preprocessor): - """Uses scikit-learn's RBFSampler to transform the feature set - - Parameters - ---------- - gamma: float - Parameter of RBF kernel: exp(-gamma * x^2) - - """ - import_hash = {'sklearn.kernel_approximation': ['RBFSampler']} - sklearn_class = RBFSampler - arg_types = (float, ) - - def __init__(self): - pass - - def preprocess_args(self, gamma): - return { - 'gamma': gamma - } diff --git a/tpot/operators_disable/preprocessors/robust_scaler.py b/tpot/operators_disable/preprocessors/robust_scaler.py deleted file mode 100644 index 214f7198..00000000 --- a/tpot/operators_disable/preprocessors/robust_scaler.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.preprocessing import RobustScaler - - -class TPOTRobustScaler(Preprocessor): - """Uses scikit-learn's RobustScaler to transform the feature set - - Parameters - ---------- - None - - """ - import_hash = {'sklearn.preprocessing': ['RobustScaler']} - sklearn_class = RobustScaler - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - return { - - } diff --git a/tpot/operators_disable/preprocessors/standard_scaler.py b/tpot/operators_disable/preprocessors/standard_scaler.py deleted file mode 100644 index c4f9b73a..00000000 --- a/tpot/operators_disable/preprocessors/standard_scaler.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Preprocessor -from sklearn.preprocessing import StandardScaler - - -class TPOTStandardScaler(Preprocessor): - """Uses scikit-learn's StandardScaler to transform the feature set - - Parameters - ---------- - None - - """ - import_hash = {'sklearn.preprocessing': ['StandardScaler']} - sklearn_class = StandardScaler - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - return { - - } diff --git a/tpot/operators_disable/preprocessors/zero_count.py b/tpot/operators_disable/preprocessors/zero_count.py deleted file mode 100644 index fd37bd32..00000000 --- a/tpot/operators_disable/preprocessors/zero_count.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -import numpy as np - -from .base import Preprocessor -from sklearn.base import BaseEstimator -from sklearn.utils import check_array - - -class ZeroCount(BaseEstimator): - - """Preprocessor that adds two virtual features to the dataset, one for the count of zero values in the feature set, and one for the count of non-zeros in the feature set""" - - def __init__(self): - pass - - def fit(self, X, y=None): - """Dummy function to fit in with the sklearn API""" - return self - - def transform(self, X, y=None): - """Transform data by adding two virtual features - - Parameters - ---------- - X: numpy ndarray, {n_samples, n_components} - New data, where n_samples is the number of samples and n_components - is the number of components. - y: None - Unused - - Returns - ------- - X_transformed: array-like, shape (n_samples, n_features) - The transformed feature set - """ - X = check_array(X) - n_features = X.shape[1] - - X_transformed = np.copy(X) - - non_zero = np.apply_along_axis(lambda row: np.count_nonzero(row), - axis=1, arr=X_transformed) - zero_col = np.apply_along_axis(lambda row: (n_features - np.count_nonzero(row)), - axis=1, arr=X_transformed) - - X_transformed = np.insert(X_transformed, n_features, non_zero, axis=1) - X_transformed = np.insert(X_transformed, n_features + 1, zero_col, axis=1) - - return X_transformed - - -class TPOTZeroCount(Preprocessor): - - """Uses TPOT's ZeroCount to transform the feature set""" - - import_hash = {'tpot.operators.preprocessors': ['ZeroCount']} - sklearn_class = ZeroCount - arg_types = () - - def __init__(self): - """Creates a new TPOTZeroCount instance""" - pass - - def preprocess_args(self): - """Preprocesses the arguments in case they need to be constrained in some way""" - return {} diff --git a/tpot/operators_disable/regressors/__init__.py b/tpot/operators_disable/regressors/__init__.py deleted file mode 100644 index f9f5a91a..00000000 --- a/tpot/operators_disable/regressors/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import * -from .elastic_net import * -from .extra_trees import * -from .gradient_boosting import * -from .knnr import * -from .lasso_lars_cv import * -from .linear_svr import * -from .random_forest import * -try: - from .xg_boost_r import * -except ImportError: - pass diff --git a/tpot/operators_disable/regressors/base.py b/tpot/operators_disable/regressors/base.py deleted file mode 100644 index e6b1897f..00000000 --- a/tpot/operators_disable/regressors/base.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from tpot.operators import Operator - - -class Regressor(Operator): - """Parent class for regressors in TPOT""" - - root = True # Whether this operator type can be the root of the tree - regression = True # Whether this operator can be used in a regression problem - classification = False # Whether the operator can be used for classification diff --git a/tpot/operators_disable/regressors/elastic_net.py b/tpot/operators_disable/regressors/elastic_net.py deleted file mode 100644 index 345bf8a6..00000000 --- a/tpot/operators_disable/regressors/elastic_net.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Regressor -from sklearn.linear_model import ElasticNet - - -class TPOTElasticNet(Regressor): - """Fits a Elastic Net Regressor - - Parameters - ---------- - alpha: float - Constant that multiplies the penalty terms. - l1_ratio: int - The ElasticNet mixing parameter, with 0 <= l1_ratio <= 1 - - """ - import_hash = {'sklearn.linear_model': ['ElasticNet']} - sklearn_class = ElasticNet - arg_types = (float, float) - - def __init__(self): - pass - - def preprocess_args(self, alpha, l1_ratio): - alpha = min(1., max(0.0001, alpha)) - l1_ratio = min(1., max(0.0001, l1_ratio)) - - return { - 'alpha': alpha, - 'l1_ratio': l1_ratio - } diff --git a/tpot/operators_disable/regressors/extra_trees.py b/tpot/operators_disable/regressors/extra_trees.py deleted file mode 100644 index 7f10f28d..00000000 --- a/tpot/operators_disable/regressors/extra_trees.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Regressor -from sklearn.ensemble import ExtraTreesRegressor - - -class TPOTExtraTreesRegressor(Regressor): - """Fits an Extra Trees Regressor - - Parameters - ---------- - criterion: int - Integer that is used to select from the list of valid criteria, - either 'gini', or 'entropy' - max_features: float - The number of features to consider when looking for the best split - - """ - import_hash = {'sklearn.ensemble': ['ExtraTreesRegressor']} - sklearn_class = ExtraTreesRegressor - arg_types = (float, ) - - def __init__(self): - pass - - def preprocess_args(self, max_features): - max_features = min(1., max(0., max_features)) - - return { - 'max_features': max_features, - 'n_estimators': 500 - } diff --git a/tpot/operators_disable/regressors/gradient_boosting.py b/tpot/operators_disable/regressors/gradient_boosting.py deleted file mode 100644 index 7eaeea3d..00000000 --- a/tpot/operators_disable/regressors/gradient_boosting.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Regressor -from sklearn.ensemble import GradientBoostingRegressor - - -class TPOTGradientBRegressor(Regressor): - """Fits a Gradient Boosting Regressor - - Parameters - ---------- - learning_rate: float - Shrinks the contribution of each tree by learning_rate - max_features: float - Maximum number of features to use (proportion of total features) - - """ - import_hash = {'sklearn.ensemble': ['GradientBoostingRegressor']} - sklearn_class = GradientBoostingRegressor - arg_types = (float, float) - - def __init__(self): - pass - - def preprocess_args(self, learning_rate, max_features): - learning_rate = min(1., max(learning_rate, 0.0001)) - max_features = min(1., max(0., learning_rate)) - - return { - 'learning_rate': learning_rate, - 'max_features': max_features, - 'n_estimators': 500 - } diff --git a/tpot/operators_disable/regressors/knnr.py b/tpot/operators_disable/regressors/knnr.py deleted file mode 100644 index 276260dc..00000000 --- a/tpot/operators_disable/regressors/knnr.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Regressor -from sklearn.neighbors import KNeighborsRegressor - - -class TPOTKNeighborsRegressor(Regressor): - """Fits a k-nearest neighbor Regressor - - Parameters - ---------- - n_neighbors: int - Number of neighbors to use by default for k_neighbors queries; must be a positive value - weights: int - Selects a value from the list: ['uniform', 'distance'] - - """ - import_hash = {'sklearn.neighbors': ['KNeighborsRegressor']} - sklearn_class = KNeighborsRegressor - arg_types = (int, int) - - def __init__(self): - pass - - def preprocess_args(self, n_neighbors, weights): - n_neighbors = max(min(5, n_neighbors), 2) - - weights_values = ['uniform', 'distance'] - weights_selection = weights_values[weights % len(weights_values)] - - return { - 'n_neighbors': n_neighbors, - 'weights': weights_selection - } diff --git a/tpot/operators_disable/regressors/lasso_lars_cv.py b/tpot/operators_disable/regressors/lasso_lars_cv.py deleted file mode 100644 index c1454bbd..00000000 --- a/tpot/operators_disable/regressors/lasso_lars_cv.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from ...gp_types import Bool -from .base import Regressor -from sklearn.linear_model import LassoLarsCV - - -class TPOTLassoLarsCV(Regressor): - """Fits a LassoLarsCV Regressor - - Parameters - ---------- - normalize: bool - If True, the regressors X will be normalized before regression. - - """ - import_hash = {'sklearn.linear_model': ['LassoLarsCV']} - sklearn_class = LassoLarsCV - arg_types = (Bool, ) - - def __init__(self): - pass - - def preprocess_args(self, normalize): - return { - 'normalize': normalize - } diff --git a/tpot/operators_disable/regressors/linear_svr.py b/tpot/operators_disable/regressors/linear_svr.py deleted file mode 100644 index 61b16cda..00000000 --- a/tpot/operators_disable/regressors/linear_svr.py +++ /dev/null @@ -1,50 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from ...gp_types import Bool -from .base import Regressor -from sklearn.svm import LinearSVR - - -class TPOTLinearSVR(Regressor): - """Fits a Linear Support Vector Regressor - - Parameters - ---------- - C: float - Penalty parameter C of the error term. - dual: bool - Select the algorithm to either solve the dual or primal optimization problem. - - """ - import_hash = {'sklearn.svm': ['LinearSVR']} - sklearn_class = LinearSVR - arg_types = (float, Bool) - - def __init__(self): - pass - - def preprocess_args(self, C, dual): - C = min(25., max(0.0001, C)) - - return { - 'C': C, - 'dual': dual - } diff --git a/tpot/operators_disable/regressors/random_forest.py b/tpot/operators_disable/regressors/random_forest.py deleted file mode 100644 index e4d7f05c..00000000 --- a/tpot/operators_disable/regressors/random_forest.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Regressor -from sklearn.ensemble import RandomForestRegressor - - -class TPOTRandomForestClassifier(Regressor): - - """Fits a random forest Regressor. - - Parameters - ---------- - None - """ - - import_hash = {'sklearn.ensemble': ['RandomForestRegressor']} - sklearn_class = RandomForestRegressor - arg_types = () - - def __init__(self): - pass - - def preprocess_args(self): - """Preprocess the arguments in case they need to be limited to a certain value range""" - return { - 'n_estimators': 500 - } diff --git a/tpot/operators_disable/regressors/xg_boost_r.py b/tpot/operators_disable/regressors/xg_boost_r.py deleted file mode 100644 index 2adb10db..00000000 --- a/tpot/operators_disable/regressors/xg_boost_r.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Regressor -from xgboost import XGBRegressor - - -class TPOTXGBRegressor(Regressor): - """Fits an XGBoost Regressor - - Parameters - ---------- - max_depth: int - Maximum tree depth for base learners - min_child_weight: int - Minimum sum of instance weight(hessian) needed in a child - learning_rate: float - Shrinks the contribution of each tree by learning_rate - subsample: float - Subsample ratio of the training instance - """ - import_hash = {'xgboost': ['XGBRegressor']} - sklearn_class = XGBRegressor - arg_types = (int, int, float, float) - - def __init__(self): - pass - - def preprocess_args(self, max_depth, min_child_weight, learning_rate, subsample): - max_depth = min(10, max(max_depth, 1)) - min_child_weight = min(20, max(min_child_weight, 1)) - learning_rate = min(1., max(learning_rate, 0.0001)) - subsample = min(1., max(subsample, 0.05)) - - return { - 'max_depth': max_depth, - 'min_child_weight': min_child_weight, - 'learning_rate': learning_rate, - 'subsample': subsample, - 'n_estimators': 500 - } diff --git a/tpot/operators_disable/selectors/__init__.py b/tpot/operators_disable/selectors/__init__.py deleted file mode 100644 index f4e3de7b..00000000 --- a/tpot/operators_disable/selectors/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import * -# Temporarily remove the RFE operator. In many cases it seems to be slow and causes TPOT to freeze. -# TODO: Dig into the freezing issue with RFE and see if we can add it back under certain constraints. -#from .rfe import * -from .select_fwe import * -from .select_kbest import * -from .select_percentile import * -from .variance_threshold import * -from .select_from_model import * -from .select_from_model_r import * diff --git a/tpot/operators_disable/selectors/base.py b/tpot/operators_disable/selectors/base.py deleted file mode 100644 index 57034699..00000000 --- a/tpot/operators_disable/selectors/base.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from tpot.operators import Operator - - -class Selector(Operator): - """Parent class for Feature Selectors in TPOT""" - - root = False # Whether this operator type can be the root of the tree - regression = True # Whether this operator can be used in a regression problem - classification = True # Whether the operator can be used for classification diff --git a/tpot/operators_disable/selectors/rfe.py b/tpot/operators_disable/selectors/rfe.py deleted file mode 100644 index e0da0e74..00000000 --- a/tpot/operators_disable/selectors/rfe.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Selector -from sklearn.feature_selection import RFE -from sklearn.svm import SVC - - -class TPOTRFE(Selector): - """Uses scikit-learn's RFE to transform the feature set - - Parameters - ---------- - step: float - The percentage of features to drop each iteration - - """ - import_hash = {'sklearn.feature_selection': ['RFE'], 'sklearn.svm': ['SVC']} - sklearn_class = RFE - arg_types = (float, ) - regression = False # Can not be used in regression due to SVC estimator - - def __init__(self): - pass - - def preprocess_args(self, step): - step = max(min(0.99, step), 0.1) - - return { - 'step': step, - 'estimator': SVC(kernel='linear', random_state=42) - } diff --git a/tpot/operators_disable/selectors/select_from_model.py b/tpot/operators_disable/selectors/select_from_model.py deleted file mode 100644 index 0417771d..00000000 --- a/tpot/operators_disable/selectors/select_from_model.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Selector -from sklearn.feature_selection import SelectFromModel -from sklearn.ensemble import ExtraTreesClassifier - - -class TPOTSelectFromModel(Selector): - """Uses scikit-learn's ExtraTreesClassifier combined with SelectFromModel - to transform the feature set. - - Parameters - ---------- - threshold: float - Features whose importance is greater or equal are kept while the others - are discarded. - criterion: int - For the ExtraTreesClassifier: - Integer that is used to select from the list of valid criteria, - either 'gini', or 'entropy' - max_features: float - For the ExtraTreesClassifier: - The number of features to consider when looking for the best split - - """ - import_hash = { - 'sklearn.feature_selection': ['SelectFromModel'], - 'sklearn.ensemble': ['ExtraTreesClassifier'] - } - sklearn_class = SelectFromModel - arg_types = (float, int, float) - regression = False # Can not be used in regression due to ExtraTreesClassifier - - def __init__(self): - pass - - def preprocess_args(self, threshold, criterion, max_features): - threshold = min(1., max(0., threshold)) - - # Select criterion string from list of valid parameters - criterion_values = ['gini', 'entropy'] - criterion_selection = criterion_values[criterion % len(criterion_values)] - - max_features = min(1., max(0., max_features)) - - return { - 'estimator': ExtraTreesClassifier(criterion=criterion_selection, max_features=max_features), - 'threshold': threshold - } diff --git a/tpot/operators_disable/selectors/select_from_model_r.py b/tpot/operators_disable/selectors/select_from_model_r.py deleted file mode 100644 index 086d3a13..00000000 --- a/tpot/operators_disable/selectors/select_from_model_r.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Selector -from sklearn.feature_selection import SelectFromModel -from sklearn.ensemble import ExtraTreesRegressor - - -class TPOTSelectFromModelR(Selector): - """Uses scikit-learn's ExtraTreesRegressor combined with SelectFromModel - to transform the feature set. - - Parameters - ---------- - threshold: float - Features whose importance is greater or equal are kept while the others - are discarded. - max_features: float - For the ExtraTreesRegressor: - The number of features to consider when looking for the best split - - """ - import_hash = { - 'sklearn.feature_selection': ['SelectFromModel'], - 'sklearn.ensemble': ['ExtraTreesRegressor'] - } - sklearn_class = SelectFromModel - arg_types = (float, float) - classification = False - - def __init__(self): - pass - - def preprocess_args(self, threshold, max_features): - threshold = min(1., max(0., threshold)) - - max_features = min(1., max(0., max_features)) - - return { - 'estimator': ExtraTreesRegressor(max_features=max_features), - 'threshold': threshold - } diff --git a/tpot/operators_disable/selectors/select_fwe.py b/tpot/operators_disable/selectors/select_fwe.py deleted file mode 100644 index ef799552..00000000 --- a/tpot/operators_disable/selectors/select_fwe.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Selector -from sklearn.feature_selection import SelectFwe, f_classif - - -class TPOTSelectFwe(Selector): - """Uses scikit-learn's SelectFwe to transform the feature set - - Parameters - ---------- - alpha: float in the range [0.001, 0.05] - The highest uncorrected p-value for features to keep - - """ - import_hash = {'sklearn.feature_selection': ['SelectFwe', 'f_classif']} - sklearn_class = SelectFwe - arg_types = (float, ) - - def __init__(self): - pass - - def preprocess_args(self, alpha): - alpha = max(min(0.05, alpha), 0.001) - - return { - 'score_func': f_classif, - 'alpha': alpha - } diff --git a/tpot/operators_disable/selectors/select_kbest.py b/tpot/operators_disable/selectors/select_kbest.py deleted file mode 100644 index 38e18ea9..00000000 --- a/tpot/operators_disable/selectors/select_kbest.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Selector -from sklearn.feature_selection import SelectKBest, f_classif - - -class TPOTSelectKBest(Selector): - """Uses scikit-learn's SelectKBest to transform the feature set - - Parameters - ---------- - k: int - The top k features to keep from the original set of features in the training data - - """ - import_hash = {'sklearn.feature_selection': ['SelectKBest', 'f_classif']} - sklearn_class = SelectKBest - arg_types = (int, ) - - def __init__(self): - pass - - def preprocess_args(self, k): - k = max(1, k) - - return { - 'score_func': f_classif, - 'k': k - } diff --git a/tpot/operators_disable/selectors/select_percentile.py b/tpot/operators_disable/selectors/select_percentile.py deleted file mode 100644 index 049a628e..00000000 --- a/tpot/operators_disable/selectors/select_percentile.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Selector -from sklearn.feature_selection import SelectPercentile, f_classif - - -class TPOTSelectPercentile(Selector): - """Uses scikit-learn's SelectPercentile to transform the feature set - - Parameters - ---------- - percentile: int - The features that belong in the top percentile to keep from the original - set of features in the training data - - """ - import_hash = {'sklearn.feature_selection': ['SelectPercentile', 'f_classif']} - sklearn_class = SelectPercentile - arg_types = (int, ) - - def __init__(self): - pass - - def preprocess_args(self, percentile): - percentile = max(min(99, percentile), 1) - - return { - 'score_func': f_classif, - 'percentile': percentile - } diff --git a/tpot/operators_disable/selectors/variance_threshold.py b/tpot/operators_disable/selectors/variance_threshold.py deleted file mode 100644 index ade8bbe7..00000000 --- a/tpot/operators_disable/selectors/variance_threshold.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -from .base import Selector -from sklearn.feature_selection import VarianceThreshold - - -class TPOTVarianceThreshold(Selector): - """Uses scikit-learn's VarianceThreshold to transform the feature set - - Parameters - ---------- - threshold: float - The variance threshold that removes features that fall under the threshold - - """ - import_hash = {'sklearn.feature_selection': ['VarianceThreshold']} - sklearn_class = VarianceThreshold - arg_types = (float, ) - - def __init__(self): - pass - - def preprocess_args(self, threshold): - return { - 'threshold': threshold - } diff --git a/tpot/tpot.py b/tpot/tpot.py index 4215151a..3bc944c0 100644 --- a/tpot/tpot.py +++ b/tpot/tpot.py @@ -20,14 +20,16 @@ from .base import TPOTBase from .config_classifier import classifier_config_dict -#from config_regressor import regressor_config_dic +from .config_regressor import regressor_config_dict class TPOTClassifier(TPOTBase): """TPOT estimator for classification problems""" scoring_function = 'balanced_accuracy' # Classification scoring - operator_dict = classifier_config_dict + operator_dict = classifier_config_dict # Classification dictionary + classification = True + regression = False def _ignore_operator(self, op): """Filter that describes which operators are not used @@ -44,6 +46,9 @@ class TPOTRegressor(TPOTBase): """TPOT estimator for regression problems""" scoring_function = 'neg_mean_squared_error' # Regression scoring + operator_dict = regressor_config_dict # Regression dictionary + classification = False + regression = True def _ignore_operator(self, op): """Filter that describes which operators are not used diff --git a/tpot_export_class_conf.py b/tpot_export_class_conf.py index 2cd39c4b..124b6c03 100644 --- a/tpot_export_class_conf.py +++ b/tpot_export_class_conf.py @@ -1,11 +1,9 @@ import numpy as np -from sklearn.ensemble import VotingClassifier +from sklearn.decomposition import PCA from sklearn.model_selection import train_test_split -from sklearn.naive_bayes import BernoulliNB from sklearn.neighbors import KNeighborsClassifier -from sklearn.pipeline import make_pipeline, make_union -from sklearn.preprocessing import FunctionTransformer +from sklearn.pipeline import make_pipeline # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -14,8 +12,8 @@ train_test_split(features, tpot_data['class'], random_state=42) exported_pipeline = make_pipeline( - make_union(VotingClassifier([("est", BernoulliNB(alpha=10.0, fit_prior=False))]), FunctionTransformer(lambda X: X)), - KNeighborsClassifier(n_neighbors=10, p=1, weights="uniform") + PCA(iterated_power=7, svd_solver="randomized"), + KNeighborsClassifier(n_neighbors=7, p=1, weights="distance") ) exported_pipeline.fit(training_features, training_classes) diff --git a/tpot_export_reg_conf.py b/tpot_export_reg_conf.py new file mode 100644 index 00000000..b94ce290 --- /dev/null +++ b/tpot_export_reg_conf.py @@ -0,0 +1,15 @@ +import numpy as np + +from sklearn.linear_model import RidgeCV +from sklearn.model_selection import train_test_split + +# NOTE: Make sure that the class is labeled 'class' in the data file +tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) +features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) +training_features, testing_features, training_classes, testing_classes = \ + train_test_split(features, tpot_data['class'], random_state=42) + +exported_pipeline = RidgeCV() + +exported_pipeline.fit(training_features, training_classes) +results = exported_pipeline.predict(testing_features) diff --git a/tpot_test_config_dict.py b/tpot_test_config_dict.py index 3bae715f..ede65c9a 100644 --- a/tpot_test_config_dict.py +++ b/tpot_test_config_dict.py @@ -1,5 +1,5 @@ -from tpot import TPOTClassifier -from sklearn.datasets import make_classification +from tpot import TPOTClassifier, TPOTRegressor +from sklearn.datasets import make_classification, make_regression from sklearn.model_selection import train_test_split import time @@ -14,4 +14,17 @@ tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) tpot.export('tpot_export_class_conf.py') -print('\nTime usages:',time.time()-time_start) +print('\nTime usages for TPOTClassifier:',time.time()-time_start) + + +X, y = make_regression(n_samples=200, n_features=50, + n_informative=10, n_targets=5, random_state=42) +X_train, X_test, y_train, y_test = train_test_split(X, y, + train_size=0.75, test_size=0.25) + +tpot = TPOTRegressor(generations=4, population_size=20, verbosity=2, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +tpot.export('tpot_export_reg_conf.py') +print('\nTime usages for TPOTRegressor:',time.time()-time_start) From 79a225a132c2d398fee2be7710f75aae6324e149 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 27 Jan 2017 17:55:40 -0500 Subject: [PATCH 052/154] remove classfication and regession from op class --- tpot/base.py | 6 +----- tpot/operator_utils.py | 6 +----- tpot/tpot.py | 24 ------------------------ 3 files changed, 2 insertions(+), 34 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 8e4ea00c..5563212f 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -187,10 +187,8 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, 'Please check the dictionary file') self.operators = [] self.arguments = [] - for key in sorted(self.operator_dict.keys()): - op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key], - classification = self.classification, regression = self.regression) + op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key]) self.operators.append(op_class) self.arguments += arg_types @@ -267,8 +265,6 @@ def _setup_pset(self): # Add all operators to the primitive set for op in self.operators: - if self._ignore_operator(op): - continue if op.root: # We need to add rooted primitives twice so that they can diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index f03db116..acc9124b 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -35,8 +35,6 @@ def __name__(self): """ return self.__class__.sklearn_class.__name__ root = False # Whether this operator type can be the root of the tree - regression = False # Whether this operator can be used in a regression problem - classification = False # Whether the operator can be used for classification import_hash = None sklearn_class = None arg_types = None @@ -84,7 +82,7 @@ def ARGTypeClassFactory(classname, prange, BaseClass=ARGType): """ return type(classname, (BaseClass,), {'values':prange}) -def TPOTOperatorClassFactory(opsourse, opdict, regression=False, classification=False, BaseClass=Operator): +def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator): """Dynamically create operator class Parameters ---------- @@ -110,8 +108,6 @@ def TPOTOperatorClassFactory(opsourse, opdict, regression=False, classification= class_profile = {} - class_profile['regression'] = regression - class_profile['classification'] = classification dep_op_list = {} import_str, op_str, op_obj = source_decode(opsourse) diff --git a/tpot/tpot.py b/tpot/tpot.py index 3bc944c0..d7e2838c 100644 --- a/tpot/tpot.py +++ b/tpot/tpot.py @@ -28,18 +28,6 @@ class TPOTClassifier(TPOTBase): scoring_function = 'balanced_accuracy' # Classification scoring operator_dict = classifier_config_dict # Classification dictionary - classification = True - regression = False - - def _ignore_operator(self, op): - """Filter that describes which operators are not used - - Parameters - ---------- - op: Operator - TPOT Pipeline operator being tested - """ - return not op.classification class TPOTRegressor(TPOTBase): @@ -47,15 +35,3 @@ class TPOTRegressor(TPOTBase): scoring_function = 'neg_mean_squared_error' # Regression scoring operator_dict = regressor_config_dict # Regression dictionary - classification = False - regression = True - - def _ignore_operator(self, op): - """Filter that describes which operators are not used - - Parameters - ---------- - op: Operator - TPOT Pipeline operator being tested - """ - return not op.regression From 154add50a17fec0754ab365ea88c3c485e49cd04 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 30 Jan 2017 11:04:58 -0500 Subject: [PATCH 053/154] fix customized dict import --- config_classifier_test.py | 188 ++++++++++++++++++++++++++++++++++++++ tpot/base.py | 8 +- tpot/config_classifier.py | 3 +- tpot/config_regressor.py | 2 - tpot/tpot.py | 4 + tpot_export_class_conf.py | 8 +- tpot_test_config_dict.py | 6 +- 7 files changed, 207 insertions(+), 12 deletions(-) create mode 100644 config_classifier_test.py diff --git a/config_classifier_test.py b/config_classifier_test.py new file mode 100644 index 00000000..c4de25ef --- /dev/null +++ b/config_classifier_test.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- + +""" +Copyright 2016 Randal S. Olson + +This file is part of the TPOT library. + +The TPOT library is free software: you can redistribute it and/or +modify it under the terms of the GNU General Public License as published by the +Free Software Foundation, either version 3 of the License, or (at your option) +any later version. + +The TPOT library is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +details. You should have received a copy of the GNU General Public License along +with the TPOT library. If not, see http://www.gnu.org/licenses/. + +""" + +""" +dictionary format (json-like format): +key: + operator name +value: + source: module source (e.g sklearn.tree) + dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency + params: a dictionary of parameter names (keys) and parameter ranges (values); None for no dependency +""" +import numpy as np + +classifier_config_dict = { + + # Classifiers + 'sklearn.naive_bayes.GaussianNB': { + }, + + 'sklearn.naive_bayes.BernoulliNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + }, + + 'sklearn.naive_bayes.MultinomialNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + }, + + 'sklearn.tree.DecisionTreeClassifier': { + 'criterion': ["gini", "entropy"], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21) + }, + + 'sklearn.ensemble.ExtraTreesClassifier': { + 'criterion': ["gini", "entropy"], + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] + }, + + 'sklearn.ensemble.RandomForestClassifier': { + 'criterion': ["gini", "entropy"], + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] + }, + + 'sklearn.ensemble.GradientBoostingClassifier': { + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'subsample': np.arange(0.05, 1.01, 0.05), + 'max_features': np.arange(0, 1.01, 0.05) + }, + + 'sklearn.neighbors.KNeighborsClassifier': { + 'n_neighbors': range(1, 101), + 'weights': ["uniform", "distance"], + 'p': [1, 2] + }, + + 'sklearn.svm.LinearSVC': { + 'penalty': ["l1", "l2"], + 'loss': ["hinge", "squared_hinge"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] + }, + + 'sklearn.linear_model.LogisticRegression': { + 'penalty': ["l1", "l2"], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'dual': [True, False] + }, + + 'xgboost.XGBClassifier': { + 'max_depth': range(1, 11), + 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], + 'subsample': np.arange(0.05, 1.01, 0.05), + 'min_child_weight': range(1, 21) + }, + + # Preprocesssors + 'sklearn.preprocessing.Binarizer': { + 'threshold': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.decomposition.FastICA': { + 'tol': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.cluster.FeatureAgglomeration': { + 'linkage': ['ward', 'complete', 'average'], + 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] + }, + + 'sklearn.preprocessing.MaxAbsScaler': { + }, + + 'sklearn.preprocessing.MinMaxScaler': { + }, + + 'sklearn.preprocessing.Normalizer': { + 'norm': ['l1', 'l2', 'max'] + }, + + 'sklearn.kernel_approximation.Nystroem': { + 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], + 'gamma': np.arange(0.0, 1.01, 0.05), + 'n_components': range(1, 11) + }, + + 'sklearn.decomposition.PCA': { + 'svd_solver': ['randomized'], + 'iterated_power': range(1, 11) + }, + + 'sklearn.preprocessing.PolynomialFeatures': { + 'degree': [2], + 'include_bias': [False], + 'interaction_only': [False] + }, + + 'sklearn.kernel_approximation.RBFSampler': { + 'gamma': np.arange(0.0, 1.01, 0.05) + }, + + 'sklearn.preprocessing.RobustScaler': { + }, + + 'sklearn.preprocessing.StandardScaler': { + }, + + 'tpot.build_in_operators.ZeroCount': { + }, + + # Selectors + 'sklearn.feature_selection.SelectFwe': { + 'alpha': np.arange(0, 0.05, 0.001), + 'score_func': { + 'sklearn.feature_selection.f_classif': None + } # read from dependencies ! need add an exception in preprocess_args + + }, + + 'sklearn.feature_selection.SelectKBest': { + 'k': range(1, 100), # need check range! + 'score_func': { + 'sklearn.feature_selection.f_classif': None + } + }, + + 'sklearn.feature_selection.SelectPercentile': { + 'percentile': range(1, 100), + 'score_func': { + 'sklearn.feature_selection.f_classif': None + } + }, + + 'sklearn.feature_selection.VarianceThreshold': { + 'threshold': np.arange(0.05, 1.01, 0.05) + } + +} diff --git a/tpot/base.py b/tpot/base.py index 5563212f..79833f8a 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -181,8 +181,12 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, # define operator dictionary if operator_dict_file: try: - exec(open(operator_dict_file, 'r').read()) - except: + with open(operator_dict_file,'r') as inf: + file_string = inf.read() + operator_dict = eval(file_string[file_string.find('{'):(file_string.rfind('}')+1)]) + self.operator_dict = operator_dict + except Exception as e: + print(e) raise TypeError('The operator dictionary file is in bad format or not available! ' 'Please check the dictionary file') self.operators = [] diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py index 9a63d2f2..94ab9fe7 100644 --- a/tpot/config_classifier.py +++ b/tpot/config_classifier.py @@ -203,7 +203,6 @@ 'max_features': np.arange(0, 1.01, 0.05) } } - }, - + } } diff --git a/tpot/config_regressor.py b/tpot/config_regressor.py index 534a7355..7d6190d9 100644 --- a/tpot/config_regressor.py +++ b/tpot/config_regressor.py @@ -192,6 +192,4 @@ } - - } diff --git a/tpot/tpot.py b/tpot/tpot.py index d7e2838c..6d1b8c60 100644 --- a/tpot/tpot.py +++ b/tpot/tpot.py @@ -28,6 +28,8 @@ class TPOTClassifier(TPOTBase): scoring_function = 'balanced_accuracy' # Classification scoring operator_dict = classifier_config_dict # Classification dictionary + classification = True + regression = False class TPOTRegressor(TPOTBase): @@ -35,3 +37,5 @@ class TPOTRegressor(TPOTBase): scoring_function = 'neg_mean_squared_error' # Regression scoring operator_dict = regressor_config_dict # Regression dictionary + classification = False + regression = True diff --git a/tpot_export_class_conf.py b/tpot_export_class_conf.py index 124b6c03..1433f261 100644 --- a/tpot_export_class_conf.py +++ b/tpot_export_class_conf.py @@ -1,9 +1,9 @@ import numpy as np -from sklearn.decomposition import PCA +from sklearn.ensemble import ExtraTreesClassifier from sklearn.model_selection import train_test_split -from sklearn.neighbors import KNeighborsClassifier from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import RobustScaler # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -12,8 +12,8 @@ train_test_split(features, tpot_data['class'], random_state=42) exported_pipeline = make_pipeline( - PCA(iterated_power=7, svd_solver="randomized"), - KNeighborsClassifier(n_neighbors=7, p=1, weights="distance") + RobustScaler(), + ExtraTreesClassifier(bootstrap=False, criterion="entropy", max_features=0.9500000000000001, min_samples_leaf=3, min_samples_split=20) ) exported_pipeline.fit(training_features, training_classes) diff --git a/tpot_test_config_dict.py b/tpot_test_config_dict.py index ede65c9a..ae2afa96 100644 --- a/tpot_test_config_dict.py +++ b/tpot_test_config_dict.py @@ -9,7 +9,8 @@ train_size=0.75, test_size=0.25) -tpot = TPOTClassifier(generations=4, population_size=20, verbosity=2, random_state = 42) +tpot = TPOTClassifier(generations=4, population_size=20, verbosity=2, + random_state = 42, operator_dict_file='config_classifier_test.py') time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) @@ -17,7 +18,7 @@ print('\nTime usages for TPOTClassifier:',time.time()-time_start) -X, y = make_regression(n_samples=200, n_features=50, +"""X, y = make_regression(n_samples=200, n_features=50, n_informative=10, n_targets=5, random_state=42) X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, test_size=0.25) @@ -28,3 +29,4 @@ print(tpot.score(X_test, y_test)) tpot.export('tpot_export_reg_conf.py') print('\nTime usages for TPOTRegressor:',time.time()-time_start) +""" From b4068f27593a4c6bd001c75643a1faadf77a32a7 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 30 Jan 2017 11:17:29 -0500 Subject: [PATCH 054/154] add driver --- tpot/base.py | 2 +- tpot/driver.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 79833f8a..ceed0240 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -143,7 +143,7 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, warm_start: bool (default: False) Flag indicating whether TPOT will reuse models from previous calls to fit() for faster operation - operator_dict_file: a file including a python dictionary (default: None + operator_dict_file: a file including a python dictionary (default: None) The python dictionary need to be named as classifier_config_dict for TPOTClassifier but regressor_config_dic for TPOTOperator The customized python dictionary to specify the list of operators and diff --git a/tpot/driver.py b/tpot/driver.py index 2272163a..ab1b888f 100644 --- a/tpot/driver.py +++ b/tpot/driver.py @@ -160,6 +160,10 @@ def main(): choices=[0, 1, 2, 3], type=int, help='How much information TPOT ' 'communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all.') + parser.add_argument('--operator_dict', dest='OPERATOR', default='', + type=str, help='File including a customized python dictionary to specify ' + 'operators and their arguments') + parser.add_argument('--no-update-check', action='store_true', dest='DISABLE_UPDATE_CHECK', default=False, help='Flag indicating whether the TPOT version checker should be disabled.') @@ -168,6 +172,9 @@ def main(): version='TPOT {version}'.format(version=__version__), help='Show TPOT\'s version number and exit.') + + + args = parser.parse_args() if args.VERBOSITY >= 2: @@ -206,7 +213,7 @@ def main(): scoring=args.SCORING_FN, max_time_mins=args.MAX_TIME_MINS, max_eval_time_mins=args.MAX_EVAL_MINS, random_state=args.RANDOM_STATE, verbosity=args.VERBOSITY, - disable_update_check=args.DISABLE_UPDATE_CHECK) + disable_update_check=args.DISABLE_UPDATE_CHECK, operator_dict_file=args.OPERATOR) tpot.fit(training_features, training_classes) From b11bc09d5acf40f454b790a4f88b61424e016d07 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 30 Jan 2017 14:44:52 -0500 Subject: [PATCH 055/154] fix unit tests (unfinished) --- tests.py | 87 +++++++++++++++++++++------------------ tpot/base.py | 2 - tpot/operator_utils.py | 11 ++--- tpot_export_class_conf.py | 13 +++--- 4 files changed, 58 insertions(+), 55 deletions(-) diff --git a/tests.py b/tests.py index 57ecdf36..ea234d55 100644 --- a/tests.py +++ b/tests.py @@ -22,10 +22,10 @@ from sklearn.datasets import load_digits, load_boston from sklearn.model_selection import train_test_split -from sklearn.ensemble import RandomForestClassifier from deap import creator from tqdm import tqdm +import deap # Set up the MNIST data set for testing mnist_data = load_digits() @@ -41,7 +41,7 @@ random.seed(42) test_operator_key = 'sklearn.feature_selection.SelectKBest' -TPOTSelectKBest = TPOTOperatorClassFactory(test_operator_key, +TPOTSelectKBest,TPOTSelectKBest_args = TPOTOperatorClassFactory(test_operator_key, classifier_config_dict[test_operator_key]) @@ -136,40 +136,40 @@ def test_score(): def test_score_2(): """Assert that the TPOTClassifier score function outputs a known score for a fixed pipeline""" -tpot_obj = TPOTClassifier() -tpot_obj._pbar = tqdm(total=1, disable=True) -known_score = 0.986318199045 # Assumes use of the TPOT balanced_accuracy function + tpot_obj = TPOTClassifier() + tpot_obj._pbar = tqdm(total=1, disable=True) + known_score = 0.91748994287679708 # Assumes use of the TPOT balanced_accuracy function -# Reify pipeline with known score -tpot_obj._optimized_pipeline = creator.Individual.\ - from_string("RandomForestClassifier(True, \'gini\', \'auto\', 1, 2, input_matrix)", tpot_obj._pset) -tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) -tpot_obj._fitted_pipeline.fit(training_features, training_classes) + # Reify pipeline with known score + tpot_obj._optimized_pipeline = creator.Individual.\ + from_string('GaussianNB(input_matrix)', tpot_obj._pset) + tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) + tpot_obj._fitted_pipeline.fit(training_features, training_classes) -# Get score from TPOT -score = tpot_obj.score(testing_features, testing_classes) -# http://stackoverflow.com/questions/5595425/ -def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): - return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + # Get score from TPOT + score = tpot_obj.score(testing_features, testing_classes) + + # http://stackoverflow.com/questions/5595425/ + def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) -assert isclose(known_score, score) + assert isclose(known_score, score) def test_score_3(): """Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline""" - tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') + tpot_obj = TPOTRegressor(scoring='mean_squared_error') tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 8.9673743407873712 # Assumes use of mse + known_score = 22.1748763753 # Assumes use of mse # Reify pipeline with known score tpot_obj._optimized_pipeline = creator.Individual.\ - from_string('ExtraTreesRegressor(GradientBoostingRegressor(input_matrix, 100.0, 0.11), 0.17999999999999999)', tpot_obj._pset) + from_string('RidgeCV(input_matrix)', tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r) # Get score from TPOT score = tpot_obj.score(testing_features_r, testing_classes_r) - # http://stackoverflow.com/questions/5595425/ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) @@ -181,10 +181,10 @@ def test_sample_weight_func(): tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 9.61954007496 # Assumes use of mse + known_score = 21.9145695521 # Assumes use of mse # Reify pipeline with known score tpot_obj._optimized_pipeline = creator.Individual.\ - from_string('ExtraTreesRegressor(GradientBoostingRegressor(input_matrix, 100.0, 0.11), 0.17999999999999999)', tpot_obj._pset) + from_string('RidgeCV(input_matrix)', tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) # make up a sample weight training_classes_r_weight = range(1, len(training_classes_r)+1) @@ -217,7 +217,7 @@ def test_predict_2(): tpot_obj = TPOTClassifier() tpot_obj._optimized_pipeline = creator.Individual.\ - from_string('DecisionTreeClassifier(input_matrix)', tpot_obj._pset) + from_string('GaussianNB(input_matrix)', tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -231,7 +231,7 @@ def test_predict_proba(): tpot_obj = TPOTClassifier() tpot_obj._optimized_pipeline = creator.Individual. \ - from_string('DecisionTreeClassifier(input_matrix)', tpot_obj._pset) + from_string('GaussianNB(input_matrix)', tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -246,7 +246,7 @@ def test_predict_proba2(): tpot_obj = TPOTClassifier() tpot_obj._optimized_pipeline = creator.Individual. \ - from_string('DecisionTreeClassifier(input_matrix)', tpot_obj._pset) + from_string('GaussianNB(input_matrix)', tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -290,9 +290,8 @@ def test_fit(): -def check_export(op): +def check_export(op, tpot_obj): """Assert that a TPOT operator exports as expected""" - tpot_obj = TPOTClassifier(random_state=42) prng = np.random.RandomState(42) np.random.seed(42) @@ -300,7 +299,6 @@ def check_export(op): args = [] for type_ in op.parameter_types()[0][1:]: args.append(prng.choice(tpot_obj._pset.terminals[type_]).value) - export_string = op.export(*args) assert export_string.startswith(op.__name__ + "(") and export_string.endswith(")") @@ -308,10 +306,11 @@ def check_export(op): def test_operators(): """Assert that the TPOT operators match the output of their sklearn counterparts""" - for op in Operator.inheritors(): + tpot_obj = TPOTClassifier(random_state=42) + for op in tpot_obj.operators: check_export.description = ("Assert that the TPOT {} operator exports " "as expected".format(op.__name__)) - yield check_export, op + yield check_export, op, tpot_obj def test_export(): @@ -326,23 +325,30 @@ def test_export(): def test_generate_pipeline_code(): + + tpot_obj = TPOTClassifier() """Assert that generate_pipeline_code() returns the correct code given a specific pipeline""" pipeline = ['KNeighborsClassifier', ['CombineDFs', ['GradientBoostingClassifier', 'input_matrix', 38.0, - 0.87], + 5, + 5, + 5, + 0.05, + 0.5], ['GaussianNB', ['ZeroCount', 'input_matrix']]], 18, - 33] + 'uniform', + 2] expected_code = """make_pipeline( make_union( make_union(VotingClassifier([('branch', - GradientBoostingClassifier(learning_rate=1.0, max_features=1.0, n_estimators=500) + GradientBoostingClassifier(learning_rate=38.0, max_depth=5, max_features=0.05, min_samples_leaf=5, min_samples_split=5, n_estimators=500, subsample=0.5) )]), FunctionTransformer(lambda X: X)), make_union(VotingClassifier([('branch', make_pipeline( @@ -351,10 +357,10 @@ def test_generate_pipeline_code(): ) )]), FunctionTransformer(lambda X: X)) ), - KNeighborsClassifier(n_neighbors=5, weights="distance") + KNeighborsClassifier(n_neighbors=18, p=2, weights="uniform") )""" - - assert expected_code == generate_pipeline_code(pipeline) + print(generate_pipeline_code(pipeline, tpot_obj.operators)) + assert expected_code == generate_pipeline_code(pipeline, tpot_obj.operators) def test_generate_import_code(): @@ -376,8 +382,9 @@ def test_generate_import_code(): training_features, testing_features, training_classes, testing_classes = \\ train_test_split(features, tpot_data['class'], random_state=42) """ + print(generate_import_code(pipeline, tpot_obj.operators)) - assert expected_code == generate_import_code(pipeline) + assert expected_code == generate_import_code(pipeline, tpot_obj.operators) def test_mutNodeReplacement(): """Assert that mutNodeReplacement() returns the correct type of mutation node in a fixed pipeline""" @@ -493,8 +500,8 @@ def test_export_pipeline_3(): def test_operator_export(): """Assert that a TPOT operator can export properly with a function as a parameter to a classifier""" - export_string = TPOTSelectKBest().export(5) - assert export_string == "SelectKBest(k=5, score_func=f_classif)" + export_string = TPOTSelectKBest.export(5) + assert export_string == "SelectKBest(score_func=f_classif, k=5)" def test_indent(): @@ -515,7 +522,7 @@ def test_indent(): def test_operator_type(): """Assert that TPOT operators return their type, e.g. "Classifier", "Preprocessor" """ - assert TPOTSelectKBest().type == "Selector" + assert TPOTSelectKBest.type() == "Preprocessor or Selector" def test_get_by_name(): diff --git a/tpot/base.py b/tpot/base.py index ceed0240..1e508cfe 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -144,8 +144,6 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, Flag indicating whether TPOT will reuse models from previous calls to fit() for faster operation operator_dict_file: a file including a python dictionary (default: None) - The python dictionary need to be named as classifier_config_dict for TPOTClassifier - but regressor_config_dic for TPOTOperator The customized python dictionary to specify the list of operators and their arguments. Format examples: config_regressor.py and config_classifier.py diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index acc9124b..918d71bc 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -27,13 +27,6 @@ class Operator(object): """Base class for operators in TPOT""" def __init__(self): pass - - @property - def __name__(self): - """Necessary for deap so that it can generate a string identifier for - each opeartor. - """ - return self.__class__.sklearn_class.__name__ root = False # Whether this operator type can be the root of the tree import_hash = None sklearn_class = None @@ -127,6 +120,7 @@ def op_type(): class_profile['type'] = op_type class_profile['sklearn_class'] = op_obj + import_hash = {} import_hash[import_str] = [op_str] arg_types = [] @@ -220,6 +214,7 @@ def export(*args): class_profile['export'] = export - op_classname = '_{}'.format(op_str) + op_classname = 'TPOT_{}'.format(op_str) op_class = type(op_classname, (BaseClass,), class_profile) + op_class.__name__ = op_str return op_class, arg_types diff --git a/tpot_export_class_conf.py b/tpot_export_class_conf.py index 1433f261..2b9feb05 100644 --- a/tpot_export_class_conf.py +++ b/tpot_export_class_conf.py @@ -1,9 +1,9 @@ import numpy as np -from sklearn.ensemble import ExtraTreesClassifier +from sklearn.ensemble import GradientBoostingClassifier, VotingClassifier from sklearn.model_selection import train_test_split -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import RobustScaler +from sklearn.pipeline import make_pipeline, make_union +from sklearn.preprocessing import FunctionTransformer # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -12,8 +12,11 @@ train_test_split(features, tpot_data['class'], random_state=42) exported_pipeline = make_pipeline( - RobustScaler(), - ExtraTreesClassifier(bootstrap=False, criterion="entropy", max_features=0.9500000000000001, min_samples_leaf=3, min_samples_split=20) + make_union( + FunctionTransformer(lambda X: X), + FunctionTransformer(lambda X: X) + ), + GradientBoostingClassifier(learning_rate=0.5, max_depth=5, max_features=0.8500000000000001, min_samples_leaf=17, min_samples_split=11, subsample=0.45) ) exported_pipeline.fit(training_features, training_classes) From 372443b9676f7ec103acc608385084ba5b42d155 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 30 Jan 2017 17:04:01 -0500 Subject: [PATCH 056/154] unit test fixed --- config_classifier_test.py | 188 -------------------------------------- tests.py | 96 ++++++++++++------- tpot_export_class_conf.py | 23 ----- tpot_export_reg_conf.py | 15 --- tpot_test_config_dict.py | 32 ------- 5 files changed, 62 insertions(+), 292 deletions(-) delete mode 100644 config_classifier_test.py delete mode 100644 tpot_export_class_conf.py delete mode 100644 tpot_export_reg_conf.py delete mode 100644 tpot_test_config_dict.py diff --git a/config_classifier_test.py b/config_classifier_test.py deleted file mode 100644 index c4de25ef..00000000 --- a/config_classifier_test.py +++ /dev/null @@ -1,188 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Copyright 2016 Randal S. Olson - -This file is part of the TPOT library. - -The TPOT library is free software: you can redistribute it and/or -modify it under the terms of the GNU General Public License as published by the -Free Software Foundation, either version 3 of the License, or (at your option) -any later version. - -The TPOT library is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -details. You should have received a copy of the GNU General Public License along -with the TPOT library. If not, see http://www.gnu.org/licenses/. - -""" - -""" -dictionary format (json-like format): -key: - operator name -value: - source: module source (e.g sklearn.tree) - dependencies: depended module (e.g. SVC in selectors RFE); None for no dependency - params: a dictionary of parameter names (keys) and parameter ranges (values); None for no dependency -""" -import numpy as np - -classifier_config_dict = { - - # Classifiers - 'sklearn.naive_bayes.GaussianNB': { - }, - - 'sklearn.naive_bayes.BernoulliNB': { - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False] - }, - - 'sklearn.naive_bayes.MultinomialNB': { - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False] - }, - - 'sklearn.tree.DecisionTreeClassifier': { - 'criterion': ["gini", "entropy"], - 'max_depth': range(1, 11), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21) - }, - - 'sklearn.ensemble.ExtraTreesClassifier': { - 'criterion': ["gini", "entropy"], - 'max_features': np.arange(0, 1.01, 0.05), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'bootstrap': [True, False] - }, - - 'sklearn.ensemble.RandomForestClassifier': { - 'criterion': ["gini", "entropy"], - 'max_features': np.arange(0, 1.01, 0.05), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'bootstrap': [True, False] - }, - - 'sklearn.ensemble.GradientBoostingClassifier': { - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'max_depth': range(1, 11), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'subsample': np.arange(0.05, 1.01, 0.05), - 'max_features': np.arange(0, 1.01, 0.05) - }, - - 'sklearn.neighbors.KNeighborsClassifier': { - 'n_neighbors': range(1, 101), - 'weights': ["uniform", "distance"], - 'p': [1, 2] - }, - - 'sklearn.svm.LinearSVC': { - 'penalty': ["l1", "l2"], - 'loss': ["hinge", "squared_hinge"], - 'dual': [True, False], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] - }, - - 'sklearn.linear_model.LogisticRegression': { - 'penalty': ["l1", "l2"], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], - 'dual': [True, False] - }, - - 'xgboost.XGBClassifier': { - 'max_depth': range(1, 11), - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'subsample': np.arange(0.05, 1.01, 0.05), - 'min_child_weight': range(1, 21) - }, - - # Preprocesssors - 'sklearn.preprocessing.Binarizer': { - 'threshold': np.arange(0.0, 1.01, 0.05) - }, - - 'sklearn.decomposition.FastICA': { - 'tol': np.arange(0.0, 1.01, 0.05) - }, - - 'sklearn.cluster.FeatureAgglomeration': { - 'linkage': ['ward', 'complete', 'average'], - 'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine', 'precomputed'] - }, - - 'sklearn.preprocessing.MaxAbsScaler': { - }, - - 'sklearn.preprocessing.MinMaxScaler': { - }, - - 'sklearn.preprocessing.Normalizer': { - 'norm': ['l1', 'l2', 'max'] - }, - - 'sklearn.kernel_approximation.Nystroem': { - 'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'], - 'gamma': np.arange(0.0, 1.01, 0.05), - 'n_components': range(1, 11) - }, - - 'sklearn.decomposition.PCA': { - 'svd_solver': ['randomized'], - 'iterated_power': range(1, 11) - }, - - 'sklearn.preprocessing.PolynomialFeatures': { - 'degree': [2], - 'include_bias': [False], - 'interaction_only': [False] - }, - - 'sklearn.kernel_approximation.RBFSampler': { - 'gamma': np.arange(0.0, 1.01, 0.05) - }, - - 'sklearn.preprocessing.RobustScaler': { - }, - - 'sklearn.preprocessing.StandardScaler': { - }, - - 'tpot.build_in_operators.ZeroCount': { - }, - - # Selectors - 'sklearn.feature_selection.SelectFwe': { - 'alpha': np.arange(0, 0.05, 0.001), - 'score_func': { - 'sklearn.feature_selection.f_classif': None - } # read from dependencies ! need add an exception in preprocess_args - - }, - - 'sklearn.feature_selection.SelectKBest': { - 'k': range(1, 100), # need check range! - 'score_func': { - 'sklearn.feature_selection.f_classif': None - } - }, - - 'sklearn.feature_selection.SelectPercentile': { - 'percentile': range(1, 100), - 'score_func': { - 'sklearn.feature_selection.f_classif': None - } - }, - - 'sklearn.feature_selection.VarianceThreshold': { - 'threshold': np.arange(0.05, 1.01, 0.05) - } - -} diff --git a/tests.py b/tests.py index ea234d55..531827e3 100644 --- a/tests.py +++ b/tests.py @@ -7,7 +7,7 @@ from tpot import TPOTClassifier, TPOTRegressor from tpot.base import TPOTBase from tpot.driver import positive_integer, float_range -from tpot.export_utils import export_pipeline, generate_import_code, _indent, generate_pipeline_code +from tpot.export_utils import export_pipeline, generate_import_code, _indent, generate_pipeline_code, get_by_name from tpot.gp_types import Output_DF from tpot.gp_deap import mutNodeReplacement @@ -159,7 +159,7 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): def test_score_3(): """Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline""" - tpot_obj = TPOTRegressor(scoring='mean_squared_error') + tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') tpot_obj._pbar = tqdm(total=1, disable=True) known_score = 22.1748763753 # Assumes use of mse # Reify pipeline with known score @@ -289,6 +289,40 @@ def test_fit(): assert not (tpot_obj._start_datetime is None) +def testTPOTOperatorClassFactory(): + """Assert that the TPOT operators class factory""" + test_operator_dict = { + 'sklearn.svm.LinearSVC': { + 'penalty': ["l1", "l2"], + 'loss': ["hinge", "squared_hinge"], + 'dual': [True, False], + 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] + }, + + 'sklearn.linear_model.LogisticRegression': { + 'penalty': ["l1", "l2"], + 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], + 'dual': [True, False] + }, + + 'sklearn.preprocessing.Binarizer': { + 'threshold': np.arange(0.0, 1.01, 0.05) + } + } + tpot_operator_list = [] + tpot_argument_list = [] + for key in sorted(test_operator_dict.keys()): + op,args = TPOTOperatorClassFactory(key,test_operator_dict[key]) + tpot_operator_list.append(op) + tpot_argument_list += args + assert len(tpot_operator_list) == 3 + assert len(tpot_argument_list) == 9 + assert tpot_operator_list[0].root == True + assert tpot_operator_list[1].root == False + assert tpot_operator_list[2].type() == "Classifier or Regressor" + assert tpot_argument_list[1].values == [True, False] + def check_export(op, tpot_obj): """Assert that a TPOT operator exports as expected""" @@ -348,7 +382,7 @@ def test_generate_pipeline_code(): expected_code = """make_pipeline( make_union( make_union(VotingClassifier([('branch', - GradientBoostingClassifier(learning_rate=38.0, max_depth=5, max_features=0.05, min_samples_leaf=5, min_samples_split=5, n_estimators=500, subsample=0.5) + GradientBoostingClassifier(learning_rate=38.0, max_depth=5, max_features=5, min_samples_leaf=5, min_samples_split=0.05, subsample=0.5) )]), FunctionTransformer(lambda X: X)), make_union(VotingClassifier([('branch', make_pipeline( @@ -357,9 +391,8 @@ def test_generate_pipeline_code(): ) )]), FunctionTransformer(lambda X: X)) ), - KNeighborsClassifier(n_neighbors=18, p=2, weights="uniform") + KNeighborsClassifier(n_neighbors=18, p="uniform", weights=2) )""" - print(generate_pipeline_code(pipeline, tpot_obj.operators)) assert expected_code == generate_pipeline_code(pipeline, tpot_obj.operators) @@ -367,14 +400,14 @@ def test_generate_import_code(): """Assert that generate_import_code() returns the correct set of dependancies for a given pipeline""" tpot_obj = TPOTClassifier() pipeline = creator.Individual.\ - from_string('DecisionTreeClassifier(SelectKBest(input_matrix, 7))', tpot_obj._pset) + from_string('GaussianNB(RobustScaler(input_matrix))', tpot_obj._pset) expected_code = """import numpy as np -from sklearn.feature_selection import SelectKBest, f_classif from sklearn.model_selection import train_test_split +from sklearn.naive_bayes import GaussianNB from sklearn.pipeline import make_pipeline -from sklearn.tree import DecisionTreeClassifier +from sklearn.preprocessing import RobustScaler # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -382,8 +415,6 @@ def test_generate_import_code(): training_features, testing_features, training_classes, testing_classes = \\ train_test_split(features, tpot_data['class'], random_state=42) """ - print(generate_import_code(pipeline, tpot_obj.operators)) - assert expected_code == generate_import_code(pipeline, tpot_obj.operators) def test_mutNodeReplacement(): @@ -410,16 +441,16 @@ def test_export_pipeline(): """Assert that exported_pipeline() generated a compile source file as expected given a fixed complex pipeline""" tpot_obj = TPOTClassifier() pipeline = creator.Individual.\ - from_string("KNeighborsClassifier(CombineDFs(GradientBoostingClassifier(input_matrix, 38.0, 0.87), SelectKBest(input_matrix, 5)), 18, 33)", tpot_obj._pset) + from_string("GaussianNB(CombineDFs(ZeroCount(input_matrix), RobustScaler(input_matrix)))", tpot_obj._pset) expected_code = """import numpy as np -from sklearn.ensemble import GradientBoostingClassifier, VotingClassifier -from sklearn.feature_selection import SelectKBest, f_classif +from sklearn.ensemble import VotingClassifier from sklearn.model_selection import train_test_split -from sklearn.neighbors import KNeighborsClassifier +from sklearn.naive_bayes import GaussianNB from sklearn.pipeline import make_pipeline, make_union -from sklearn.preprocessing import FunctionTransformer +from sklearn.preprocessing import FunctionTransformer, RobustScaler +from tpot.build_in_operators import ZeroCount # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -429,30 +460,27 @@ def test_export_pipeline(): exported_pipeline = make_pipeline( make_union( - make_union(VotingClassifier([('branch', - GradientBoostingClassifier(learning_rate=1.0, max_features=1.0, n_estimators=500) - )]), FunctionTransformer(lambda X: X)), - SelectKBest(k=5, score_func=f_classif) + ZeroCount(), + RobustScaler() ), - KNeighborsClassifier(n_neighbors=5, weights="distance") + GaussianNB() ) exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) """ - - assert expected_code == export_pipeline(pipeline) + assert expected_code == export_pipeline(pipeline,tpot_obj.operators) def test_export_pipeline_2(): """Assert that exported_pipeline() generated a compile source file as expected given a fixed simple pipeline (only one classifier)""" tpot_obj = TPOTClassifier() pipeline = creator.Individual.\ - from_string("KNeighborsClassifier(input_matrix, 18, 33)", tpot_obj._pset) + from_string("GaussianNB(input_matrix)", tpot_obj._pset) expected_code = """import numpy as np from sklearn.model_selection import train_test_split -from sklearn.neighbors import KNeighborsClassifier +from sklearn.naive_bayes import GaussianNB # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -460,26 +488,25 @@ def test_export_pipeline_2(): training_features, testing_features, training_classes, testing_classes = \\ train_test_split(features, tpot_data['class'], random_state=42) -exported_pipeline = KNeighborsClassifier(n_neighbors=5, weights="distance") +exported_pipeline = GaussianNB() exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) """ - - assert expected_code == export_pipeline(pipeline) + assert expected_code == export_pipeline(pipeline, tpot_obj.operators) def test_export_pipeline_3(): """Assert that exported_pipeline() generated a compile source file as expected given a fixed simple pipeline with a preprocessor""" tpot_obj = TPOTClassifier() pipeline = creator.Individual.\ - from_string("DecisionTreeClassifier(SelectKBest(input_matrix, 5))", tpot_obj._pset) + from_string("GaussianNB(MaxAbsScaler(input_matrix))", tpot_obj._pset) expected_code = """import numpy as np -from sklearn.feature_selection import SelectKBest, f_classif from sklearn.model_selection import train_test_split +from sklearn.naive_bayes import GaussianNB from sklearn.pipeline import make_pipeline -from sklearn.tree import DecisionTreeClassifier +from sklearn.preprocessing import MaxAbsScaler # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -488,14 +515,14 @@ def test_export_pipeline_3(): train_test_split(features, tpot_data['class'], random_state=42) exported_pipeline = make_pipeline( - SelectKBest(k=5, score_func=f_classif), - DecisionTreeClassifier() + MaxAbsScaler(), + GaussianNB() ) exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) """ - assert expected_code == export_pipeline(pipeline) + assert expected_code == export_pipeline(pipeline, tpot_obj.operators) def test_operator_export(): @@ -527,7 +554,8 @@ def test_operator_type(): def test_get_by_name(): """Assert that the Operator class returns operators by name appropriately""" - assert Operator.get_by_name("SelectKBest").__class__ == TPOTSelectKBest + tpot_obj = TPOTClassifier() + assert get_by_name("SelectKBest", tpot_obj.operators).__class__ == TPOTSelectKBest.__class__ def test_gen(): diff --git a/tpot_export_class_conf.py b/tpot_export_class_conf.py deleted file mode 100644 index 2b9feb05..00000000 --- a/tpot_export_class_conf.py +++ /dev/null @@ -1,23 +0,0 @@ -import numpy as np - -from sklearn.ensemble import GradientBoostingClassifier, VotingClassifier -from sklearn.model_selection import train_test_split -from sklearn.pipeline import make_pipeline, make_union -from sklearn.preprocessing import FunctionTransformer - -# NOTE: Make sure that the class is labeled 'class' in the data file -tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) -features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) -training_features, testing_features, training_classes, testing_classes = \ - train_test_split(features, tpot_data['class'], random_state=42) - -exported_pipeline = make_pipeline( - make_union( - FunctionTransformer(lambda X: X), - FunctionTransformer(lambda X: X) - ), - GradientBoostingClassifier(learning_rate=0.5, max_depth=5, max_features=0.8500000000000001, min_samples_leaf=17, min_samples_split=11, subsample=0.45) -) - -exported_pipeline.fit(training_features, training_classes) -results = exported_pipeline.predict(testing_features) diff --git a/tpot_export_reg_conf.py b/tpot_export_reg_conf.py deleted file mode 100644 index b94ce290..00000000 --- a/tpot_export_reg_conf.py +++ /dev/null @@ -1,15 +0,0 @@ -import numpy as np - -from sklearn.linear_model import RidgeCV -from sklearn.model_selection import train_test_split - -# NOTE: Make sure that the class is labeled 'class' in the data file -tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) -features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) -training_features, testing_features, training_classes, testing_classes = \ - train_test_split(features, tpot_data['class'], random_state=42) - -exported_pipeline = RidgeCV() - -exported_pipeline.fit(training_features, training_classes) -results = exported_pipeline.predict(testing_features) diff --git a/tpot_test_config_dict.py b/tpot_test_config_dict.py deleted file mode 100644 index ae2afa96..00000000 --- a/tpot_test_config_dict.py +++ /dev/null @@ -1,32 +0,0 @@ -from tpot import TPOTClassifier, TPOTRegressor -from sklearn.datasets import make_classification, make_regression -from sklearn.model_selection import train_test_split -import time - -X, y = make_classification(n_samples=200, n_features=50, - n_informative=10, n_redundant=10, random_state=42) -X_train, X_test, y_train, y_test = train_test_split(X, y, - train_size=0.75, test_size=0.25) - - -tpot = TPOTClassifier(generations=4, population_size=20, verbosity=2, - random_state = 42, operator_dict_file='config_classifier_test.py') -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -tpot.export('tpot_export_class_conf.py') -print('\nTime usages for TPOTClassifier:',time.time()-time_start) - - -"""X, y = make_regression(n_samples=200, n_features=50, - n_informative=10, n_targets=5, random_state=42) -X_train, X_test, y_train, y_test = train_test_split(X, y, - train_size=0.75, test_size=0.25) - -tpot = TPOTRegressor(generations=4, population_size=20, verbosity=2, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -tpot.export('tpot_export_reg_conf.py') -print('\nTime usages for TPOTRegressor:',time.time()-time_start) -""" From b5449906acdeca82e1fd2f87b66ff17a30c848cc Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 30 Jan 2017 17:24:30 -0500 Subject: [PATCH 057/154] add reg decorators --- tpot/decorators.py | 8 ++++++-- tpot_export_class_conf.py | 15 +++++++++++++++ tpot_export_reg_conf.py | 20 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 tpot_export_class_conf.py create mode 100644 tpot_export_reg_conf.py diff --git a/tpot/decorators.py b/tpot/decorators.py index 0b515f09..5f9c3d28 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -22,11 +22,12 @@ from functools import wraps import sys import warnings -from sklearn.datasets import make_classification +from sklearn.datasets import make_classification, make_regression from .export_utils import expr_to_tree, generate_pipeline_code # generate a small data set for a new pipeline, in order to check if the pipeline # has unsuppported combinations in params pretest_X, pretest_y = make_classification(n_samples=50, n_features=10, random_state=42) +pretest_X_reg, pretest_y_reg = make_regression(n_samples=50, n_features=10, random_state=42) def _timeout(func): """Runs a function with time limit @@ -158,7 +159,10 @@ def check_pipeline(self, *args, **kwargs): expr = func(self, *args, **kwargs) #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) # debug sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(expr), self.operators), self.operators_context) - sklearn_pipeline.fit(pretest_X, pretest_y) + if self.classification: + sklearn_pipeline.fit(pretest_X, pretest_y) + else: + sklearn_pipeline.fit(pretest_X_reg, pretest_y_reg) bad_pipeline = False except: pass diff --git a/tpot_export_class_conf.py b/tpot_export_class_conf.py new file mode 100644 index 00000000..c0ee54b5 --- /dev/null +++ b/tpot_export_class_conf.py @@ -0,0 +1,15 @@ +import numpy as np + +from sklearn.model_selection import train_test_split +from sklearn.neighbors import KNeighborsClassifier + +# NOTE: Make sure that the class is labeled 'class' in the data file +tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) +features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) +training_features, testing_features, training_classes, testing_classes = \ + train_test_split(features, tpot_data['class'], random_state=42) + +exported_pipeline = KNeighborsClassifier(n_neighbors=32, p=1, weights="distance") + +exported_pipeline.fit(training_features, training_classes) +results = exported_pipeline.predict(testing_features) diff --git a/tpot_export_reg_conf.py b/tpot_export_reg_conf.py new file mode 100644 index 00000000..afef0b64 --- /dev/null +++ b/tpot_export_reg_conf.py @@ -0,0 +1,20 @@ +import numpy as np + +from sklearn.linear_model import RidgeCV +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from tpot.build_in_operators import ZeroCount + +# NOTE: Make sure that the class is labeled 'class' in the data file +tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) +features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) +training_features, testing_features, training_classes, testing_classes = \ + train_test_split(features, tpot_data['class'], random_state=42) + +exported_pipeline = make_pipeline( + ZeroCount(), + RidgeCV() +) + +exported_pipeline.fit(training_features, training_classes) +results = exported_pipeline.predict(testing_features) From 4100d095edc43a66ad3f01a6f9789a20730efa08 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 30 Jan 2017 17:25:01 -0500 Subject: [PATCH 058/154] clean codes --- tpot_export_class_conf.py | 15 --------------- tpot_export_reg_conf.py | 20 -------------------- 2 files changed, 35 deletions(-) delete mode 100644 tpot_export_class_conf.py delete mode 100644 tpot_export_reg_conf.py diff --git a/tpot_export_class_conf.py b/tpot_export_class_conf.py deleted file mode 100644 index c0ee54b5..00000000 --- a/tpot_export_class_conf.py +++ /dev/null @@ -1,15 +0,0 @@ -import numpy as np - -from sklearn.model_selection import train_test_split -from sklearn.neighbors import KNeighborsClassifier - -# NOTE: Make sure that the class is labeled 'class' in the data file -tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) -features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) -training_features, testing_features, training_classes, testing_classes = \ - train_test_split(features, tpot_data['class'], random_state=42) - -exported_pipeline = KNeighborsClassifier(n_neighbors=32, p=1, weights="distance") - -exported_pipeline.fit(training_features, training_classes) -results = exported_pipeline.predict(testing_features) diff --git a/tpot_export_reg_conf.py b/tpot_export_reg_conf.py deleted file mode 100644 index afef0b64..00000000 --- a/tpot_export_reg_conf.py +++ /dev/null @@ -1,20 +0,0 @@ -import numpy as np - -from sklearn.linear_model import RidgeCV -from sklearn.model_selection import train_test_split -from sklearn.pipeline import make_pipeline -from tpot.build_in_operators import ZeroCount - -# NOTE: Make sure that the class is labeled 'class' in the data file -tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) -features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) -training_features, testing_features, training_classes, testing_classes = \ - train_test_split(features, tpot_data['class'], random_state=42) - -exported_pipeline = make_pipeline( - ZeroCount(), - RidgeCV() -) - -exported_pipeline.fit(training_features, training_classes) -results = exported_pipeline.predict(testing_features) From ff9b5286b033129d87d95a9af9a005bceb78690f Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 30 Jan 2017 18:08:30 -0500 Subject: [PATCH 059/154] linux and mac has different value in RigdeCV --- tests.py | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/tests.py b/tests.py index 531827e3..730a6ad0 100644 --- a/tests.py +++ b/tests.py @@ -156,49 +156,6 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): assert isclose(known_score, score) -def test_score_3(): - """Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline""" - - tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') - tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 22.1748763753 # Assumes use of mse - # Reify pipeline with known score - tpot_obj._optimized_pipeline = creator.Individual.\ - from_string('RidgeCV(input_matrix)', tpot_obj._pset) - tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) - tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r) - - # Get score from TPOT - score = tpot_obj.score(testing_features_r, testing_classes_r) - # http://stackoverflow.com/questions/5595425/ - def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): - return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) - - assert isclose(known_score, score) - -def test_sample_weight_func(): - """Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline with sample weights""" - - tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') - tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 21.9145695521 # Assumes use of mse - # Reify pipeline with known score - tpot_obj._optimized_pipeline = creator.Individual.\ - from_string('RidgeCV(input_matrix)', tpot_obj._pset) - tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) - # make up a sample weight - training_classes_r_weight = range(1, len(training_classes_r)+1) - training_classes_r_weight_dict = tpot_obj._set_param_recursive(tpot_obj._fitted_pipeline .steps, 'random_state', 42, training_classes_r_weight) - tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r, **training_classes_r_weight_dict) - - # Get score from TPOT - score = tpot_obj.score(testing_features_r, testing_classes_r) - # http://stackoverflow.com/questions/5595425/ - def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): - return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) - - assert isclose(known_score, score) - def test_predict(): """Assert that the TPOT predict function raises a ValueError when no optimized pipeline exists""" From fb97330384ddb84b66dae1780cfefa2f551ef0ef Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 31 Jan 2017 11:07:01 -0500 Subject: [PATCH 060/154] fix python2.7 suppurt --- tpot/operator_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 918d71bc..3648c5e6 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -110,8 +110,8 @@ def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator): optype = "Classifier or Regressor" else: optype = "Preprocessor or Selector" - - def op_type(): + @classmethod + def op_type(cls): """Returns the type of the operator, e.g: ("Classifier", "Regressor", "Selector", "Preprocessor") """ @@ -145,8 +145,8 @@ def op_type(): class_profile['arg_types'] = tuple(arg_types) class_profile['import_hash'] = import_hash class_profile['dep_op_list'] = dep_op_list - - def parameter_types(): + @classmethod + def parameter_types(cls): """Return tuple of argument types for calling of the operator and the return type of the operator @@ -165,8 +165,8 @@ def parameter_types(): class_profile['parameter_types'] = parameter_types - - def export(*args): + @classmethod + def export(cls, *args): """Represent the operator as a string so that it can be exported to a file From 1f622eab7db0c310d33e8853ece08aa6366c01b1 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 31 Jan 2017 11:10:12 -0500 Subject: [PATCH 061/154] py2.7 support --- tpot/operator_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 3648c5e6..5669cf3c 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -34,11 +34,10 @@ def __init__(self): dep_op_list = {} # the estimator or score_func as params in this operators - class ARGType(object): - """Base class for parameter specifications""" - def __init__(self): - pass + """Base class for parameter specifications""" + def __init__(self): + pass def source_decode(sourcecode): From 9f5277b904156b2e6aa0dc4a272f372ffccdacf8 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 31 Jan 2017 14:45:35 -0500 Subject: [PATCH 062/154] clean old dict --- opt_dict/config_classifier.txt | 86 ---------------------------------- opt_dict/config_regressor.txt | 74 ----------------------------- opt_dict/opt_demo.txt | 24 ---------- opt_dict/opt_input_control.py | 2 - 4 files changed, 186 deletions(-) delete mode 100644 opt_dict/config_classifier.txt delete mode 100644 opt_dict/config_regressor.txt delete mode 100644 opt_dict/opt_demo.txt delete mode 100644 opt_dict/opt_input_control.py diff --git a/opt_dict/config_classifier.txt b/opt_dict/config_classifier.txt deleted file mode 100644 index 7fb86045..00000000 --- a/opt_dict/config_classifier.txt +++ /dev/null @@ -1,86 +0,0 @@ -{ - - 'sklearn.tree.DecisionTreeClassifier': { - 'criterion': ['gini', 'entropy'], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - }, - - 'sklearn.naive_bayes.BernoulliNB': { - 'criterion': ['gini', 'entropy'], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - }, - - 'sklearn.naive_bayes.GaussianNB': { - "criterion": ['gini', 'entropy'], - "max_features": [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - "bootstrap": [True, False] - }, - - 'sklearn.ensemble.ExtraTreesClassifier': { - "criterion": ["gini", "entropy"], - "max_features": [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - "bootstrap": [True, False] - }, - - 'sklearn.ensemble.GradientBoostingClassifier': { - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'subsample': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.] - }, - - 'sklearn.neighbors.KNeighborsClassifier': { - 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'weights': ["uniform", "distance"], - 'p': [1, 2] - }, - - 'sklearn.svm.LinearSVC': { - 'penalty': ["l1", "l2"], - 'loss': ["hinge", "squared_hinge"], - 'dual': [True, False], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.] - }, - - - 'sklearn.linear_model.LogisticRegression': { - 'penalty': ["l1", "l2"], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], - 'dual': [True, False] - }, - - 'sklearn.naive_bayes.MultinomialNB': { - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False] - }, - - 'sklearn.ensemble.RandomForestClassifier': { - 'criterion': ["gini", "entropy"], - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], - 'bootstrap': [True, False] - } -} - diff --git a/opt_dict/config_regressor.txt b/opt_dict/config_regressor.txt deleted file mode 100644 index 65e4a417..00000000 --- a/opt_dict/config_regressor.txt +++ /dev/null @@ -1,74 +0,0 @@ -{ - - 'sklearn.linear_model.ElasticNetCV': { - 'l1_ratio': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1] - }, - - 'sklearn.ensemble.ExtraTreesRegressor': { - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'bootstrap': [True, False] - }, - - 'sklearn.ensemble.GradientBoostingRegressor': { - 'loss': ["ls", "lad", "huber", "quantile"], - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'subsample':[0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'alpha': [0.75, 0.8, 0.85, 0.9, 0.95, 0.99] - }, - - 'sklearn.ensemble.AdaBoostRegressor': { - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - 'loss': ["linear", "square", "exponential"], - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - }, - - 'sklearn.tree.DecisionTreeRegressor': { - 'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - }, - - 'sklearn.neighbors.KNeighborsRegressor': { - 'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'weights': ["uniform", "distance"], - 'p': [1, 2] - }, - - 'sklearn.linear_model.LassoLarsCV': { - 'normalize': [True, False] - }, - - 'sklearn.svm.LinearSVR': { - 'loss': ["epsilon_insensitive", "squared_epsilon_insensitive"], - 'dual': [True, False], - 'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1], - 'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.], - 'epsilon': [1e-4, 1e-3, 1e-2, 1e-1, 1.] - }, - - 'sklearn.ensemble.RandomForestRegressor': { - 'max_features': [0., 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, - 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, - 0.9, 0.95, 1.], - 'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'min_samples_leaf': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], - 'bootstrap': [True, False] - }, - - 'sklearn.linear_model.RidgeCV': {} -} \ No newline at end of file diff --git a/opt_dict/opt_demo.txt b/opt_dict/opt_demo.txt deleted file mode 100644 index 95109218..00000000 --- a/opt_dict/opt_demo.txt +++ /dev/null @@ -1,24 +0,0 @@ -{ - sklearn...'DecisionTree': { - 'Type': ['Root', 'Classifier'], #--> op.root = True, op.classification = True, op.regression = False - 'Params': { - 'criterion': ["gini", "entropy"], - 'max_depth': range(1, 11), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - } - }, - - 'BernoulliNB': { - 'Type': ['Root', 'Classifier'], #--> op.root = True, op.classification = True, op.regression = False - 'Params': { - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False], - } - }, - - 'AdaBoost': { - 'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.], - - } -} diff --git a/opt_dict/opt_input_control.py b/opt_dict/opt_input_control.py deleted file mode 100644 index 77c890d3..00000000 --- a/opt_dict/opt_input_control.py +++ /dev/null @@ -1,2 +0,0 @@ -# import control of opt -# exec() From 5e52906d83e0ea6d7878be5dcd16d5b856ac3999 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Feb 2017 16:29:45 -0500 Subject: [PATCH 063/154] random seed for unit test works --- tests.py | 65 +++++++++++++++++++++++++++++++++++++++------- tpot/decorators.py | 2 -- 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/tests.py b/tests.py index 730a6ad0..86ed9bf6 100644 --- a/tests.py +++ b/tests.py @@ -134,15 +134,15 @@ def test_score(): def test_score_2(): - """Assert that the TPOTClassifier score function outputs a known score for a fixed pipeline""" + """Assert that the TPOTClassifier score function outputs a known score for a ramdom pipeline""" tpot_obj = TPOTClassifier() tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 0.91748994287679708 # Assumes use of the TPOT balanced_accuracy function + known_score = 0.96710588996037627 # Assumes use of the TPOT balanced_accuracy function # Reify pipeline with known score - tpot_obj._optimized_pipeline = creator.Individual.\ - from_string('GaussianNB(input_matrix)', tpot_obj._pset) + np.random.seed(43) + tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -155,6 +155,51 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): assert isclose(known_score, score) +def test_score_3(): + """Assert that the TPOTRegressor score function outputs a known score for a random pipeline""" + + tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') + tpot_obj._pbar = tqdm(total=1, disable=True) + known_score = 14.375172822194937 # Assumes use of mse + + # Reify pipeline with known score + np.random.seed(45) + tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() + tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) + tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r) + + # Get score from TPOT + score = tpot_obj.score(testing_features_r, testing_classes_r) + + # http://stackoverflow.com/questions/5595425/ + def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + + assert isclose(known_score, score) + +def test_sample_weight_func(): + """Assert that the TPOTRegressor score function outputs a known score for a random pipeline with sample weights""" + + tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') + tpot_obj._pbar = tqdm(total=1, disable=True) + known_score = 13.672380235317991 # Assumes use of mse + # Reify pipeline with known score + np.random.seed(45) + tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() + tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) + # make up a sample weight + training_classes_r_weight = range(1, len(training_classes_r)+1) + training_classes_r_weight_dict = tpot_obj._set_param_recursive(tpot_obj._fitted_pipeline .steps, 'random_state', 42, training_classes_r_weight) + tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r, **training_classes_r_weight_dict) + + # Get score from TPOT + score = tpot_obj.score(testing_features_r, testing_classes_r) + # http://stackoverflow.com/questions/5595425/ + def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + + assert isclose(known_score, score) + def test_predict(): @@ -173,8 +218,8 @@ def test_predict_2(): """Assert that the TPOT predict function returns a numpy matrix of shape (num_testing_rows,)""" tpot_obj = TPOTClassifier() - tpot_obj._optimized_pipeline = creator.Individual.\ - from_string('GaussianNB(input_matrix)', tpot_obj._pset) + np.random.seed(49) + tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -187,8 +232,8 @@ def test_predict_proba(): """Assert that the TPOT predict_proba function returns a numpy matrix of shape (num_testing_rows, num_testing_classes)""" tpot_obj = TPOTClassifier() - tpot_obj._optimized_pipeline = creator.Individual. \ - from_string('GaussianNB(input_matrix)', tpot_obj._pset) + np.random.seed(51) + tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -202,8 +247,8 @@ def test_predict_proba2(): """Assert that the TPOT predict_proba function returns a numpy matrix filled with probabilities (float)""" tpot_obj = TPOTClassifier() - tpot_obj._optimized_pipeline = creator.Individual. \ - from_string('GaussianNB(input_matrix)', tpot_obj._pset) + np.random.seed(53) + tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) diff --git a/tpot/decorators.py b/tpot/decorators.py index 5f9c3d28..40a5742a 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -150,8 +150,6 @@ def _pre_test(func): def check_pipeline(self, *args, **kwargs): bad_pipeline = True num_test = 0 # number of tests - """with warnings.catch_warnings(): - warnings.simplefilter('ignore')""" while bad_pipeline and num_test < 10: # a pool for workable pipeline try: with warnings.catch_warnings(): From 8fa80d679b93c9b3a14ee82f8df7bbf9f72bec79 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Feb 2017 16:49:26 -0500 Subject: [PATCH 064/154] add more tests in random pipeline --- tests.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/tests.py b/tests.py index 86ed9bf6..a1cfc260 100644 --- a/tests.py +++ b/tests.py @@ -120,6 +120,48 @@ def test_set_params_2(): assert tpot_obj.generations == 3 +def test_random_ind(): + """Assert that the TPOTClassifier can generate the same pipeline with same random seed""" + + tpot_obj = TPOTClassifier() + tpot_obj._pbar = tqdm(total=1, disable=True) + + np.random.seed(43) + pipeline1 = tpot_obj._toolbox.individual() + np.random.seed(43) + pipeline2 = tpot_obj._toolbox.individual() + + assert pipeline1 == pipeline2 + +def test_random_ind_2(): + """Assert that the TPOTClassifier can generate the same pipeline export with random seed of 45""" + + tpot_obj = TPOTClassifier() + tpot_obj._pbar = tqdm(total=1, disable=True) + np.random.seed(45) + pipeline = tpot_obj._toolbox.individual() + expected_code = """import numpy as np + +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from tpot.build_in_operators import ZeroCount + +# NOTE: Make sure that the class is labeled 'class' in the data file +tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) +features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1) +training_features, testing_features, training_classes, testing_classes = \\ + train_test_split(features, tpot_data['class'], random_state=42) + +exported_pipeline = make_pipeline( + ZeroCount(), + LogisticRegression(C=0.0001, dual=False, penalty="l2") +) + +exported_pipeline.fit(training_features, training_classes) +results = exported_pipeline.predict(testing_features) +""" + assert expected_code == export_pipeline(pipeline, tpot_obj.operators) def test_score(): """Assert that the TPOT score function raises a ValueError when no optimized pipeline exists""" @@ -361,9 +403,8 @@ def test_export(): def test_generate_pipeline_code(): - - tpot_obj = TPOTClassifier() """Assert that generate_pipeline_code() returns the correct code given a specific pipeline""" + tpot_obj = TPOTClassifier() pipeline = ['KNeighborsClassifier', ['CombineDFs', ['GradientBoostingClassifier', From 10482e0ee0291a2d0c949f990a5b5ca870102d51 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Feb 2017 16:59:45 -0500 Subject: [PATCH 065/154] some reg model in linux and Mac shows diff score --- tests.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/tests.py b/tests.py index a1cfc260..3c36c8ff 100644 --- a/tests.py +++ b/tests.py @@ -202,10 +202,10 @@ def test_score_3(): tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 14.375172822194937 # Assumes use of mse + known_score = 15.724128278216726 # Assumes use of mse # Reify pipeline with known score - np.random.seed(45) + np.random.seed(53) tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r) @@ -219,29 +219,6 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): assert isclose(known_score, score) -def test_sample_weight_func(): - """Assert that the TPOTRegressor score function outputs a known score for a random pipeline with sample weights""" - - tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') - tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 13.672380235317991 # Assumes use of mse - # Reify pipeline with known score - np.random.seed(45) - tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() - tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) - # make up a sample weight - training_classes_r_weight = range(1, len(training_classes_r)+1) - training_classes_r_weight_dict = tpot_obj._set_param_recursive(tpot_obj._fitted_pipeline .steps, 'random_state', 42, training_classes_r_weight) - tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r, **training_classes_r_weight_dict) - - # Get score from TPOT - score = tpot_obj.score(testing_features_r, testing_classes_r) - # http://stackoverflow.com/questions/5595425/ - def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): - return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) - - assert isclose(known_score, score) - def test_predict(): From eb160d5cca69419fa357d91efa4ab36647a6998a Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 3 Feb 2017 14:20:13 -0500 Subject: [PATCH 066/154] fix random seed issue in pset --- tests.py | 32 ++++++++++---------------------- tpot/base.py | 4 ++++ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/tests.py b/tests.py index 3c36c8ff..ef6fed95 100644 --- a/tests.py +++ b/tests.py @@ -25,7 +25,6 @@ from deap import creator from tqdm import tqdm -import deap # Set up the MNIST data set for testing mnist_data = load_digits() @@ -122,23 +121,17 @@ def test_set_params_2(): def test_random_ind(): """Assert that the TPOTClassifier can generate the same pipeline with same random seed""" - - tpot_obj = TPOTClassifier() - tpot_obj._pbar = tqdm(total=1, disable=True) - - np.random.seed(43) - pipeline1 = tpot_obj._toolbox.individual() - np.random.seed(43) - pipeline2 = tpot_obj._toolbox.individual() - + tpot_obj = TPOTClassifier(random_state=43) + pipeline1 = str(tpot_obj._toolbox.individual()) + tpot_obj = TPOTClassifier(random_state=43) + pipeline2 = str(tpot_obj._toolbox.individual()) assert pipeline1 == pipeline2 def test_random_ind_2(): """Assert that the TPOTClassifier can generate the same pipeline export with random seed of 45""" - tpot_obj = TPOTClassifier() + tpot_obj = TPOTClassifier(random_state=45) tpot_obj._pbar = tqdm(total=1, disable=True) - np.random.seed(45) pipeline = tpot_obj._toolbox.individual() expected_code = """import numpy as np @@ -178,12 +171,11 @@ def test_score(): def test_score_2(): """Assert that the TPOTClassifier score function outputs a known score for a ramdom pipeline""" - tpot_obj = TPOTClassifier() + tpot_obj = TPOTClassifier(random_state=43) tpot_obj._pbar = tqdm(total=1, disable=True) known_score = 0.96710588996037627 # Assumes use of the TPOT balanced_accuracy function # Reify pipeline with known score - np.random.seed(43) tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -200,12 +192,11 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): def test_score_3(): """Assert that the TPOTRegressor score function outputs a known score for a random pipeline""" - tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') + tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error', random_state=53) tpot_obj._pbar = tqdm(total=1, disable=True) known_score = 15.724128278216726 # Assumes use of mse # Reify pipeline with known score - np.random.seed(53) tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r) @@ -236,8 +227,7 @@ def test_predict(): def test_predict_2(): """Assert that the TPOT predict function returns a numpy matrix of shape (num_testing_rows,)""" - tpot_obj = TPOTClassifier() - np.random.seed(49) + tpot_obj = TPOTClassifier(random_state=49) tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -250,8 +240,7 @@ def test_predict_2(): def test_predict_proba(): """Assert that the TPOT predict_proba function returns a numpy matrix of shape (num_testing_rows, num_testing_classes)""" - tpot_obj = TPOTClassifier() - np.random.seed(51) + tpot_obj = TPOTClassifier(random_state=51) tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -265,8 +254,7 @@ def test_predict_proba(): def test_predict_proba2(): """Assert that the TPOT predict_proba function returns a numpy matrix filled with probabilities (float)""" - tpot_obj = TPOTClassifier() - np.random.seed(53) + tpot_obj = TPOTClassifier(random_state=53) tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) diff --git a/tpot/base.py b/tpot/base.py index 1e508cfe..fef6abce 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -258,6 +258,10 @@ def _setup_pset(self): # creating dynamically create operator class + if self.random_state is not None: + random.seed(self.random_state) + np.random.seed(self.random_state) + self._pset = gp.PrimitiveSetTyped('MAIN', [np.ndarray], Output_DF) From 55127c546cb09d40da54f13fa91b72454866904a Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 3 Feb 2017 14:43:00 -0500 Subject: [PATCH 067/154] codes clean --- tpot/base.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index fef6abce..0f19a7c0 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -176,15 +176,14 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.offspring_size = offspring_size else: self.offspring_size = population_size - # define operator dictionary + + # define operator dictionary based on files if operator_dict_file: try: with open(operator_dict_file,'r') as inf: file_string = inf.read() - operator_dict = eval(file_string[file_string.find('{'):(file_string.rfind('}')+1)]) - self.operator_dict = operator_dict - except Exception as e: - print(e) + self.operator_dict = eval(file_string[file_string.find('{'):(file_string.rfind('}')+1)]) + except: raise TypeError('The operator dictionary file is in bad format or not available! ' 'Please check the dictionary file') self.operators = [] From fab192d0f1f91c103147aaeebf6200742887f9ea Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 3 Feb 2017 14:46:46 -0500 Subject: [PATCH 068/154] codes clean --- tpot/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 0f19a7c0..12e12a8b 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -215,8 +215,6 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, 'FunctionTransformer': FunctionTransformer } - if self.verbosity > 1: - print('{} operators are imported.'.format(len(self.operators))) self._pbar = None @@ -300,6 +298,11 @@ def _setup_pset(self): for val in _type.values: self._pset.addTerminal(val, _type) + if self.verbosity > 2: + print('{} operators are imported.'.format(len(self.operators))) + + + def _setup_toolbox(self): creator.create('FitnessMulti', base.Fitness, weights=(-1.0, 1.0)) creator.create('Individual', gp.PrimitiveTree, fitness=creator.FitnessMulti) From 05cd6638c8e08e3f19b65c57ccfeb3b2f22a16bb Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 13 Feb 2017 13:45:17 -0500 Subject: [PATCH 069/154] work --- test_dict.txt | 31 +++++++++++++++++++++++++++++++ tpot/base.py | 2 +- tpot_test_config_dict.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 test_dict.txt create mode 100644 tpot_test_config_dict.py diff --git a/test_dict.txt b/test_dict.txt new file mode 100644 index 00000000..9e6a348b --- /dev/null +++ b/test_dict.txt @@ -0,0 +1,31 @@ +classifier_config_dict = { + + # Classifiers + 'sklearn.naive_bayes.GaussianNB': { + }, + + 'sklearn.naive_bayes.BernoulliNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + }, + + 'sklearn.naive_bayes.MultinomialNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + }, + + 'sklearn.tree.DecisionTreeClassifier': { + 'criterion': ["gini", "entropy"], + 'max_depth': range(1, 11), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21) + }, + + 'sklearn.ensemble.ExtraTreesClassifier': { + 'criterion': ["gini", "entropy"], + 'max_features': np.arange(0, 1.01, 0.05), + 'min_samples_split': range(2, 21), + 'min_samples_leaf': range(1, 21), + 'bootstrap': [True, False] + } +} diff --git a/tpot/base.py b/tpot/base.py index 12e12a8b..a85cadf1 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -178,7 +178,7 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.offspring_size = population_size # define operator dictionary based on files - if operator_dict_file: + if operator_dict_file: #### put to driver !!! try: with open(operator_dict_file,'r') as inf: file_string = inf.read() diff --git a/tpot_test_config_dict.py b/tpot_test_config_dict.py new file mode 100644 index 00000000..9dfe5bbd --- /dev/null +++ b/tpot_test_config_dict.py @@ -0,0 +1,31 @@ +from tpot import TPOTClassifier, TPOTRegressor +from sklearn.datasets import make_classification, make_regression +from sklearn.model_selection import train_test_split +import time + +X, y = make_classification(n_samples=200, n_features=50, + n_informative=10, n_redundant=10, random_state=42) +X_train, X_test, y_train, y_test = train_test_split(X, y, + train_size=0.75, test_size=0.25) + + +tpot = TPOTClassifier(generations=4, population_size=20, verbosity=2, + random_state = 42, operator_dict_file="test_dict.txt") +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +tpot.export('tpot_export_class_conf.py') +print('\nTime usages for TPOTClassifier:',time.time()-time_start) + +""" +X, y = make_regression(n_samples=200, n_features=50, + n_informative=10, n_targets=5, random_state=42) +X_train, X_test, y_train, y_test = train_test_split(X, y, + train_size=0.75, test_size=0.25) + +tpot = TPOTRegressor(generations=4, population_size=20, verbosity=2, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +tpot.export('tpot_export_reg_conf.py') +print('\nTime usages for TPOTRegressor:',time.time()-time_start)""" From d1d8a70ff65267f698fa95374baf2940acc78206 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 14 Feb 2017 11:29:02 -0500 Subject: [PATCH 070/154] driver update --- tests.py | 12 +++++------- tpot/base.py | 25 ++++++++++--------------- tpot/driver.py | 13 ++++++++++++- tpot/gp_deap.py | 12 +++++------- tpot/tpot.py | 4 ++-- tpot_test_config_dict.py | 31 ------------------------------- 6 files changed, 34 insertions(+), 63 deletions(-) delete mode 100644 tpot_test_config_dict.py diff --git a/tests.py b/tests.py index ef6fed95..fa185bee 100644 --- a/tests.py +++ b/tests.py @@ -11,7 +11,7 @@ from tpot.gp_types import Output_DF from tpot.gp_deap import mutNodeReplacement -from tpot.operator_utils import Operator, TPOTOperatorClassFactory +from tpot.operator_utils import TPOTOperatorClassFactory from tpot.config_classifier import classifier_config_dict @@ -92,7 +92,8 @@ def test_get_params(): 'population_size': 500, 'generations': 1000, 'offspring_size': 2000, - 'verbosity': 1 + 'verbosity': 1, + 'operator_dict': classifier_config_dict } tpot_obj = TPOTClassifier(**kwargs) @@ -427,11 +428,8 @@ def test_generate_import_code(): def test_mutNodeReplacement(): """Assert that mutNodeReplacement() returns the correct type of mutation node in a fixed pipeline""" - tpot_obj = TPOTClassifier() - pipeline = creator.Individual.\ - from_string("KNeighborsClassifier(CombineDFs(GradientBoostingClassifier(input_matrix, 38.0, 0.87), SelectKBest(input_matrix, 5)), 18, 33)", tpot_obj._pset) - # change the last operato's type to Output_DF as op.root = True - pipeline[0].ret = Output_DF + tpot_obj = TPOTClassifier(random_state=42) + pipeline = tpot_obj._toolbox.individual() old_ret_type_list = [node.ret for node in pipeline] old_prims_list = [node for node in pipeline if node.arity != 0] mut_ind = mutNodeReplacement(pipeline, pset = tpot_obj._pset) diff --git a/tpot/base.py b/tpot/base.py index a85cadf1..7b9a87d1 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -45,7 +45,6 @@ from .operator_utils import TPOTOperatorClassFactory from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code from .decorators import _timeout, _pre_test -from .operator_utils import operators, argument_types from .build_in_operators import CombineDFs from .gp_types import Bool, Output_DF from .metrics import SCORERS @@ -78,7 +77,7 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, scoring=None, cv=5, n_jobs=1, max_time_mins=None, max_eval_time_mins=5, random_state=None, verbosity=0, - disable_update_check=False, warm_start=False, operator_dict_file=None): + disable_update_check=False, warm_start=False, operator_dict=None): """Sets up the genetic programming algorithm for pipeline optimization. Parameters @@ -143,7 +142,7 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, warm_start: bool (default: False) Flag indicating whether TPOT will reuse models from previous calls to fit() for faster operation - operator_dict_file: a file including a python dictionary (default: None) + operator_dict: a customized python dictionary (default: None) The customized python dictionary to specify the list of operators and their arguments. Format examples: config_regressor.py and config_classifier.py @@ -178,14 +177,11 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.offspring_size = population_size # define operator dictionary based on files - if operator_dict_file: #### put to driver !!! - try: - with open(operator_dict_file,'r') as inf: - file_string = inf.read() - self.operator_dict = eval(file_string[file_string.find('{'):(file_string.rfind('}')+1)]) - except: - raise TypeError('The operator dictionary file is in bad format or not available! ' - 'Please check the dictionary file') + if operator_dict: + self.operator_dict = operator_dict + else: + self.operator_dict = self.default_operator_dict + self.operators = [] self.arguments = [] for key in sorted(self.operator_dict.keys()): @@ -193,7 +189,6 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.operators.append(op_class) self.arguments += arg_types - # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out if not (max_time_mins is None): @@ -344,7 +339,7 @@ def fit(self, features, classes, sample_weight=None): # Set the seed for the GP run if self.random_state is not None: - random.seed(self.random_state) + random.seed(self.random_state) # deap use random np.random.seed(self.random_state) self._start_datetime = datetime.now() @@ -382,9 +377,9 @@ def pareto_eq(ind1, ind2): # Start the progress bar if self.max_time_mins: - total_evals = self.population_size + total_evals = self.offspring_size else: - total_evals = self.population_size * (self.generations + 1) + total_evals = self.offspring_size * (self.generations + 1) self._pbar = tqdm(total=total_evals, unit='pipeline', leave=False, disable=not (self.verbosity >= 2), desc='Optimization Progress') diff --git a/tpot/driver.py b/tpot/driver.py index ab1b888f..10e0836b 100644 --- a/tpot/driver.py +++ b/tpot/driver.py @@ -207,13 +207,24 @@ def main(): else: tpot_type = TPOTRegressor + if args.OPERATOR: + try: + with open(args.OPERATOR,'r') as inf: + file_string = inf.read() + operator_dict = eval(file_string[file_string.find('{'):(file_string.rfind('}')+1)]) + except: + raise TypeError('The operator dictionary file is in bad format or not available! ' + 'Please check the dictionary file') + else: + operator_dict = None + tpot = tpot_type(generations=args.GENERATIONS, population_size=args.POPULATION_SIZE, offspring_size=args.OFFSPRING_SIZE, mutation_rate=args.MUTATION_RATE, crossover_rate=args.CROSSOVER_RATE, cv=args.NUM_CV_FOLDS, n_jobs=args.NUM_JOBS, scoring=args.SCORING_FN, max_time_mins=args.MAX_TIME_MINS, max_eval_time_mins=args.MAX_EVAL_MINS, random_state=args.RANDOM_STATE, verbosity=args.VERBOSITY, - disable_update_check=args.DISABLE_UPDATE_CHECK, operator_dict_file=args.OPERATOR) + disable_update_check=args.DISABLE_UPDATE_CHECK, operator_dict=operator_dict) tpot.fit(training_features, training_classes) diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 4f266215..2479d31e 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -16,8 +16,6 @@ details. You should have received a copy of the GNU General Public License along with the TPOT library. If not, see http://www.gnu.org/licenses/. """ - -import random import numpy as np from deap import tools, gp from inspect import isclass @@ -57,23 +55,23 @@ def varOr(population, toolbox, lambda_, cxpb, mutpb): offspring = [] for _ in range(lambda_): - op_choice = random.random() + op_choice = np.random.random() if op_choice < cxpb: # Apply crossover - ind1, ind2 = map(toolbox.clone, random.sample(population, 2)) + ind1, ind2 = map(toolbox.clone, list(np.random.choice(population, 2))) ind_str = str(ind1) ind1, ind2 = toolbox.mate(ind1, ind2) if ind_str != str(ind1): # check if crossover generated a new pipeline del ind1.fitness.values offspring.append(ind1) elif op_choice < cxpb + mutpb: # Apply mutation - ind = toolbox.clone(random.choice(population)) + ind = toolbox.clone(np.random.choice(population)) ind_str = str(ind) ind, = toolbox.mutate(ind) if ind_str != str(ind): # check if mutation happend del ind.fitness.values offspring.append(ind) else: # Apply reproduction - offspring.append(random.choice(population)) + offspring.append(np.random.choice(population)) return offspring @@ -204,7 +202,7 @@ def mutNodeReplacement(individual, pset): """ - index = random.randrange(len(individual)) + index = np.random.randint(0, len(individual)) node = individual[index] slice_ = individual.searchSubtree(index) diff --git a/tpot/tpot.py b/tpot/tpot.py index 6d1b8c60..384e96ca 100644 --- a/tpot/tpot.py +++ b/tpot/tpot.py @@ -27,7 +27,7 @@ class TPOTClassifier(TPOTBase): """TPOT estimator for classification problems""" scoring_function = 'balanced_accuracy' # Classification scoring - operator_dict = classifier_config_dict # Classification dictionary + default_operator_dict = classifier_config_dict # Classification dictionary classification = True regression = False @@ -36,6 +36,6 @@ class TPOTRegressor(TPOTBase): """TPOT estimator for regression problems""" scoring_function = 'neg_mean_squared_error' # Regression scoring - operator_dict = regressor_config_dict # Regression dictionary + default_operator_dict = regressor_config_dict # Regression dictionary classification = False regression = True diff --git a/tpot_test_config_dict.py b/tpot_test_config_dict.py deleted file mode 100644 index 9dfe5bbd..00000000 --- a/tpot_test_config_dict.py +++ /dev/null @@ -1,31 +0,0 @@ -from tpot import TPOTClassifier, TPOTRegressor -from sklearn.datasets import make_classification, make_regression -from sklearn.model_selection import train_test_split -import time - -X, y = make_classification(n_samples=200, n_features=50, - n_informative=10, n_redundant=10, random_state=42) -X_train, X_test, y_train, y_test = train_test_split(X, y, - train_size=0.75, test_size=0.25) - - -tpot = TPOTClassifier(generations=4, population_size=20, verbosity=2, - random_state = 42, operator_dict_file="test_dict.txt") -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -tpot.export('tpot_export_class_conf.py') -print('\nTime usages for TPOTClassifier:',time.time()-time_start) - -""" -X, y = make_regression(n_samples=200, n_features=50, - n_informative=10, n_targets=5, random_state=42) -X_train, X_test, y_train, y_test = train_test_split(X, y, - train_size=0.75, test_size=0.25) - -tpot = TPOTRegressor(generations=4, population_size=20, verbosity=2, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -tpot.export('tpot_export_reg_conf.py') -print('\nTime usages for TPOTRegressor:',time.time()-time_start)""" From 8b076fc461ef64788cc63f64915c56af1cc4d405 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 14 Feb 2017 11:30:30 -0500 Subject: [PATCH 071/154] clean codes --- test_dict.txt | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 test_dict.txt diff --git a/test_dict.txt b/test_dict.txt deleted file mode 100644 index 9e6a348b..00000000 --- a/test_dict.txt +++ /dev/null @@ -1,31 +0,0 @@ -classifier_config_dict = { - - # Classifiers - 'sklearn.naive_bayes.GaussianNB': { - }, - - 'sklearn.naive_bayes.BernoulliNB': { - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False] - }, - - 'sklearn.naive_bayes.MultinomialNB': { - 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], - 'fit_prior': [True, False] - }, - - 'sklearn.tree.DecisionTreeClassifier': { - 'criterion': ["gini", "entropy"], - 'max_depth': range(1, 11), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21) - }, - - 'sklearn.ensemble.ExtraTreesClassifier': { - 'criterion': ["gini", "entropy"], - 'max_features': np.arange(0, 1.01, 0.05), - 'min_samples_split': range(2, 21), - 'min_samples_leaf': range(1, 21), - 'bootstrap': [True, False] - } -} From f11e23261d0bc3448b2fb67e5298c115c566873f Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 14 Feb 2017 12:19:26 -0500 Subject: [PATCH 072/154] pbar fix --- tpot/base.py | 4 ++-- tpot/gp_deap.py | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 7b9a87d1..89fd9cee 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -377,9 +377,9 @@ def pareto_eq(ind1, ind2): # Start the progress bar if self.max_time_mins: - total_evals = self.offspring_size + total_evals = self.population_size else: - total_evals = self.offspring_size * (self.generations + 1) + total_evals = self.offspring_size * self.generations + self.population_size self._pbar = tqdm(total=total_evals, unit='pipeline', leave=False, disable=not (self.verbosity >= 2), desc='Optimization Progress') diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 2479d31e..0cd16c53 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -57,21 +57,24 @@ def varOr(population, toolbox, lambda_, cxpb, mutpb): for _ in range(lambda_): op_choice = np.random.random() if op_choice < cxpb: # Apply crossover - ind1, ind2 = map(toolbox.clone, list(np.random.choice(population, 2))) + idxs = np.random.randint(0, len(population),size=2) + ind1, ind2 = population[idxs[0]],population[idxs[1]] ind_str = str(ind1) ind1, ind2 = toolbox.mate(ind1, ind2) if ind_str != str(ind1): # check if crossover generated a new pipeline del ind1.fitness.values offspring.append(ind1) elif op_choice < cxpb + mutpb: # Apply mutation - ind = toolbox.clone(np.random.choice(population)) + idx = np.random.randint(0, len(population)) + ind = population[idx] ind_str = str(ind) ind, = toolbox.mutate(ind) if ind_str != str(ind): # check if mutation happend del ind.fitness.values offspring.append(ind) - else: # Apply reproduction - offspring.append(np.random.choice(population)) + else: # Apply reproduction + idx = np.random.randint(0, len(population)) + offspring.append(population[idx]) return offspring @@ -141,6 +144,13 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in offspring if not ind.fitness.valid] + + # update pbar for valid_ind + if not pbar.disable: + pbar.update(len(offspring)-len(invalid_ind)) + if not (max_time_mins is None) and pbar.n >= pbar.total: + pbar.total += lambda_ + fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit @@ -167,15 +177,6 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, abs(pipeline_scores.wvalues[1]), pipeline)) pbar.write('') - # Sometimes the actual evaluated pipeline count does not match the - # supposed count because DEAP can cache pipelines. Here any missed - # evaluations are added back to the progress bar. - if pbar.n < gen * mu: - missing_pipelines = (gen * mu) - pbar.n - pbar.update(missing_pipelines) - - if not (max_time_mins is None) and pbar.n >= pbar.total: - pbar.total += mu # Update the statistics with the new population record = stats.compile(population) if stats is not None else {} From 41045ec91ec24e32a26c871beea6079f3e9f070c Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 14 Feb 2017 12:40:48 -0500 Subject: [PATCH 073/154] add doc --- docs_sources/using.md | 41 +++++++++++++++++++++++++++++++++++++++++ tests.py | 5 +++-- tpot/base.py | 10 +++++----- tpot/driver.py | 12 ++++++------ 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/docs_sources/using.md b/docs_sources/using.md index cb564396..9781760a 100644 --- a/docs_sources/using.md +++ b/docs_sources/using.md @@ -100,6 +100,27 @@ TPOT offers several arguments that can be provided at the command line: Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future. +-operator +OPERATOR +String path to a file +File including a customized python dictionary to specify operators and their arguments. For example, the file's format could be like: +
+classifier_config_dict = {
+    'sklearn.naive_bayes.GaussianNB': {
+    },
+    'sklearn.naive_bayes.BernoulliNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    },
+    'sklearn.naive_bayes.MultinomialNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    }
+}
+
+ + + -v VERBOSITY {0, 1, 2, 3} @@ -207,6 +228,26 @@ Note that you can pass several parameters to the TPOT instantiation call: The random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. +operator_dict +Python dictionary +A customized python dictionary to specify operators and their arguments. For example: +
+classifier_config_dict = {
+    'sklearn.naive_bayes.GaussianNB': {
+    },
+    'sklearn.naive_bayes.BernoulliNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    },
+    'sklearn.naive_bayes.MultinomialNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    }
+}
+
+ + + verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit(). diff --git a/tests.py b/tests.py index fa185bee..8c4c147f 100644 --- a/tests.py +++ b/tests.py @@ -92,8 +92,9 @@ def test_get_params(): 'population_size': 500, 'generations': 1000, 'offspring_size': 2000, - 'verbosity': 1, - 'operator_dict': classifier_config_dict + 'operator_dict': classifier_config_dict, + 'verbosity': 1 + } tpot_obj = TPOTClassifier(**kwargs) diff --git a/tpot/base.py b/tpot/base.py index 89fd9cee..34380932 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -76,8 +76,8 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, mutation_rate=0.9, crossover_rate=0.05, scoring=None, cv=5, n_jobs=1, max_time_mins=None, max_eval_time_mins=5, - random_state=None, verbosity=0, - disable_update_check=False, warm_start=False, operator_dict=None): + random_state=None, operator_dict=None, verbosity=0, + disable_update_check=False, warm_start=False): """Sets up the genetic programming algorithm for pipeline optimization. Parameters @@ -134,6 +134,9 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, The random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. + operator_dict: a customized python dictionary (default: None) + The customized python dictionary to specify the list of operators and + their arguments. Format examples: config_regressor.py and config_classifier.py verbosity: int (default: 0) How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = all @@ -142,9 +145,6 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, warm_start: bool (default: False) Flag indicating whether TPOT will reuse models from previous calls to fit() for faster operation - operator_dict: a customized python dictionary (default: None) - The customized python dictionary to specify the list of operators and - their arguments. Format examples: config_regressor.py and config_classifier.py Returns ------- diff --git a/tpot/driver.py b/tpot/driver.py index 10e0836b..db4bbb86 100644 --- a/tpot/driver.py +++ b/tpot/driver.py @@ -156,14 +156,14 @@ def main(): 'this seed if you want your TPOT run to be reproducible with the same ' 'seed and data set in the future.') + parser.add_argument('-operator', action='store', dest='OPERATOR', default='', + type=str, help='File including a customized python dictionary to specify ' + 'operators and their arguments') + parser.add_argument('-v', action='store', dest='VERBOSITY', default=1, choices=[0, 1, 2, 3], type=int, help='How much information TPOT ' 'communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all.') - parser.add_argument('--operator_dict', dest='OPERATOR', default='', - type=str, help='File including a customized python dictionary to specify ' - 'operators and their arguments') - parser.add_argument('--no-update-check', action='store_true', dest='DISABLE_UPDATE_CHECK', default=False, help='Flag indicating whether the TPOT version checker should be disabled.') @@ -223,8 +223,8 @@ def main(): cv=args.NUM_CV_FOLDS, n_jobs=args.NUM_JOBS, scoring=args.SCORING_FN, max_time_mins=args.MAX_TIME_MINS, max_eval_time_mins=args.MAX_EVAL_MINS, - random_state=args.RANDOM_STATE, verbosity=args.VERBOSITY, - disable_update_check=args.DISABLE_UPDATE_CHECK, operator_dict=operator_dict) + random_state=args.RANDOM_STATE, operator_dict=operator_dict, verbosity=args.VERBOSITY, + disable_update_check=args.DISABLE_UPDATE_CHECK) tpot.fit(training_features, training_classes) From 5ff798bf89fa577e9b0d6e4c2f5dc25b9ae6bf63 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 23 Dec 2016 15:06:45 -0500 Subject: [PATCH 074/154] add multiprocessing --- tpot/base.py | 170 ++++++++++++++++++++++++++++++++++++++++++------ tpot/gp_deap.py | 8 +-- tpot_test.py | 20 ++++++ 3 files changed, 171 insertions(+), 27 deletions(-) create mode 100644 tpot_test.py diff --git a/tpot/base.py b/tpot/base.py index 34380932..17bc0be8 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -26,6 +26,9 @@ from functools import partial from datetime import datetime from inspect import isclass +from pathos.multiprocessing import Pool +#from joblib import Parallel, delayed + import numpy as np import deap @@ -40,6 +43,7 @@ from sklearn.metrics.scorer import make_scorer from update_checker import update_check +from joblib import Parallel, delayed from ._version import __version__ from .operator_utils import TPOTOperatorClassFactory @@ -52,6 +56,8 @@ + + # hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS if sys.platform.startswith('win'): import win32api @@ -66,7 +72,7 @@ def handler(dwCtrlType, hook_sigint=_thread.interrupt_main): return 0 win32api.SetConsoleCtrlHandler(handler, 1) # add time limit for imported function -cross_val_score = _timeout(cross_val_score) +#cross_val_score = _timeout(cross_val_score) class TPOTBase(BaseEstimator): @@ -344,7 +350,7 @@ def fit(self, features, classes, sample_weight=None): self._start_datetime = datetime.now() - self._toolbox.register('evaluate', self._evaluate_individual, features=features, classes=classes, sample_weight=sample_weight) + self._toolbox.register('evaluate', self._evaluate_individuals, features=features, classes=classes, sample_weight=sample_weight) # assign population, self._pop can only be not None if warm_start is enabled if self._pop: @@ -352,28 +358,10 @@ def fit(self, features, classes, sample_weight=None): else: pop = self._toolbox.population(n=self.population_size) - def pareto_eq(ind1, ind2): - """Determines whether two individuals are equal on the Pareto front - - Parameters - ---------- - ind1: DEAP individual from the GP population - First individual to compare - ind2: DEAP individual from the GP population - Second individual to compare - - Returns - ---------- - individuals_equal: bool - Boolean indicating whether the two individuals are equal on - the Pareto front - - """ - return np.all(ind1.fitness.values == ind2.fitness.values) # generate new pareto front if it doesn't already exist for warm start if not self.warm_start or not self._pareto_front: - self._pareto_front = tools.ParetoFront(similar=pareto_eq) + self._pareto_front = tools.ParetoFront(similar=self._pareto_eq) # Start the progress bar if self.max_time_mins: @@ -411,6 +399,7 @@ def pareto_eq(ind1, ind2): if self._pareto_front: top_score = -float('inf') for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): + print(pipeline_scores.wvalues) if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline top_score = pipeline_scores.wvalues[1] @@ -693,6 +682,107 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No raise ValueError('Scoring function does not return a float') + def _evaluate_individuals(self, individuals, features, classes, sample_weight = None): + """Determines the `individual`'s fitness + + Parameters + ---------- + individuals: a list of DEAP individual + One individual is a list of pipeline operators and model parameters that can be + compiled by DEAP into a callable function + features: numpy.ndarray {n_samples, n_features} + A numpy matrix containing the training and testing features for the + `individual`'s evaluation + classes: numpy.ndarray {n_samples, } + A numpy matrix containing the training and testing classes for the + `individual`'s evaluation + + Returns + ------- + fitness: float + Returns a float value indicating the `individual`'s fitness + according to its performance on the provided data + + """ + if self.max_time_mins: + total_mins_elapsed = (datetime.now() - self._start_datetime).total_seconds() / 60. + if total_mins_elapsed >= self.max_time_mins: + raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) + # return individuals with fitness scores + ret_individuals = [] + # 3 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing + eval_individuals = [] + sklearn_pipeline_list = [] + operator_count_list = [] + for individual in individuals: + # Disallow certain combinations of operators because they will take too long or take up too much RAM + # This is a fairly hacky way to prevent TPOT from getting stuck on bad pipelines and should be improved in a future release + individual_str = str(individual) + if (individual_str.count('PolynomialFeatures') > 1): + print('Invalid pipeline -- skipping its evaluation') + individual.fitness.value = (max(1, operator_count), resulting_score) + ret_individuals.append(individual) + if not self._pbar.disable: + self._pbar.update(1) + + # check if the individual are evaluated before + elif individual_str in self.eval_ind: + # get fitness score from previous evaluation + individual.fitness.value = self.eval_ind[individual_str] + if self.verbosity == 3: + self._pbar.write("Pipeline #{0} has been evaluated previously. " + "Continuing to the next pipeline.".format(self._pbar.n + 1)) + ret_individuals.append(individual) + if not self._pbar.disable: + self._pbar.update(1) + + else: + # Transform the tree expression into an sklearn pipeline + sklearn_pipeline = self._toolbox.compile(expr=individual) + + # Fix random state when the operator allows and build sample weight dictionary + sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) + + # Count the number of pipeline operators as a measure of pipeline complexity + operator_count = 0 + # add time limit for evaluation of pipeline + for i in range(len(individual)): + node = individual[i] + if ((type(node) is deap.gp.Terminal) or + type(node) is deap.gp.Primitive and node.name == 'CombineDFs'): + continue + operator_count += 1 + + eval_individuals.append(individual) + operator_count_list.append(operator_count) + sklearn_pipeline_list.append(sklearn_pipeline) + + # make partial for pool.map + partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, + num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) + + pool = Pool(processes=2) + """parallel = Parallel(n_jobs=2, verbose=0) + resulting_score_list = parallel(delayed(wrapped_cross_val_score)(sklearn_pipeline, features, classes, + self.num_cv_folds, self.scoring_function, sample_weight_dict) for sklearn_pipeline in sklearn_pipeline_list)""" + resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + #print(resulting_score_list) + + for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): + individual_str = str(individual) + if type(resulting_score) in [float, np.float64, np.float32]: + self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) + individual.fitness.value = self.eval_ind[individual_str] + else: + raise ValueError('Scoring function does not return a float') + ret_individuals.append(individual) + + for ind in ret_individuals: + print(ind.fitness.value) + return ret_individuals + + + def _random_mutation_operator(self, individual): """Perform a replacement, insert, or shrink mutation on an individual @@ -812,3 +902,41 @@ def _generate(self, pset, min_, max_, condition, type_=None): stack.append((depth+1, arg)) return expr + + def _pareto_eq(self, ind1, ind2): + """Determines whether two individuals are equal on the Pareto front + + Parameters + ---------- + ind1: DEAP individual from the GP population + First individual to compare + ind2: DEAP individual from the GP population + Second individual to compare + + Returns + ---------- + individuals_equal: bool + Boolean indicating whether the two individuals are equal on + the Pareto front + + """ + return np.all(ind1.fitness.values == ind2.fitness.values) + + + def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_folds, scoring_function, sample_weight_dict): + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + cv_scores = cross_val_score(sklearn_pipeline, features, classes, + cv=num_cv_folds, scoring=scoring_function, + n_jobs=1, fit_params=sample_weight_dict) + try: + resulting_score = np.mean(cv_scores) + except TypeError: + raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') + except: + resulting_score = -float('inf') + print(resulting_score) + if not self._pbar.disable: + self._pbar.update(1) + return resulting_score diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 0cd16c53..26a95d4e 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -127,9 +127,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in population if not ind.fitness.valid] - fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) - for ind, fit in zip(invalid_ind, fitnesses): - ind.fitness.values = fit + invalid_ind = toolbox.evaluate(invalid_ind) if halloffame is not None: halloffame.update(population) @@ -151,9 +149,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, if not (max_time_mins is None) and pbar.n >= pbar.total: pbar.total += lambda_ - fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) - for ind, fit in zip(invalid_ind, fitnesses): - ind.fitness.values = fit + invalid_ind = toolbox.evaluate(invalid_ind) # Update the hall of fame with the generated individuals if halloffame is not None: diff --git a/tpot_test.py b/tpot_test.py new file mode 100644 index 00000000..4aa6a1d4 --- /dev/null +++ b/tpot_test.py @@ -0,0 +1,20 @@ +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split +import time + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.25, test_size=0.75) + +#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) +#time_start = time.time() +#tpot.fit(X_train, y_train) +#print(tpot.score(X_test, y_test)) +#print('\nTime used with num_cpu = 1:',time.time()-time_start) + +tpot = TPOTClassifier(generations=1, population_size=5, verbosity=0, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used with num_cpu = 2:',time.time()-time_start) From a669c83e5a07d5fb2890f8520f5790984023d7e5 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 31 Dec 2016 13:02:58 -0500 Subject: [PATCH 075/154] fit bugs in pool and null fitness scores --- tpot/base.py | 62 +++++++++++++++++++++++++++++-------------------- tpot/gp_deap.py | 11 +++++++-- tpot_test.py | 2 +- 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 17bc0be8..d0360984 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -357,6 +357,7 @@ def fit(self, features, classes, sample_weight=None): pop = self._pop else: pop = self._toolbox.population(n=self.population_size) + print(self.population_size, len(pop)) # generate new pareto front if it doesn't already exist for warm start @@ -399,7 +400,7 @@ def fit(self, features, classes, sample_weight=None): if self._pareto_front: top_score = -float('inf') for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): - print(pipeline_scores.wvalues) + print(pipeline,pipeline_scores.wvalues) if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline top_score = pipeline_scores.wvalues[1] @@ -710,6 +711,9 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) # return individuals with fitness scores ret_individuals = [] + fitnesses = [] + num_ind = len(individuals) + print(num_ind) # 3 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing eval_individuals = [] sklearn_pipeline_list = [] @@ -720,15 +724,20 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): print('Invalid pipeline -- skipping its evaluation') - individual.fitness.value = (max(1, operator_count), resulting_score) - ret_individuals.append(individual) + #individual.fitness.value = (max(1, operator_count), resulting_score) + fitness = (max(1, operator_count), resulting_score) + fitnesses.append(fitness) + #ret_individuals.append(individual) if not self._pbar.disable: self._pbar.update(1) # check if the individual are evaluated before elif individual_str in self.eval_ind: # get fitness score from previous evaluation - individual.fitness.value = self.eval_ind[individual_str] + #individual.fitness.value = self.eval_ind[individual_str] + fitness = self.eval_ind[individual_str] + fitnesses.append(fitness) + print('duplicated pipeline', self.eval_ind[individual_str]) if self.verbosity == 3: self._pbar.write("Pipeline #{0} has been evaluated previously. " "Continuing to the next pipeline.".format(self._pbar.n + 1)) @@ -758,28 +767,31 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = sklearn_pipeline_list.append(sklearn_pipeline) # make partial for pool.map - partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, - num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) - - pool = Pool(processes=2) - """parallel = Parallel(n_jobs=2, verbose=0) - resulting_score_list = parallel(delayed(wrapped_cross_val_score)(sklearn_pipeline, features, classes, - self.num_cv_folds, self.scoring_function, sample_weight_dict) for sklearn_pipeline in sklearn_pipeline_list)""" - resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) - #print(resulting_score_list) - - for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): - individual_str = str(individual) - if type(resulting_score) in [float, np.float64, np.float32]: - self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) - individual.fitness.value = self.eval_ind[individual_str] - else: - raise ValueError('Scoring function does not return a float') - ret_individuals.append(individual) + """for ind in ret_individuals: + print(ind.fitness.value)""" + partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, + num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) + + pool = Pool(processes=2) + """parallel = Parallel(n_jobs=2, verbose=0) + resulting_score_list = parallel(delayed(wrapped_cross_val_score)(sklearn_pipeline, features, classes, + self.num_cv_folds, self.scoring_function, sample_weight_dict) for sklearn_pipeline in sklearn_pipeline_list)""" + resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + print(len(resulting_score_list),resulting_score_list) + + print('after_evaluation',len(resulting_score_list), len(operator_count_list)) + for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): + individual_str = str(individual) + if type(resulting_score) in [float, np.float64, np.float32]: + self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) + fitness = self.eval_ind[individual_str] + fitnesses.append(fitness) + else: + raise ValueError('Scoring function does not return a float') - for ind in ret_individuals: - print(ind.fitness.value) - return ret_individuals + print('eval_done') + #return ret_individuals + return fitnesses diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 26a95d4e..8137f3db 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -127,7 +127,10 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in population if not ind.fitness.valid] - invalid_ind = toolbox.evaluate(invalid_ind) + + fitnesses = toolbox.evaluate(invalid_ind) + for ind, fit in zip(invalid_ind, fitnesses): + ind.fitness.values = fit if halloffame is not None: halloffame.update(population) @@ -137,6 +140,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, # Begin the generational process for gen in range(1, ngen + 1): + # Vary the population offspring = varOr(population, toolbox, lambda_, cxpb, mutpb) @@ -149,7 +153,10 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, if not (max_time_mins is None) and pbar.n >= pbar.total: pbar.total += lambda_ - invalid_ind = toolbox.evaluate(invalid_ind) + fitnesses = toolbox.evaluate(invalid_ind) + for ind, fit in zip(invalid_ind, fitnesses): + ind.fitness.values = fit + # Update the hall of fame with the generated individuals if halloffame is not None: diff --git a/tpot_test.py b/tpot_test.py index 4aa6a1d4..0fe2a2c8 100644 --- a/tpot_test.py +++ b/tpot_test.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=1, population_size=5, verbosity=0, random_state = 42) +tpot = TPOTClassifier(generations=2, population_size=5, verbosity=0, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From 93da7e763147a02df256bcce3edaa8ab20a06ad1 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 31 Dec 2016 13:32:32 -0500 Subject: [PATCH 076/154] clean codes, pbar works and fix sample_weight_dict bug --- tpot/base.py | 57 ++++++++++++++++++++++++---------------------------- tpot_test.py | 4 ++-- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index d0360984..bb48fefd 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -357,7 +357,6 @@ def fit(self, features, classes, sample_weight=None): pop = self._pop else: pop = self._toolbox.population(n=self.population_size) - print(self.population_size, len(pop)) # generate new pareto front if it doesn't already exist for warm start @@ -400,12 +399,12 @@ def fit(self, features, classes, sample_weight=None): if self._pareto_front: top_score = -float('inf') for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): - print(pipeline,pipeline_scores.wvalues) if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline top_score = pipeline_scores.wvalues[1] # It won't raise error for a small test like in a unit test because a few pipeline sometimes # may fail due to the training data does not fit the operator's requirement. +<<<<<<< a669c83e5a07d5fb2890f8520f5790984023d7e5 if not self._optimized_pipeline: print('There was an error in the TPOT optimization ' 'process. This could be because the data was ' @@ -415,6 +414,18 @@ def fit(self, features, classes, sample_weight=None): 'passed the data to TPOT correctly.') else: self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) +======= + if not self._optimized_pipeline: + print('There was an error in the TPOT optimization ' + 'process. This could be because the data was ' + 'not formatted properly, or because data for ' + 'a regression problem was provided to the ' + 'TPOTClassifier object. Please make sure you ' + 'passed the data to TPOT correctly.') + else: + self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) + +>>>>>>> clean codes, pbar works and fix sample_weight_dict bug with warnings.catch_warnings(): warnings.simplefilter('ignore') self._fitted_pipeline.fit(features, classes) @@ -601,6 +612,7 @@ def _set_param_recursive(self, pipeline_steps, parameter, value, sample_weight = return None +<<<<<<< a669c83e5a07d5fb2890f8520f5790984023d7e5 def _evaluate_individual(self, individual, features, classes, sample_weight = None): """Determines the `individual`'s fitness @@ -683,6 +695,8 @@ def _evaluate_individual(self, individual, features, classes, sample_weight = No raise ValueError('Scoring function does not return a float') +======= +>>>>>>> clean codes, pbar works and fix sample_weight_dict bug def _evaluate_individuals(self, individuals, features, classes, sample_weight = None): """Determines the `individual`'s fitness @@ -700,8 +714,8 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = Returns ------- - fitness: float - Returns a float value indicating the `individual`'s fitness + fitnesses: float + Returns a list of tuple value indicating the `individual`'s fitness according to its performance on the provided data """ @@ -709,11 +723,11 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = total_mins_elapsed = (datetime.now() - self._start_datetime).total_seconds() / 60. if total_mins_elapsed >= self.max_time_mins: raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) - # return individuals with fitness scores - ret_individuals = [] + if not sample_weight: + sample_weight_dict = None + + # return fitness scores fitnesses = [] - num_ind = len(individuals) - print(num_ind) # 3 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing eval_individuals = [] sklearn_pipeline_list = [] @@ -724,24 +738,18 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): print('Invalid pipeline -- skipping its evaluation') - #individual.fitness.value = (max(1, operator_count), resulting_score) fitness = (max(1, operator_count), resulting_score) fitnesses.append(fitness) - #ret_individuals.append(individual) if not self._pbar.disable: self._pbar.update(1) # check if the individual are evaluated before elif individual_str in self.eval_ind: # get fitness score from previous evaluation - #individual.fitness.value = self.eval_ind[individual_str] - fitness = self.eval_ind[individual_str] - fitnesses.append(fitness) - print('duplicated pipeline', self.eval_ind[individual_str]) + fitnesses.append(self.eval_ind[individual_str]) if self.verbosity == 3: self._pbar.write("Pipeline #{0} has been evaluated previously. " "Continuing to the next pipeline.".format(self._pbar.n + 1)) - ret_individuals.append(individual) if not self._pbar.disable: self._pbar.update(1) @@ -766,31 +774,19 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = operator_count_list.append(operator_count) sklearn_pipeline_list.append(sklearn_pipeline) - # make partial for pool.map - """for ind in ret_individuals: - print(ind.fitness.value)""" partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) - - pool = Pool(processes=2) - """parallel = Parallel(n_jobs=2, verbose=0) - resulting_score_list = parallel(delayed(wrapped_cross_val_score)(sklearn_pipeline, features, classes, - self.num_cv_folds, self.scoring_function, sample_weight_dict) for sklearn_pipeline in sklearn_pipeline_list)""" + # parallel computing in evaluation of pipeline + pool = Pool(processes=self.n_jobs) resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) - print(len(resulting_score_list),resulting_score_list) - print('after_evaluation',len(resulting_score_list), len(operator_count_list)) for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): individual_str = str(individual) if type(resulting_score) in [float, np.float64, np.float32]: self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) - fitness = self.eval_ind[individual_str] - fitnesses.append(fitness) + fitnesses.append(self.eval_ind[individual_str]) else: raise ValueError('Scoring function does not return a float') - - print('eval_done') - #return ret_individuals return fitnesses @@ -948,7 +944,6 @@ def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_f raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') except: resulting_score = -float('inf') - print(resulting_score) if not self._pbar.disable: self._pbar.update(1) return resulting_score diff --git a/tpot_test.py b/tpot_test.py index 0fe2a2c8..6f2ccd5a 100644 --- a/tpot_test.py +++ b/tpot_test.py @@ -13,8 +13,8 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=2, population_size=5, verbosity=0, random_state = 42) +tpot = TPOTClassifier(generations=2, population_size=5, verbosity=3, n_jobs = 3, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 2:',time.time()-time_start) +print('\nTime used with num_cpu = 3:',time.time()-time_start) From 4b8a46f42720337bd01f33905e899dadd34868f9 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 31 Dec 2016 13:40:24 -0500 Subject: [PATCH 077/154] clean codes --- tpot/base.py | 1 + tpot_test.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index bb48fefd..cd24828d 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -911,6 +911,7 @@ def _generate(self, pset, min_, max_, condition, type_=None): return expr + # make the function pickleable def _pareto_eq(self, ind1, ind2): """Determines whether two individuals are equal on the Pareto front diff --git a/tpot_test.py b/tpot_test.py index 6f2ccd5a..8ff842e1 100644 --- a/tpot_test.py +++ b/tpot_test.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=2, population_size=5, verbosity=3, n_jobs = 3, random_state = 42) +tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, n_jobs = 3, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From f5e5a8a7f825cf5da81a8038faf697c96dba4818 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 31 Dec 2016 13:41:19 -0500 Subject: [PATCH 078/154] clean codes --- tpot_test.py => tpot_test_multi_process.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tpot_test.py => tpot_test_multi_process.py (100%) diff --git a/tpot_test.py b/tpot_test_multi_process.py similarity index 100% rename from tpot_test.py rename to tpot_test_multi_process.py From 462d20aa514915fd009efa2878d1b312f46ff3ce Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 3 Jan 2017 22:06:30 -1000 Subject: [PATCH 079/154] timeout works --- tpot/base.py | 189 +++++++++++-------------------------- tpot/decorators.py | 101 +++++++++++--------- tpot_test_multi_process.py | 2 +- 3 files changed, 115 insertions(+), 177 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index cd24828d..e2f79181 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -25,8 +25,10 @@ import sys from functools import partial from datetime import datetime + from inspect import isclass from pathos.multiprocessing import Pool + #from joblib import Parallel, delayed @@ -195,6 +197,9 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.operators.append(op_class) self.arguments += arg_types + global max_e_time_mins + max_e_time_mins = max_eval_time_mins + # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out if not (max_time_mins is None): @@ -404,7 +409,6 @@ def fit(self, features, classes, sample_weight=None): top_score = pipeline_scores.wvalues[1] # It won't raise error for a small test like in a unit test because a few pipeline sometimes # may fail due to the training data does not fit the operator's requirement. -<<<<<<< a669c83e5a07d5fb2890f8520f5790984023d7e5 if not self._optimized_pipeline: print('There was an error in the TPOT optimization ' 'process. This could be because the data was ' @@ -414,18 +418,7 @@ def fit(self, features, classes, sample_weight=None): 'passed the data to TPOT correctly.') else: self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) -======= - if not self._optimized_pipeline: - print('There was an error in the TPOT optimization ' - 'process. This could be because the data was ' - 'not formatted properly, or because data for ' - 'a regression problem was provided to the ' - 'TPOTClassifier object. Please make sure you ' - 'passed the data to TPOT correctly.') - else: - self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) ->>>>>>> clean codes, pbar works and fix sample_weight_dict bug with warnings.catch_warnings(): warnings.simplefilter('ignore') self._fitted_pipeline.fit(features, classes) @@ -611,92 +604,6 @@ def _set_param_recursive(self, pipeline_steps, parameter, value, sample_weight = else: return None - -<<<<<<< a669c83e5a07d5fb2890f8520f5790984023d7e5 - def _evaluate_individual(self, individual, features, classes, sample_weight = None): - """Determines the `individual`'s fitness - - Parameters - ---------- - individual: DEAP individual - A list of pipeline operators and model parameters that can be - compiled by DEAP into a callable function - features: numpy.ndarray {n_samples, n_features} - A numpy matrix containing the training and testing features for the - `individual`'s evaluation - classes: numpy.ndarray {n_samples, } - A numpy matrix containing the training and testing classes for the - `individual`'s evaluation - - Returns - ------- - fitness: float - Returns a float value indicating the `individual`'s fitness - according to its performance on the provided data - - """ - try: - if self.max_time_mins: - total_mins_elapsed = (datetime.now() - self._start_datetime).total_seconds() / 60. - if total_mins_elapsed >= self.max_time_mins: - raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) - - # Disallow certain combinations of operators because they will take too long or take up too much RAM - # This is a fairly hacky way to prevent TPOT from getting stuck on bad pipelines and should be improved in a future release - individual_str = str(individual) - if (individual_str.count('PolynomialFeatures') > 1): - raise ValueError('Invalid pipeline -- skipping its evaluation') - - # Transform the tree expression into an sklearn pipeline - sklearn_pipeline = self._toolbox.compile(expr=individual) - # Fix random state when the operator allows and build sample weight dictionary - sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) - - # Count the number of pipeline operators as a measure of pipeline complexity - operator_count = 0 - - # check if the individual are evaluated before - if individual_str in self.eval_ind: - # get fitness score from previous evaluation - operator_count, resulting_score = self.eval_ind[individual_str] - if self.verbosity == 3: - self._pbar.write("Pipeline #{0} has been evaluated previously. " - "Continuing to the next pipeline.".format(self._pbar.n + 1)) - else: - # add time limit for evaluation of pipeline - for i in range(len(individual)): - node = individual[i] - if ((type(node) is deap.gp.Terminal) or - type(node) is deap.gp.Primitive and node.name == 'CombineDFs'): - continue - operator_count += 1 - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - cv_scores = cross_val_score(self, sklearn_pipeline, features, classes, - cv=self.cv, scoring=self.scoring_function, - n_jobs=self.n_jobs, fit_params=sample_weight_dict) - resulting_score = np.mean(cv_scores) - - except Exception: - # Catch-all: Do not allow one pipeline that crashes to cause TPOT - # to crash. Instead, assign the crashing pipeline a poor fitness - #import traceback - #traceback.print_exc() - return 5000., -float('inf') - finally: - if not self._pbar.disable: - self._pbar.update(1) # One more pipeline evaluated - - if type(resulting_score) in [float, np.float64, np.float32]: - self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) - return max(1, operator_count), resulting_score - else: - raise ValueError('Scoring function does not return a float') - - -======= ->>>>>>> clean codes, pbar works and fix sample_weight_dict bug def _evaluate_individuals(self, individuals, features, classes, sample_weight = None): """Determines the `individual`'s fitness @@ -714,7 +621,7 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = Returns ------- - fitnesses: float + fitnesses_ordered: float Returns a list of tuple value indicating the `individual`'s fitness according to its performance on the provided data @@ -728,18 +635,22 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = # return fitness scores fitnesses = [] - # 3 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing - eval_individuals = [] + orderlist = [] + # 4 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing + eval_individuals_str = [] sklearn_pipeline_list = [] operator_count_list = [] - for individual in individuals: + test_idx_list = [] + for indidx in range(len(individuals)): # Disallow certain combinations of operators because they will take too long or take up too much RAM # This is a fairly hacky way to prevent TPOT from getting stuck on bad pipelines and should be improved in a future release + individual = individuals[indidx] individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): print('Invalid pipeline -- skipping its evaluation') - fitness = (max(1, operator_count), resulting_score) + fitness = (5000., -float('inf')) ## need reorder !!! fitnesses.append(fitness) + orderlist.append(indidx) if not self._pbar.disable: self._pbar.update(1) @@ -747,48 +658,57 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = elif individual_str in self.eval_ind: # get fitness score from previous evaluation fitnesses.append(self.eval_ind[individual_str]) + orderlist.append(indidx) if self.verbosity == 3: self._pbar.write("Pipeline #{0} has been evaluated previously. " "Continuing to the next pipeline.".format(self._pbar.n + 1)) if not self._pbar.disable: self._pbar.update(1) - else: - # Transform the tree expression into an sklearn pipeline - sklearn_pipeline = self._toolbox.compile(expr=individual) - - # Fix random state when the operator allows and build sample weight dictionary - sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) - - # Count the number of pipeline operators as a measure of pipeline complexity - operator_count = 0 - # add time limit for evaluation of pipeline - for i in range(len(individual)): - node = individual[i] - if ((type(node) is deap.gp.Terminal) or - type(node) is deap.gp.Primitive and node.name == 'CombineDFs'): - continue - operator_count += 1 - - eval_individuals.append(individual) + try: + # Transform the tree expression into an sklearn pipeline + sklearn_pipeline = self._toolbox.compile(expr=individual) + + # Fix random state when the operator allows and build sample weight dictionary + sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) + + # Count the number of pipeline operators as a measure of pipeline complexity + operator_count = 0 + # add time limit for evaluation of pipeline + for i in range(len(individual)): + node = individual[i] + if ((type(node) is deap.gp.Terminal) or + type(node) is deap.gp.Primitive and node.name == 'CombineDFs'): + continue + operator_count += 1 + except: + fitness = (5000., -float('inf')) ## need reorder !!! + fitnesses.append(fitness) + orderlist.append(indidx) + if not self._pbar.disable: + self._pbar.update(1) + continue + eval_individuals_str.append(individual_str) operator_count_list.append(operator_count) sklearn_pipeline_list.append(sklearn_pipeline) - - partial_cross_val_score = partial(self._wrapped_cross_val_score, features=features, classes=classes, + test_idx_list.append(indidx) + partial_cross_val_score = partial(self._wrapped_cross_val_score, self, features=features, classes=classes, num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) # parallel computing in evaluation of pipeline - pool = Pool(processes=self.n_jobs) + pool = ProcessPool(processes=self.n_jobs) resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) - for resulting_score, operator_count, individual in zip(resulting_score_list, operator_count_list, eval_individuals): - individual_str = str(individual) + for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) fitnesses.append(self.eval_ind[individual_str]) + orderlist.append(test_idx) else: raise ValueError('Scoring function does not return a float') - return fitnesses - + fitnesses_ordered = [None] * len(individuals) + for idx, fit in zip(orderlist, fitnesses): + fitnesses_ordered[idx] = fit + return fitnesses_ordered def _random_mutation_operator(self, individual): @@ -911,7 +831,7 @@ def _generate(self, pset, min_, max_, condition, type_=None): return expr - # make the function pickleable + # make the function pickleable def _pareto_eq(self, ind1, ind2): """Determines whether two individuals are equal on the Pareto front @@ -931,7 +851,7 @@ def _pareto_eq(self, ind1, ind2): """ return np.all(ind1.fitness.values == ind2.fitness.values) - + @_timeout def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_folds, scoring_function, sample_weight_dict): try: with warnings.catch_warnings(): @@ -939,10 +859,11 @@ def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_f cv_scores = cross_val_score(sklearn_pipeline, features, classes, cv=num_cv_folds, scoring=scoring_function, n_jobs=1, fit_params=sample_weight_dict) - try: - resulting_score = np.mean(cv_scores) - except TypeError: - raise TypeError('Warning: cv_scores is None due to timeout during evaluation of pipeline') + resulting_score = np.mean(cv_scores) + except RuntimeError: + if self.verbosity > 1: + self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') + resulting_score = -float('inf') except: resulting_score = -float('inf') if not self._pbar.disable: diff --git a/tpot/decorators.py b/tpot/decorators.py index 40a5742a..3c8d3067 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -18,7 +18,7 @@ """ - +from threading import Thread, current_thread from functools import wraps import sys import warnings @@ -29,6 +29,38 @@ pretest_X, pretest_y = make_classification(n_samples=50, n_features=10, random_state=42) pretest_X_reg, pretest_y_reg = make_regression(n_samples=50, n_features=10, random_state=42) +def convert_mins_to_secs(time_minute): + """Convert time from minutes to seconds""" + second = int(time_minute * 60) + # time limit should be at least 1 second + return max(second, 1) + + +class InterruptableThread(Thread): + def __init__(self, args, kwargs): + Thread.__init__(self) + self.args = args + self.kwargs = kwargs + self.result = -float('inf') + self.daemon = True + def stop(self): + self._stop() + def run(self): + try: + # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) + # Note: Need attention if using parallel execution model of scikit-learn + current_thread().name = 'MainThread' + self.result = func(*self.args, **self.kwargs) + except Exception: + pass + +def timeout_signal_handler(signum, frame): + """ + signal handler for _timeout function + rasie TIMEOUT exception + """ + raise RuntimeError("Time Out!") + def _timeout(func): """Runs a function with time limit @@ -48,25 +80,27 @@ def _timeout(func): limitedTime: function Wrapped function that raises a timeout exception if the time limit is exceeded """ - def convert_mins_to_secs(time_minute): - """Convert time from minutes to seconds""" - second = int(time_minute * 60) - # time limit should be at least 1 second - return max(second, 1) - class TIMEOUT(RuntimeError): - """ - Inhertis from RuntimeError - """ - pass - - def timeout_signal_handler(signum, frame): - """ - signal handler for _timeout function - rasie TIMEOUT exception - """ - raise TIMEOUT("Time Out!") - if sys.platform.startswith('linux'): - from signal import SIGXCPU, signal, getsignal + if not sys.platform.startswith('win'): + import signal + @wraps(func) + def limitedTime(self, *args, **kw): + old_signal_hander = signal.signal(signal.SIGALRM, timeout_signal_handler) + max_time_seconds = convert_mins_to_secs(self.max_eval_time_mins) + signal.alarm(max_time_seconds) + try: + ret = func(*args, **kw) + except RuntimeError: + raise RuntimeError("Time Out!") + """print('timeout!!') + ret = -float('inf') + if self.verbosity > 1: + self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') """ # f() always returns, in this scheme + finally: + signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored + signal.alarm(0) # Alarm removed + return ret + #return limitedTime + """from signal import SIGXCPU, signal, getsignal from resource import getrlimit, setrlimit, RLIMIT_CPU, getrusage, RUSAGE_SELF # timeout uses the CPU time @wraps(func) @@ -94,29 +128,12 @@ def limitedTime(self,*args, **kw): sys.tracebacklimit=1000 # reset signal signal(SIGXCPU, old_signal_hander) - return ret + return ret""" + else: - from threading import Thread, current_thread - class InterruptableThread(Thread): - def __init__(self, args, kwargs): - Thread.__init__(self) - self.args = args - self.kwargs = kwargs - self.result = None - self.daemon = True - def stop(self): - self._stop() - def run(self): - try: - # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) - # Note: Need attention if using parallel execution model of scikit-learn - current_thread().name = 'MainThread' - self.result = func(*self.args, **self.kwargs) - except Exception: - pass @wraps(func) def limitedTime(self, *args, **kw): - sys.tracebacklimit = 0 + #sys.tracebacklimit = 0 max_time_seconds = convert_mins_to_secs(self.max_eval_time_mins) # start thread tmp_it = InterruptableThread(args, kw) @@ -126,10 +143,10 @@ def limitedTime(self, *args, **kw): if tmp_it.isAlive(): if self.verbosity > 1: self._pbar.write('Timeout during evaluation of pipeline #{0}. Skipping to the next pipeline.'.format(self._pbar.n + 1)) - sys.tracebacklimit=1000 + #sys.tracebacklimit=1000 return tmp_it.result tmp_it.stop() - # return func + # return func return limitedTime def _pre_test(func): diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py index 8ff842e1..7e9ac260 100644 --- a/tpot_test_multi_process.py +++ b/tpot_test_multi_process.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, n_jobs = 3, random_state = 42) +tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 3, random_state = 42) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From c98c67f25efcefaeb23bdacbbc56c223107f78a6 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 3 Jan 2017 22:06:59 -1000 Subject: [PATCH 080/154] clean codes --- tpot/decorators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tpot/decorators.py b/tpot/decorators.py index 3c8d3067..a1ffd1c7 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -149,6 +149,9 @@ def limitedTime(self, *args, **kw): # return func return limitedTime + + + def _pre_test(func): """Decorator that wraps functions to check if the pipeline works with a pretest data set If not, then rerun the func until it generates a good pipeline From c14d016bcd6a4a24fc5c2cb1f3b2e4b1c162781c Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 3 Jan 2017 22:35:27 -1000 Subject: [PATCH 081/154] windows works and clean codes --- tpot/base.py | 10 ++--- tpot/decorators.py | 76 ++++++++++---------------------------- tpot_test_multi_process.py | 20 ---------- 3 files changed, 25 insertions(+), 81 deletions(-) delete mode 100644 tpot_test_multi_process.py diff --git a/tpot/base.py b/tpot/base.py index e2f79181..5cedbfa5 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -25,8 +25,6 @@ import sys from functools import partial from datetime import datetime - -from inspect import isclass from pathos.multiprocessing import Pool #from joblib import Parallel, delayed @@ -45,7 +43,6 @@ from sklearn.metrics.scorer import make_scorer from update_checker import update_check -from joblib import Parallel, delayed from ._version import __version__ from .operator_utils import TPOTOperatorClassFactory @@ -695,8 +692,11 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = partial_cross_val_score = partial(self._wrapped_cross_val_score, self, features=features, classes=classes, num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) # parallel computing in evaluation of pipeline - pool = ProcessPool(processes=self.n_jobs) - resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + if not sys.platform.startswith('win'): + pool = Pool(processes=self.n_jobs) + resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + else: + resulting_score_list = map(partial_cross_val_score, sklearn_pipeline_list) for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: diff --git a/tpot/decorators.py b/tpot/decorators.py index a1ffd1c7..3b550abf 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -36,24 +36,6 @@ def convert_mins_to_secs(time_minute): return max(second, 1) -class InterruptableThread(Thread): - def __init__(self, args, kwargs): - Thread.__init__(self) - self.args = args - self.kwargs = kwargs - self.result = -float('inf') - self.daemon = True - def stop(self): - self._stop() - def run(self): - try: - # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) - # Note: Need attention if using parallel execution model of scikit-learn - current_thread().name = 'MainThread' - self.result = func(*self.args, **self.kwargs) - except Exception: - pass - def timeout_signal_handler(signum, frame): """ signal handler for _timeout function @@ -91,49 +73,31 @@ def limitedTime(self, *args, **kw): ret = func(*args, **kw) except RuntimeError: raise RuntimeError("Time Out!") - """print('timeout!!') - ret = -float('inf') - if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') """ # f() always returns, in this scheme finally: signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored signal.alarm(0) # Alarm removed return ret - #return limitedTime - """from signal import SIGXCPU, signal, getsignal - from resource import getrlimit, setrlimit, RLIMIT_CPU, getrusage, RUSAGE_SELF - # timeout uses the CPU time - @wraps(func) - def limitedTime(self,*args, **kw): - # don't show traceback - sys.tracebacklimit=0 - # save old signal - old_signal_hander = getsignal(SIGXCPU) - # change signal - signal(SIGXCPU, timeout_signal_handler) - max_time_second = convert_mins_to_secs(self.max_eval_time_mins) - r = getrusage(RUSAGE_SELF) - cpu_time = r.ru_utime + r.ru_stime - current = getrlimit(RLIMIT_CPU) - try: - setrlimit(RLIMIT_CPU, (cpu_time+max_time_second, current[1])) - ret = func(*args, **kw) - except RuntimeError: - if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of pipeline #{0}. Skipping to the next pipeline.'.format(self._pbar.n + 1)) - ret = None - finally: - # reset cpu time limit and trackback - setrlimit(RLIMIT_CPU, current) - sys.tracebacklimit=1000 - # reset signal - signal(SIGXCPU, old_signal_hander) - return ret""" - else: + class InterruptableThread(Thread): + def __init__(self, args, kwargs): + Thread.__init__(self) + self.args = args + self.kwargs = kwargs + self.result = -float('inf') + self.daemon = True + def stop(self): + self._stop() + def run(self): + try: + # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) + # Note: Need attention if using parallel execution model of scikit-learn + current_thread().name = 'MainThread' + self.result = func(*self.args, **self.kwargs) + except Exception: + pass @wraps(func) def limitedTime(self, *args, **kw): - #sys.tracebacklimit = 0 + sys.tracebacklimit = 0 max_time_seconds = convert_mins_to_secs(self.max_eval_time_mins) # start thread tmp_it = InterruptableThread(args, kw) @@ -142,8 +106,8 @@ def limitedTime(self, *args, **kw): tmp_it.join(max_time_seconds) if tmp_it.isAlive(): if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of pipeline #{0}. Skipping to the next pipeline.'.format(self._pbar.n + 1)) - #sys.tracebacklimit=1000 + self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') + sys.tracebacklimit=1000 return tmp_it.result tmp_it.stop() # return func diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py deleted file mode 100644 index 7e9ac260..00000000 --- a/tpot_test_multi_process.py +++ /dev/null @@ -1,20 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split -import time - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.25, test_size=0.75) - -#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) -#time_start = time.time() -#tpot.fit(X_train, y_train) -#print(tpot.score(X_test, y_test)) -#print('\nTime used with num_cpu = 1:',time.time()-time_start) - -tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 3, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 3:',time.time()-time_start) From ed0c26f2a91f8c76ffd720882c9b42e5ca4d8b5d Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 3 Jan 2017 22:45:12 -1000 Subject: [PATCH 082/154] clean codes --- tpot/base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 5cedbfa5..0fe21996 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -27,8 +27,6 @@ from datetime import datetime from pathos.multiprocessing import Pool -#from joblib import Parallel, delayed - import numpy as np import deap From 92358591585a9bc336b9cb6225e0e03414c89944 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 4 Jan 2017 07:03:36 -1000 Subject: [PATCH 083/154] make a pickable class --- tpot/base.py | 4 ++-- tpot/decorators.py | 11 ++++++++--- tpot_test_multi_process.py | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 tpot_test_multi_process.py diff --git a/tpot/base.py b/tpot/base.py index 0fe21996..61799418 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -45,7 +45,7 @@ from ._version import __version__ from .operator_utils import TPOTOperatorClassFactory from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code -from .decorators import _timeout, _pre_test +from .decorators import _timeout, _pre_test, TimedOutExc from .build_in_operators import CombineDFs from .gp_types import Bool, Output_DF from .metrics import SCORERS @@ -858,7 +858,7 @@ def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_f cv=num_cv_folds, scoring=scoring_function, n_jobs=1, fit_params=sample_weight_dict) resulting_score = np.mean(cv_scores) - except RuntimeError: + except TimedOutExc: if self.verbosity > 1: self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') resulting_score = -float('inf') diff --git a/tpot/decorators.py b/tpot/decorators.py index 3b550abf..b403edc3 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -36,12 +36,17 @@ def convert_mins_to_secs(time_minute): return max(second, 1) +class TimedOutExc(RuntimeError): + """ + Raised when a timeout happens + """ + def timeout_signal_handler(signum, frame): """ signal handler for _timeout function rasie TIMEOUT exception """ - raise RuntimeError("Time Out!") + raise TimedOutExc("Time Out!") def _timeout(func): """Runs a function with time limit @@ -71,8 +76,8 @@ def limitedTime(self, *args, **kw): signal.alarm(max_time_seconds) try: ret = func(*args, **kw) - except RuntimeError: - raise RuntimeError("Time Out!") + except: + raise TimedOutExc("Time Out!") finally: signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored signal.alarm(0) # Alarm removed diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py new file mode 100644 index 00000000..7e9ac260 --- /dev/null +++ b/tpot_test_multi_process.py @@ -0,0 +1,20 @@ +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split +import time + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.25, test_size=0.75) + +#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) +#time_start = time.time() +#tpot.fit(X_train, y_train) +#print(tpot.score(X_test, y_test)) +#print('\nTime used with num_cpu = 1:',time.time()-time_start) + +tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 3, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used with num_cpu = 3:',time.time()-time_start) From 2f3dc37b267a79b6a8ef6d9349bb5f0e01ed71ce Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 19 Jan 2017 13:13:59 -0500 Subject: [PATCH 084/154] fix windows support --- tpot/decorators.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tpot/decorators.py b/tpot/decorators.py index b403edc3..fd013b65 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -110,8 +110,7 @@ def limitedTime(self, *args, **kw): #timer = Timer(max_time_seconds, interrupt_main) tmp_it.join(max_time_seconds) if tmp_it.isAlive(): - if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') + raise TimedOutExc("Time Out!") sys.tracebacklimit=1000 return tmp_it.result tmp_it.stop() From 3140c4c87d270207ad753177ce1e419a9d39fd99 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 27 Jan 2017 13:01:59 -0500 Subject: [PATCH 085/154] code clean and add install pathos in Travis --- ci/.travis_install.sh | 1 + tpot_test_multi_process.py | 20 -------------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 tpot_test_multi_process.py diff --git a/ci/.travis_install.sh b/ci/.travis_install.sh index 267d8e60..23a2b0ac 100755 --- a/ci/.travis_install.sh +++ b/ci/.travis_install.sh @@ -53,6 +53,7 @@ fi pip install update_checker pip install tqdm +pip install pathos if [[ "$COVERAGE" == "true" ]]; then pip install coverage coveralls diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py deleted file mode 100644 index 7e9ac260..00000000 --- a/tpot_test_multi_process.py +++ /dev/null @@ -1,20 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split -import time - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.25, test_size=0.75) - -#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) -#time_start = time.time() -#tpot.fit(X_train, y_train) -#print(tpot.score(X_test, y_test)) -#print('\nTime used with num_cpu = 1:',time.time()-time_start) - -tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 3, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 3:',time.time()-time_start) From f39a564d9ff2fb19d7ffe2366729f9b877cb37f3 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Tue, 14 Feb 2017 16:05:35 -0500 Subject: [PATCH 086/154] meet pickle problem again need rework multiprocess func --- tpot/base.py | 36 +++++++++++++++++++++--------------- tpot/decorators.py | 8 ++++---- tpot/operator_utils.py | 6 +++--- tpot_test_multi_process.py | 20 ++++++++++++++++++++ 4 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 tpot_test_multi_process.py diff --git a/tpot/base.py b/tpot/base.py index 61799418..a2f7bc41 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -27,13 +27,14 @@ from datetime import datetime from pathos.multiprocessing import Pool +from sklearn.externals.joblib import Parallel, delayed import numpy as np import deap from deap import algorithms, base, creator, tools, gp from tqdm import tqdm -from sklearn.base import BaseEstimator +from sklearn.base import BaseEstimator, clone from sklearn.model_selection import cross_val_score from sklearn.pipeline import make_pipeline, make_union from sklearn.preprocessing import FunctionTransformer @@ -43,7 +44,7 @@ from update_checker import update_check from ._version import __version__ -from .operator_utils import TPOTOperatorClassFactory +from .operator_utils import TPOTOperatorClassFactory, Operator, ARGType from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code from .decorators import _timeout, _pre_test, TimedOutExc from .build_in_operators import CombineDFs @@ -188,13 +189,11 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.operators = [] self.arguments = [] for key in sorted(self.operator_dict.keys()): - op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key]) + op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key], + BaseClass=Operator, ArgBaseClass=ARGType) self.operators.append(op_class) self.arguments += arg_types - global max_e_time_mins - max_e_time_mins = max_eval_time_mins - # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out if not (max_time_mins is None): @@ -687,12 +686,18 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = operator_count_list.append(operator_count) sklearn_pipeline_list.append(sklearn_pipeline) test_idx_list.append(indidx) - partial_cross_val_score = partial(self._wrapped_cross_val_score, self, features=features, classes=classes, - num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) + #partial_cross_val_score = partial(self._wrapped_cross_val_score, max_eval_time_mins = self.max_eval_time_mins, features=features, classes=classes, + #cv=self.cv, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict, pbar=self._pbar, verbosity=self.verbosity) # parallel computing in evaluation of pipeline if not sys.platform.startswith('win'): + parallel = Parallel(n_jobs=self.n_jobs) + resulting_score_list = parallel(delayed(self._wrapped_cross_val_score)(self.max_eval_time_mins, clone(sklearn_pipeline), + features=features, classes=classes, cv=self.cv, scoring_function=self.scoring_function, + sample_weight_dict=sample_weight_dict, pbar=self._pbar, verbosity=self.verbosity) + for sklearn_pipeline in sklearn_pipeline_list) + """ pool = Pool(processes=self.n_jobs) - resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list)""" else: resulting_score_list = map(partial_cross_val_score, sklearn_pipeline_list) @@ -850,20 +855,21 @@ def _pareto_eq(self, ind1, ind2): return np.all(ind1.fitness.values == ind2.fitness.values) @_timeout - def _wrapped_cross_val_score(self, sklearn_pipeline, features, classes, num_cv_folds, scoring_function, sample_weight_dict): + def _wrapped_cross_val_score(sklearn_pipeline, features, classes, cv, + scoring_function, sample_weight_dict, pbar, verbosity): try: with warnings.catch_warnings(): warnings.simplefilter('ignore') cv_scores = cross_val_score(sklearn_pipeline, features, classes, - cv=num_cv_folds, scoring=scoring_function, + cv=cv, scoring=scoring_function, n_jobs=1, fit_params=sample_weight_dict) resulting_score = np.mean(cv_scores) except TimedOutExc: - if self.verbosity > 1: - self._pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') + if verbosity > 1: + pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') resulting_score = -float('inf') except: resulting_score = -float('inf') - if not self._pbar.disable: - self._pbar.update(1) + if not pbar.disable: + pbar.update(1) return resulting_score diff --git a/tpot/decorators.py b/tpot/decorators.py index fd013b65..3c378ae6 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -70,9 +70,9 @@ def _timeout(func): if not sys.platform.startswith('win'): import signal @wraps(func) - def limitedTime(self, *args, **kw): + def limitedTime(max_eval_time_mins, *args, **kw): old_signal_hander = signal.signal(signal.SIGALRM, timeout_signal_handler) - max_time_seconds = convert_mins_to_secs(self.max_eval_time_mins) + max_time_seconds = convert_mins_to_secs(max_eval_time_mins) signal.alarm(max_time_seconds) try: ret = func(*args, **kw) @@ -101,9 +101,9 @@ def run(self): except Exception: pass @wraps(func) - def limitedTime(self, *args, **kw): + def limitedTime(max_eval_time_mins, *args, **kw): sys.tracebacklimit = 0 - max_time_seconds = convert_mins_to_secs(self.max_eval_time_mins) + max_time_seconds = convert_mins_to_secs(max_eval_time_mins) # start thread tmp_it = InterruptableThread(args, kw) tmp_it.start() diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 5669cf3c..e51a0e63 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -74,7 +74,7 @@ def ARGTypeClassFactory(classname, prange, BaseClass=ARGType): """ return type(classname, (BaseClass,), {'values':prange}) -def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator): +def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator, ArgBaseClass=ARGType): """Dynamically create operator class Parameters ---------- @@ -127,7 +127,7 @@ def op_type(cls): prange = opdict[pname] if not isinstance(prange, dict): classname = '{}__{}'.format(op_str, pname) - arg_types.append(ARGTypeClassFactory(classname, prange)) + arg_types.append(ARGTypeClassFactory(classname, prange, ArgBaseClass)) else: for dkey, dval in prange.items(): dep_import_str, dep_op_str, dep_op_obj = source_decode(dkey) @@ -140,7 +140,7 @@ def op_type(cls): for dpname in sorted(dval.keys()): dprange = dval[dpname] classname = '{}__{}__{}'.format(op_str, dep_op_str, dpname) - arg_types.append(ARGTypeClassFactory(classname, dprange)) + arg_types.append(ARGTypeClassFactory(classname, dprange, ArgBaseClass)) class_profile['arg_types'] = tuple(arg_types) class_profile['import_hash'] = import_hash class_profile['dep_op_list'] = dep_op_list diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py new file mode 100644 index 00000000..7bc3ef6f --- /dev/null +++ b/tpot_test_multi_process.py @@ -0,0 +1,20 @@ +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split +import time + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.25, test_size=0.75) + +#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) +#time_start = time.time() +#tpot.fit(X_train, y_train) +#print(tpot.score(X_test, y_test)) +#print('\nTime used with num_cpu = 1:',time.time()-time_start) + +tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 2, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used with num_cpu = 3:',time.time()-time_start) From 74fb3664f95988fe55ae24739f7a465d96b6b385 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 15 Feb 2017 11:00:11 -0500 Subject: [PATCH 087/154] pbar disable for multiprocess --- test_log.py | 53 +++++++++++++++++++++++++ tpot/base.py | 64 +++++++++++------------------- tpot/decorators.py | 99 +++++++++++++++++++++++----------------------- 3 files changed, 126 insertions(+), 90 deletions(-) create mode 100644 test_log.py diff --git a/test_log.py b/test_log.py new file mode 100644 index 00000000..489c7b83 --- /dev/null +++ b/test_log.py @@ -0,0 +1,53 @@ +# coding: utf-8 +get_ipython().magic('load tpot_test_multi_process.py') +# %load tpot_test_multi_process.py +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split +import time + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.25, test_size=0.75) + +#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) +#time_start = time.time() +#tpot.fit(X_train, y_train) +#print(tpot.score(X_test, y_test)) +#print('\nTime used with num_cpu = 1:',time.time()-time_start) + +tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 2, random_state = 42) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used with num_cpu = 3:',time.time()-time_start) +tpot.sklearn_pipeline_list[0] +from sklearn.model_selection import cross_val_score +from sklearn.base import BaseEstimator, clone +from sklearn.externals.joblib import Parallel, delayed +parallel = Parallel(n_jobs=self.n_jobs) +parallel = Parallel(n_jobs=2) +resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), + features=X, classes=y, cv=3, njobs = 1) + for sklearn_pipeline in sklearn_pipeline_list) +resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), + features=X, classes=y, cv=3, njobs = 1) + for sklearn_pipeline in tpot.sklearn_pipeline_list) +resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), + features=X_train, classes=y_train, cv=3, njobs = 1) + for sklearn_pipeline in tpot.sklearn_pipeline_list) +resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), + X=X_train, y=y_train, cv=3, njobs = 1) + for sklearn_pipeline in tpot.sklearn_pipeline_list) +resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), + X=X_train, y=y_train, cv=3, n_jobs = 1) + for sklearn_pipeline in tpot.sklearn_pipeline_list) +partial_cross_val_score = partial(cross_val_score, X=X_train, y=y_train, + cv=5, n_jobs=1) +from functools import partial +from pathos.multiprocessing import Pool +partial_cross_val_score = partial(cross_val_score, X=X_train, y=y_train, + cv=5, n_jobs=1) +pool = Pool(processes=2) +resulting_score_list = pool.map(partial_cross_val_score, tpot.sklearn_pipeline_list) +resulting_score_list diff --git a/tpot/base.py b/tpot/base.py index e03fccf5..965a39a7 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -696,30 +696,32 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = operator_count_list.append(operator_count) sklearn_pipeline_list.append(sklearn_pipeline) test_idx_list.append(indidx) -<<<<<<< HEAD - #partial_cross_val_score = partial(self._wrapped_cross_val_score, max_eval_time_mins = self.max_eval_time_mins, features=features, classes=classes, - #cv=self.cv, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict, pbar=self._pbar, verbosity=self.verbosity) - # parallel computing in evaluation of pipeline - if not sys.platform.startswith('win'): - parallel = Parallel(n_jobs=self.n_jobs) - resulting_score_list = parallel(delayed(self._wrapped_cross_val_score)(self.max_eval_time_mins, clone(sklearn_pipeline), - features=features, classes=classes, cv=self.cv, scoring_function=self.scoring_function, - sample_weight_dict=sample_weight_dict, pbar=self._pbar, verbosity=self.verbosity) - for sklearn_pipeline in sklearn_pipeline_list) - """ - pool = Pool(processes=self.n_jobs) - resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list)""" - else: - resulting_score_list = map(partial_cross_val_score, sklearn_pipeline_list) -======= - partial_cross_val_score = partial(self._wrapped_cross_val_score, self, features=features, classes=classes, - num_cv_folds=self.num_cv_folds, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict) - # parallel computing in evaluation of pipeline + + @_timeout(max_eval_time_mins=self.max_eval_time_mins) + def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classes, + cv=self.cv, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict, + verbosity=self.verbosity): + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + cv_scores = cross_val_score(sklearn_pipeline, features, classes, + cv=cv, scoring=scoring_function, + n_jobs=1, fit_params=sample_weight_dict) + resulting_score = np.mean(cv_scores) + except: + resulting_score = -float('inf') + return resulting_score + if not sys.platform.startswith('win'): + self.sklearn_pipeline_list = sklearn_pipeline_list pool = Pool(processes=self.n_jobs) - resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipeline_list) + resulting_score_list = pool.map(_wrapped_cross_val_score, sklearn_pipeline_list) else: - resulting_score_list = map(partial_cross_val_score, sklearn_pipeline_list) + resulting_score_list = map(_wrapped_cross_val_score, sklearn_pipeline_list) + + self._pbar.update(len(sklearn_pipeline_list)) + + #resulting_score_list = map(partial_cross_val_score, sklearn_pipeline_list) for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: @@ -875,23 +877,3 @@ def _pareto_eq(self, ind1, ind2): """ return np.all(ind1.fitness.values == ind2.fitness.values) - - @_timeout - def _wrapped_cross_val_score(sklearn_pipeline, features, classes, cv, - scoring_function, sample_weight_dict, pbar, verbosity): - try: - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - cv_scores = cross_val_score(sklearn_pipeline, features, classes, - cv=cv, scoring=scoring_function, - n_jobs=1, fit_params=sample_weight_dict) - resulting_score = np.mean(cv_scores) - except TimedOutExc: - if verbosity > 1: - pbar.write('Timeout during evaluation of a pipeline. Skipping to the next pipeline.') - resulting_score = -float('inf') - except: - resulting_score = -float('inf') - if not pbar.disable: - pbar.update(1) - return resulting_score diff --git a/tpot/decorators.py b/tpot/decorators.py index 2eb351f7..5058291a 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -67,7 +67,7 @@ def timeout_signal_handler(signum, frame): """ raise TimedOutExc("Time Out!") -def _timeout(func): +def _timeout(max_eval_time_mins=5): """Runs a function with time limit Parameters @@ -86,55 +86,56 @@ def _timeout(func): limitedTime: function Wrapped function that raises a timeout exception if the time limit is exceeded """ - if not sys.platform.startswith('win'): - import signal - @wraps(func) - def limitedTime(max_eval_time_mins, *args, **kw): - old_signal_hander = signal.signal(signal.SIGALRM, timeout_signal_handler) - max_time_seconds = convert_mins_to_secs(max_eval_time_mins) - signal.alarm(max_time_seconds) - try: - ret = func(*args, **kw) - except: - raise TimedOutExc("Time Out!") - finally: - signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored - signal.alarm(0) # Alarm removed - return ret - else: - class InterruptableThread(Thread): - def __init__(self, args, kwargs): - Thread.__init__(self) - self.args = args - self.kwargs = kwargs - self.result = -float('inf') - self.daemon = True - def stop(self): - self._stop() - def run(self): + def wrap_func(func): + if not sys.platform.startswith('win'): + import signal + @wraps(func) + def limitedTime(*args, **kw): + old_signal_hander = signal.signal(signal.SIGALRM, timeout_signal_handler) + max_time_seconds = convert_mins_to_secs(max_eval_time_mins) + signal.alarm(max_time_seconds) try: - # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) - # Note: Need attention if using parallel execution model of scikit-learn - current_thread().name = 'MainThread' - self.result = func(*self.args, **self.kwargs) - except Exception: - pass - @wraps(func) - def limitedTime(max_eval_time_mins, *args, **kw): - sys.tracebacklimit = 0 - max_time_seconds = convert_mins_to_secs(max_eval_time_mins) - # start thread - tmp_it = InterruptableThread(args, kw) - tmp_it.start() - #timer = Timer(max_time_seconds, interrupt_main) - tmp_it.join(max_time_seconds) - if tmp_it.isAlive(): - raise TimedOutExc("Time Out!") - sys.tracebacklimit=1000 - return tmp_it.result - tmp_it.stop() - # return func - return limitedTime + ret = func(*args, **kw) + except: + raise TimedOutExc("Time Out!") + finally: + signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored + signal.alarm(0) # Alarm removed + return ret + else: + class InterruptableThread(Thread): + def __init__(self, args, kwargs): + Thread.__init__(self) + self.args = args + self.kwargs = kwargs + self.result = -float('inf') + self.daemon = True + def stop(self): + self._stop() + def run(self): + try: + # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) + # Note: Need attention if using parallel execution model of scikit-learn + current_thread().name = 'MainThread' + self.result = func(*self.args, **self.kwargs) + except Exception: + pass + @wraps(func) + def limitedTime(*args, **kw): + sys.tracebacklimit = 0 + max_time_seconds = convert_mins_to_secs(max_eval_time_mins) + # start thread + tmp_it = InterruptableThread(args, kw) + tmp_it.start() + #timer = Timer(max_time_seconds, interrupt_main) + tmp_it.join(max_time_seconds) + if tmp_it.isAlive(): + raise TimedOutExc("Time Out!") + sys.tracebacklimit=1000 + return tmp_it.result + tmp_it.stop() + return limitedTime + return wrap_func From 9b1369f92c1be2ace95d7ef0a17ccb9f0f2cc926 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 15 Feb 2017 11:43:30 -0500 Subject: [PATCH 088/154] clean codes --- tpot/base.py | 45 +++++++++++++++----------------------- tpot_test_multi_process.py | 2 +- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 965a39a7..a6e0713d 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -381,11 +381,13 @@ def fit(self, features, classes, sample_weight=None): disable=not (self.verbosity >= 2), desc='Optimization Progress') try: - pop, _ = eaMuPlusLambda(population=pop, toolbox=self._toolbox, - mu=self.population_size, lambda_=self.offspring_size, - cxpb=self.crossover_rate, mutpb=self.mutation_rate, - ngen=self.generations, pbar=self._pbar, halloffame=self._pareto_front, - verbose=self.verbosity, max_time_mins=self.max_time_mins) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + pop, _ = eaMuPlusLambda(population=pop, toolbox=self._toolbox, + mu=self.population_size, lambda_=self.offspring_size, + cxpb=self.crossover_rate, mutpb=self.mutation_rate, + ngen=self.generations, pbar=self._pbar, halloffame=self._pareto_front, + verbose=self.verbosity, max_time_mins=self.max_time_mins) # store population for the next call if self.warm_start: @@ -638,31 +640,27 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = sample_weight_dict = None # return fitness scores - fitnesses = [] - orderlist = [] + fitnesses_dict = {} # 4 lists of DEAP individuals, their sklearn pipelines and their operator counts for parallel computing eval_individuals_str = [] sklearn_pipeline_list = [] operator_count_list = [] test_idx_list = [] - for indidx in range(len(individuals)): + for indidx, individual in enumerate(individuals): # Disallow certain combinations of operators because they will take too long or take up too much RAM # This is a fairly hacky way to prevent TPOT from getting stuck on bad pipelines and should be improved in a future release individual = individuals[indidx] individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): print('Invalid pipeline -- skipping its evaluation') - fitness = (5000., -float('inf')) ## need reorder !!! - fitnesses.append(fitness) - orderlist.append(indidx) + fitnesses_dict[indidx] = (5000., -float('inf')) if not self._pbar.disable: self._pbar.update(1) # check if the individual are evaluated before elif individual_str in self.eval_ind: # get fitness score from previous evaluation - fitnesses.append(self.eval_ind[individual_str]) - orderlist.append(indidx) + fitnesses_dict[indidx] = self.eval_ind[individual_str] if self.verbosity == 3: self._pbar.write("Pipeline #{0} has been evaluated previously. " "Continuing to the next pipeline.".format(self._pbar.n + 1)) @@ -686,9 +684,7 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = continue operator_count += 1 except: - fitness = (5000., -float('inf')) ## need reorder !!! - fitnesses.append(fitness) - orderlist.append(indidx) + fitnesses_dict[indidx] = (5000., -float('inf')) if not self._pbar.disable: self._pbar.update(1) continue @@ -699,8 +695,7 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = @_timeout(max_eval_time_mins=self.max_eval_time_mins) def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classes, - cv=self.cv, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict, - verbosity=self.verbosity): + cv=self.cv, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict): try: with warnings.catch_warnings(): warnings.simplefilter('ignore') @@ -721,21 +716,17 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe self._pbar.update(len(sklearn_pipeline_list)) - #resulting_score_list = map(partial_cross_val_score, sklearn_pipeline_list) - for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) - fitnesses.append(self.eval_ind[individual_str]) - orderlist.append(test_idx) + fitnesses_dict[test_idx] = self.eval_ind[individual_str] else: raise ValueError('Scoring function does not return a float') - fitnesses_ordered = [None] * len(individuals) - for idx, fit in zip(orderlist, fitnesses): - fitnesses_ordered[idx] = fit - return fitnesses_ordered - + fitnesses_ordered = [] + for key in sorted(fitnesses_dict.keys()): + fitnesses_ordered.append(fitnesses_dict[key]) + return fitnesses_ordered def _random_mutation_operator(self, individual): diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py index 7bc3ef6f..8adb11b4 100644 --- a/tpot_test_multi_process.py +++ b/tpot_test_multi_process.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 2, random_state = 42) +tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.02, n_jobs = 2, random_state = 44, max_time_mins=1) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From 1a5ca2ac8b09f656389db317f0a530681306d010 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 15 Feb 2017 12:02:59 -0500 Subject: [PATCH 089/154] clean codes --- tpot/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index a6e0713d..5fb51f71 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -708,7 +708,6 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe return resulting_score if not sys.platform.startswith('win'): - self.sklearn_pipeline_list = sklearn_pipeline_list pool = Pool(processes=self.n_jobs) resulting_score_list = pool.map(_wrapped_cross_val_score, sklearn_pipeline_list) else: From 93db34f0886b94f56364916690221b85cb20238f Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 15 Feb 2017 12:48:33 -0500 Subject: [PATCH 090/154] fix issue in sample weight --- tpot/base.py | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 5fb51f71..30132e19 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -573,7 +573,7 @@ def _compile_to_sklearn(self, expr): sklearn_pipeline = generate_pipeline_code(expr_to_tree(expr), self.operators) return eval(sklearn_pipeline, self.operators_context) - def _set_param_recursive(self, pipeline_steps, parameter, value, sample_weight = None): + def _set_param_recursive(self, pipeline_steps, parameter, value): """Recursively iterates through all objects in the pipeline and sets the given parameter to the specified value Parameters @@ -584,18 +584,12 @@ def _set_param_recursive(self, pipeline_steps, parameter, value, sample_weight = The parameter to assign a value for in each pipeline object value: any The value to assign the parameter to in each pipeline object - Returns ------- - sample_weight_dict: - A dictionary of sample_weight + None """ - sample_weight_dict = {} for (pname, obj) in pipeline_steps: - if inspect.getargspec(obj.fit).args.count('sample_weight') and sample_weight: - step_sw = pname + '__sample_weight' - sample_weight_dict[step_sw] = sample_weight recursive_attrs = ['steps', 'transformer_list', 'estimators'] for attr in recursive_attrs: if hasattr(obj, attr): @@ -604,10 +598,6 @@ def _set_param_recursive(self, pipeline_steps, parameter, value, sample_weight = else: if hasattr(obj, parameter): setattr(obj, parameter, value) - if sample_weight_dict: - return sample_weight_dict - else: - return None def _evaluate_individuals(self, individuals, features, classes, sample_weight = None): @@ -672,7 +662,7 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = sklearn_pipeline = self._toolbox.compile(expr=individual) # Fix random state when the operator allows and build sample weight dictionary - sample_weight_dict = self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42, sample_weight) + self._set_param_recursive(sklearn_pipeline.steps, 'random_state', 42) # Count the number of pipeline operators as a measure of pipeline complexity operator_count = 0 @@ -693,9 +683,35 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = sklearn_pipeline_list.append(sklearn_pipeline) test_idx_list.append(indidx) + def _set_sample_weight(pipeline_steps, sample_weight): + """Recursively iterates through all objects in the pipeline and sets the given parameter to the specified value + + Parameters + ---------- + pipeline_steps: array-like + List of (str, obj) tuples from a scikit-learn pipeline or related object + sample_weight: array-like + List of sample weight + Returns + ------- + sample_weight_dict: + A dictionary of sample_weight + + """ + sample_weight_dict = {} + for (pname, obj) in pipeline_steps: + if inspect.getargspec(obj.fit).args.count('sample_weight') and sample_weight: + step_sw = pname + '__sample_weight' + sample_weight_dict[step_sw] = sample_weight + if sample_weight_dict: + return sample_weight_dict + else: + return None + @_timeout(max_eval_time_mins=self.max_eval_time_mins) def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classes, - cv=self.cv, scoring_function=self.scoring_function,sample_weight_dict=sample_weight_dict): + cv=self.cv, scoring_function=self.scoring_function,sample_weight=sample_weight): + sample_weight_dict = _set_sample_weight(sklearn_pipeline.steps, sample_weight) try: with warnings.catch_warnings(): warnings.simplefilter('ignore') From a240e2476060b1e63312de6aa5f12750195118fa Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 15 Feb 2017 15:27:25 -0500 Subject: [PATCH 091/154] pbar reword --- tpot/base.py | 16 ++++++++++------ tpot_test_multi_process.py | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 30132e19..fec2d1a8 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -723,13 +723,17 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe resulting_score = -float('inf') return resulting_score - if not sys.platform.startswith('win'): - pool = Pool(processes=self.n_jobs) - resulting_score_list = pool.map(_wrapped_cross_val_score, sklearn_pipeline_list) - else: - resulting_score_list = map(_wrapped_cross_val_score, sklearn_pipeline_list) + resulting_score_list = [] + for i in range(0, len(sklearn_pipeline_list), self.n_jobs*2): + sk_chunk_list = sklearn_pipeline_list[i:i+self.n_jobs*2] + if not sys.platform.startswith('win'): + pool = Pool(processes=self.n_jobs) + resulting_score_list += pool.map(_wrapped_cross_val_score, sk_chunk_list) + else: + resulting_score_list += map(_wrapped_cross_val_score, sk_chunk_list) + self._pbar.update(self.n_jobs*2) + - self._pbar.update(len(sklearn_pipeline_list)) for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py index 8adb11b4..fc728a8b 100644 --- a/tpot_test_multi_process.py +++ b/tpot_test_multi_process.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.02, n_jobs = 2, random_state = 44, max_time_mins=1) +tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.04, n_jobs = 2, random_state = 44, max_time_mins=1) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From 9e57510dabacac54088632df079384aeb9de8097 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 15 Feb 2017 16:13:40 -0500 Subject: [PATCH 092/154] chunk --- tpot/base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index fec2d1a8..bb3199bd 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -723,15 +723,19 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe resulting_score = -float('inf') return resulting_score + # split sklearn_pipeline_list to small chunks for pbar update resulting_score_list = [] for i in range(0, len(sklearn_pipeline_list), self.n_jobs*2): sk_chunk_list = sklearn_pipeline_list[i:i+self.n_jobs*2] if not sys.platform.startswith('win'): pool = Pool(processes=self.n_jobs) - resulting_score_list += pool.map(_wrapped_cross_val_score, sk_chunk_list) + chunk_res = pool.map(_wrapped_cross_val_score, sk_chunk_list) else: - resulting_score_list += map(_wrapped_cross_val_score, sk_chunk_list) - self._pbar.update(self.n_jobs*2) + chunk_res += map(_wrapped_cross_val_score, sk_chunk_list) + for res in chunk_res: + self._pbar.update(1) + resulting_score_list += chunk_res + From 7bc9044d690d55087f392f763e43c437c5ab1191 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 09:40:15 -0500 Subject: [PATCH 093/154] pbar still not work need fix around (test_log.py) --- test_log.py | 70 ++++++++++++++++++++++------------------------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/test_log.py b/test_log.py index 489c7b83..96dd54c5 100644 --- a/test_log.py +++ b/test_log.py @@ -1,53 +1,41 @@ -# coding: utf-8 -get_ipython().magic('load tpot_test_multi_process.py') -# %load tpot_test_multi_process.py -from tpot import TPOTClassifier from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split +from sklearn.model_selection import train_test_split, cross_val_score +from sklearn.base import BaseEstimator, clone +from sklearn.externals.joblib import Parallel, delayed +from sklearn.naive_bayes import GaussianNB +from sklearn.preprocessing import StandardScaler +from sklearn.pipeline import make_pipeline import time +from tqdm import tqdm + digits = load_digits() X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, train_size=0.25, test_size=0.75) -#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) -#time_start = time.time() -#tpot.fit(X_train, y_train) -#print(tpot.score(X_test, y_test)) -#print('\nTime used with num_cpu = 1:',time.time()-time_start) - -tpot = TPOTClassifier(generations=2, population_size=10, verbosity=2, max_eval_time_mins=0.02, n_jobs = 2, random_state = 42) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 3:',time.time()-time_start) -tpot.sklearn_pipeline_list[0] -from sklearn.model_selection import cross_val_score -from sklearn.base import BaseEstimator, clone -from sklearn.externals.joblib import Parallel, delayed -parallel = Parallel(n_jobs=self.n_jobs) + +pbar = tqdm(total=10, unit='pipeline', leave=False, + disable=False, desc='Optimization Progress') + +pipeline = make_pipeline(StandardScaler(), GaussianNB(priors=None)) + +sklearn_pipelines = [pipeline]*10 + +def _wrapped_cross_val_score(sklearn_pipeline, features, classes, +cv, pbar): + cv_scores = cross_val_score(sklearn_pipeline, features, classes, + cv=cv, n_jobs=1) + pbar.update(1) + return cv_scores + parallel = Parallel(n_jobs=2) -resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), - features=X, classes=y, cv=3, njobs = 1) - for sklearn_pipeline in sklearn_pipeline_list) -resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), - features=X, classes=y, cv=3, njobs = 1) - for sklearn_pipeline in tpot.sklearn_pipeline_list) -resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), - features=X_train, classes=y_train, cv=3, njobs = 1) - for sklearn_pipeline in tpot.sklearn_pipeline_list) -resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), - X=X_train, y=y_train, cv=3, njobs = 1) - for sklearn_pipeline in tpot.sklearn_pipeline_list) -resulting_score_list = parallel(delayed(cross_val_score)(clone(sklearn_pipeline), - X=X_train, y=y_train, cv=3, n_jobs = 1) - for sklearn_pipeline in tpot.sklearn_pipeline_list) -partial_cross_val_score = partial(cross_val_score, X=X_train, y=y_train, - cv=5, n_jobs=1) +resulting_score_list = parallel(delayed(_wrapped_cross_val_score)(clone(sklearn_pipeline), + features=X_train, classes=y_train, cv=3, pbar = pbar) + for sklearn_pipeline in sklearn_pipelines) + from functools import partial from pathos.multiprocessing import Pool -partial_cross_val_score = partial(cross_val_score, X=X_train, y=y_train, - cv=5, n_jobs=1) +partial_cross_val_score = partial(_wrapped_cross_val_score, features=X_train, classes=y_train, cv=3, pbar = pbar) pool = Pool(processes=2) -resulting_score_list = pool.map(partial_cross_val_score, tpot.sklearn_pipeline_list) +resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipelines) resulting_score_list From ef7718f02b743dbecfee08b9427a09e8b49a0fab Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 13:50:54 -0500 Subject: [PATCH 094/154] pbar works lol --- test_log.py | 27 +++++++++++++++++++-------- tpot/base.py | 35 +++++++++++++++++++---------------- tpot/gp_deap.py | 3 ++- tpot_test_multi_process.py | 2 +- 4 files changed, 41 insertions(+), 26 deletions(-) diff --git a/test_log.py b/test_log.py index 96dd54c5..16e42f81 100644 --- a/test_log.py +++ b/test_log.py @@ -10,9 +10,8 @@ digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.25, test_size=0.75) - +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,train_size=0.25, test_size=0.75) +global pbar pbar = tqdm(total=10, unit='pipeline', leave=False, disable=False, desc='Optimization Progress') @@ -25,7 +24,6 @@ def _wrapped_cross_val_score(sklearn_pipeline, features, classes, cv, pbar): cv_scores = cross_val_score(sklearn_pipeline, features, classes, cv=cv, n_jobs=1) - pbar.update(1) return cv_scores parallel = Parallel(n_jobs=2) @@ -34,8 +32,21 @@ def _wrapped_cross_val_score(sklearn_pipeline, features, classes, for sklearn_pipeline in sklearn_pipelines) from functools import partial -from pathos.multiprocessing import Pool +from pathos.multiprocessing import ProcessPool partial_cross_val_score = partial(_wrapped_cross_val_score, features=X_train, classes=y_train, cv=3, pbar = pbar) -pool = Pool(processes=2) -resulting_score_list = pool.map(partial_cross_val_score, sklearn_pipelines) -resulting_score_list +pool = ProcessPool(processes=2) +resulting_score_list = pool.imap(partial_cross_val_score, sklearn_pipelines) +num_done = 0 +pbar = tqdm(total=10, unit='pipeline', leave=False, + disable=False, desc='Optimization Progress') +while True: + + if resulting_score_list._index < 10: + pass + #print(resulting_score_list._index) + else: + break + + num_update = resulting_score_list._index - num_done + pbar.update(resulting_score_list._index - pbar.n) + num_done = resulting_score_list._index diff --git a/tpot/base.py b/tpot/base.py index bb3199bd..1254f1d2 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -25,7 +25,7 @@ import sys from functools import partial from datetime import datetime -from pathos.multiprocessing import Pool +from pathos.multiprocessing import ProcessPool from sklearn.externals.joblib import Parallel, delayed @@ -723,21 +723,24 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe resulting_score = -float('inf') return resulting_score - # split sklearn_pipeline_list to small chunks for pbar update - resulting_score_list = [] - for i in range(0, len(sklearn_pipeline_list), self.n_jobs*2): - sk_chunk_list = sklearn_pipeline_list[i:i+self.n_jobs*2] - if not sys.platform.startswith('win'): - pool = Pool(processes=self.n_jobs) - chunk_res = pool.map(_wrapped_cross_val_score, sk_chunk_list) - else: - chunk_res += map(_wrapped_cross_val_score, sk_chunk_list) - for res in chunk_res: - self._pbar.update(1) - resulting_score_list += chunk_res - - - + if not sys.platform.startswith('win'): + pool = ProcessPool(processes=self.n_jobs) + res_imap = pool.imap(_wrapped_cross_val_score, sklearn_pipeline_list) + ini_pbar_n = self._pbar.n + # hacky way for pbar update by using imap in pathos.multiprocessing.ProcessPool + while True: + num_job_done = len(res_imap._items) + if not self._pbar.disable: + self._pbar.update(ini_pbar_n + num_job_done - self._pbar.n) + if num_job_done >= len(sklearn_pipeline_list): + break + resulting_score_list = list(res_imap) + else: + resulting_score_list = [] + for sklearn_pipeline in sklearn_pipeline_list: + resulting_score_list.append(_wrapped_cross_val_score(sklearn_pipeline)) + if not self._pbar.disable: + self._pbar.update(1) for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index b18fb181..51f27c2f 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -151,7 +151,8 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, pbar.update(len(offspring)-len(invalid_ind)) if not (max_time_mins is None) and pbar.n >= pbar.total: pbar.total += lambda_ - + print(pbar.total) + print(pbar.n) fitnesses = toolbox.evaluate(invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py index fc728a8b..31eb5790 100644 --- a/tpot_test_multi_process.py +++ b/tpot_test_multi_process.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.04, n_jobs = 2, random_state = 44, max_time_mins=1) +tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.06, n_jobs = 2, random_state = 44)#, max_time_mins=1) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From da6392568e8a145255e073c9e26008575162a083 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 15:14:45 -0500 Subject: [PATCH 095/154] timeout works --- test_log.py | 23 +++++++---------------- tpot/base.py | 27 +++++++++++++++++++-------- tpot/decorators.py | 14 +++++++++----- tpot/gp_deap.py | 3 +-- 4 files changed, 36 insertions(+), 31 deletions(-) diff --git a/test_log.py b/test_log.py index 16e42f81..a077dd2a 100644 --- a/test_log.py +++ b/test_log.py @@ -8,10 +8,11 @@ import time from tqdm import tqdm +from functools import partial +from pathos.multiprocessing import ProcessPool digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,train_size=0.25, test_size=0.75) -global pbar +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,train_size=0.95, test_size=0.05) pbar = tqdm(total=10, unit='pipeline', leave=False, disable=False, desc='Optimization Progress') @@ -22,31 +23,21 @@ def _wrapped_cross_val_score(sklearn_pipeline, features, classes, cv, pbar): +import numpy as np cv_scores = cross_val_score(sklearn_pipeline, features, classes, cv=cv, n_jobs=1) - return cv_scores + return np.mean(cv_scores) -parallel = Parallel(n_jobs=2) -resulting_score_list = parallel(delayed(_wrapped_cross_val_score)(clone(sklearn_pipeline), - features=X_train, classes=y_train, cv=3, pbar = pbar) - for sklearn_pipeline in sklearn_pipelines) -from functools import partial -from pathos.multiprocessing import ProcessPool partial_cross_val_score = partial(_wrapped_cross_val_score, features=X_train, classes=y_train, cv=3, pbar = pbar) pool = ProcessPool(processes=2) resulting_score_list = pool.imap(partial_cross_val_score, sklearn_pipelines) -num_done = 0 pbar = tqdm(total=10, unit='pipeline', leave=False, disable=False, desc='Optimization Progress') while True: - + pbar.update(resulting_score_list._index - pbar.n) if resulting_score_list._index < 10: pass - #print(resulting_score_list._index) + print(len(resulting_score_list._items)) else: break - - num_update = resulting_score_list._index - num_done - pbar.update(resulting_score_list._index - pbar.n) - num_done = resulting_score_list._index diff --git a/tpot/base.py b/tpot/base.py index 1254f1d2..f8c130e0 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -49,7 +49,7 @@ from .operator_utils import TPOTOperatorClassFactory, Operator, ARGType from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code -from .decorators import _timeout, _pre_test, TimedOutExc +from .decorators import _timeout, _pre_test from .build_in_operators import CombineDFs @@ -651,9 +651,6 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = elif individual_str in self.eval_ind: # get fitness score from previous evaluation fitnesses_dict[indidx] = self.eval_ind[individual_str] - if self.verbosity == 3: - self._pbar.write("Pipeline #{0} has been evaluated previously. " - "Continuing to the next pipeline.".format(self._pbar.n + 1)) if not self._pbar.disable: self._pbar.update(1) else: @@ -712,6 +709,7 @@ def _set_sample_weight(pipeline_steps, sample_weight): def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classes, cv=self.cv, scoring_function=self.scoring_function,sample_weight=sample_weight): sample_weight_dict = _set_sample_weight(sklearn_pipeline.steps, sample_weight) + from .decorators import TimedOutExc try: with warnings.catch_warnings(): warnings.simplefilter('ignore') @@ -719,6 +717,8 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe cv=cv, scoring=scoring_function, n_jobs=1, fit_params=sample_weight_dict) resulting_score = np.mean(cv_scores) + except TimedOutExc: + resulting_score = -500000000. except: resulting_score = -float('inf') return resulting_score @@ -729,16 +729,27 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe ini_pbar_n = self._pbar.n # hacky way for pbar update by using imap in pathos.multiprocessing.ProcessPool while True: - num_job_done = len(res_imap._items) - if not self._pbar.disable: + tmp_fitness = np.array(res_imap._items) + num_job_done = len(tmp_fitness) + if not self._pbar.disable and num_job_done: + timeout_index = list(np.where(tmp_fitness[:,1] == -500000000.)[0]) + for idx in timeout_index: + if self._pbar.n - ini_pbar_n <= idx: + self._pbar.write("Skip pipeline #{0} due to time out. " + "Continuing to the next pipeline.".format(ini_pbar_n + idx + 1)) self._pbar.update(ini_pbar_n + num_job_done - self._pbar.n) - if num_job_done >= len(sklearn_pipeline_list): + if num_job_done >= len(sklearn_pipeline_list): break resulting_score_list = list(res_imap) else: resulting_score_list = [] for sklearn_pipeline in sklearn_pipeline_list: - resulting_score_list.append(_wrapped_cross_val_score(sklearn_pipeline)) + resulting_score = _wrapped_cross_val_score(sklearn_pipeline) + if resulting_score == -500000000.: + if not self._pbar.disable: + self._pbar.write("Skip pipeline #{0} due to time out. " + "Continuing to the next pipeline.".format(self._pbar.n + 1)) + resulting_score_list.append(resulting_score) if not self._pbar.disable: self._pbar.update(1) diff --git a/tpot/decorators.py b/tpot/decorators.py index 5058291a..b13a20a9 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -95,7 +95,9 @@ def limitedTime(*args, **kw): max_time_seconds = convert_mins_to_secs(max_eval_time_mins) signal.alarm(max_time_seconds) try: - ret = func(*args, **kw) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + ret = func(*args, **kw) except: raise TimedOutExc("Time Out!") finally: @@ -117,15 +119,17 @@ def run(self): # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) # Note: Need attention if using parallel execution model of scikit-learn current_thread().name = 'MainThread' - self.result = func(*self.args, **self.kwargs) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self.result = func(*self.args, **self.kwargs) except Exception: - pass + self.result = -float('inf') @wraps(func) - def limitedTime(*args, **kw): + def limitedTime(*args, **kwargs): sys.tracebacklimit = 0 max_time_seconds = convert_mins_to_secs(max_eval_time_mins) # start thread - tmp_it = InterruptableThread(args, kw) + tmp_it = InterruptableThread(args, kwargs) tmp_it.start() #timer = Timer(max_time_seconds, interrupt_main) tmp_it.join(max_time_seconds) diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 51f27c2f..9dde3eca 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -151,8 +151,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, pbar.update(len(offspring)-len(invalid_ind)) if not (max_time_mins is None) and pbar.n >= pbar.total: pbar.total += lambda_ - print(pbar.total) - print(pbar.n) + fitnesses = toolbox.evaluate(invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit From cdfdab47e1e72ab7d0140254d39ef9f9db1badcb Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 15:17:36 -0500 Subject: [PATCH 096/154] timeout func works --- test_log.py | 43 -------------------------------------- tpot/base.py | 3 ++- tpot_test_multi_process.py | 20 ------------------ 3 files changed, 2 insertions(+), 64 deletions(-) delete mode 100644 test_log.py delete mode 100644 tpot_test_multi_process.py diff --git a/test_log.py b/test_log.py deleted file mode 100644 index a077dd2a..00000000 --- a/test_log.py +++ /dev/null @@ -1,43 +0,0 @@ -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split, cross_val_score -from sklearn.base import BaseEstimator, clone -from sklearn.externals.joblib import Parallel, delayed -from sklearn.naive_bayes import GaussianNB -from sklearn.preprocessing import StandardScaler -from sklearn.pipeline import make_pipeline -import time -from tqdm import tqdm - -from functools import partial -from pathos.multiprocessing import ProcessPool - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,train_size=0.95, test_size=0.05) - -pbar = tqdm(total=10, unit='pipeline', leave=False, - disable=False, desc='Optimization Progress') - -pipeline = make_pipeline(StandardScaler(), GaussianNB(priors=None)) - -sklearn_pipelines = [pipeline]*10 - -def _wrapped_cross_val_score(sklearn_pipeline, features, classes, -cv, pbar): -import numpy as np - cv_scores = cross_val_score(sklearn_pipeline, features, classes, - cv=cv, n_jobs=1) - return np.mean(cv_scores) - - -partial_cross_val_score = partial(_wrapped_cross_val_score, features=X_train, classes=y_train, cv=3, pbar = pbar) -pool = ProcessPool(processes=2) -resulting_score_list = pool.imap(partial_cross_val_score, sklearn_pipelines) -pbar = tqdm(total=10, unit='pipeline', leave=False, - disable=False, desc='Optimization Progress') -while True: - pbar.update(resulting_score_list._index - pbar.n) - if resulting_score_list._index < 10: - pass - print(len(resulting_score_list._items)) - else: - break diff --git a/tpot/base.py b/tpot/base.py index f8c130e0..04634f49 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -726,7 +726,8 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe if not sys.platform.startswith('win'): pool = ProcessPool(processes=self.n_jobs) res_imap = pool.imap(_wrapped_cross_val_score, sklearn_pipeline_list) - ini_pbar_n = self._pbar.n + if not self._pbar.disable: + ini_pbar_n = self._pbar.n # hacky way for pbar update by using imap in pathos.multiprocessing.ProcessPool while True: tmp_fitness = np.array(res_imap._items) diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py deleted file mode 100644 index 31eb5790..00000000 --- a/tpot_test_multi_process.py +++ /dev/null @@ -1,20 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split -import time - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.25, test_size=0.75) - -#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) -#time_start = time.time() -#tpot.fit(X_train, y_train) -#print(tpot.score(X_test, y_test)) -#print('\nTime used with num_cpu = 1:',time.time()-time_start) - -tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.06, n_jobs = 2, random_state = 44)#, max_time_mins=1) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 3:',time.time()-time_start) From 6fc18bcd728688e05bdc7b65e8791bf1966c5533 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 15:32:46 -0500 Subject: [PATCH 097/154] retest --- tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests.py b/tests.py index 8c4c147f..74a5f2e5 100644 --- a/tests.py +++ b/tests.py @@ -276,7 +276,7 @@ def test_predict_proba2(): def test_warm_start(): """Assert that the TPOT warm_start flag stores the pop and pareto_front from the first run""" - tpot_obj = TPOTClassifier(random_state=42, population_size=2, offspring_size=4, generations=1, verbosity=0, warm_start=True) + tpot_obj = TPOTClassifier(random_state=42, population_size=1, offspring_size=2, generations=1, verbosity=0, warm_start=True) tpot_obj.fit(training_features, training_classes) assert tpot_obj._pop != None @@ -293,7 +293,7 @@ def test_warm_start(): def test_fit(): """Assert that the TPOT fit function provides an optimized pipeline""" - tpot_obj = TPOTClassifier(random_state=42, population_size=2, offspring_size=4, generations=1, verbosity=0) + tpot_obj = TPOTClassifier(random_state=42, population_size=1, offspring_size=2, generations=1, verbosity=0) tpot_obj.fit(training_features, training_classes) assert isinstance(tpot_obj._optimized_pipeline, creator.Individual) From 184162f294b4f8d2301237bcaff265aeaad5f7a9 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 15:50:29 -0500 Subject: [PATCH 098/154] not sure about issue in Traivs CI build --- tpot/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tpot/base.py b/tpot/base.py index 04634f49..b6664400 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -23,6 +23,7 @@ import inspect import warnings import sys +import time from functools import partial from datetime import datetime from pathos.multiprocessing import ProcessPool @@ -741,6 +742,8 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe self._pbar.update(ini_pbar_n + num_job_done - self._pbar.n) if num_job_done >= len(sklearn_pipeline_list): break + else: + time.sleep(0.2) resulting_score_list = list(res_imap) else: resulting_score_list = [] From 25fcda58f3eb8694166d3dcdc9dcc7dba0ce1301 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 16:03:41 -0500 Subject: [PATCH 099/154] clean codes --- tpot/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index b6664400..d1a15d9f 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -28,10 +28,6 @@ from datetime import datetime from pathos.multiprocessing import ProcessPool - -from sklearn.externals.joblib import Parallel, delayed - - import numpy as np import deap from deap import algorithms, base, creator, tools, gp @@ -730,7 +726,7 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe if not self._pbar.disable: ini_pbar_n = self._pbar.n # hacky way for pbar update by using imap in pathos.multiprocessing.ProcessPool - while True: + while not self._pbar.disable: tmp_fitness = np.array(res_imap._items) num_job_done = len(tmp_fitness) if not self._pbar.disable and num_job_done: From 32aac2820a768e8f1b35030b94702c002328df7c Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 16:04:17 -0500 Subject: [PATCH 100/154] clean codes --- tpot/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index d1a15d9f..5dc2f719 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -33,7 +33,7 @@ from deap import algorithms, base, creator, tools, gp from tqdm import tqdm -from sklearn.base import BaseEstimator, clone +from sklearn.base import BaseEstimator from sklearn.model_selection import cross_val_score from sklearn.pipeline import make_pipeline, make_union from sklearn.preprocessing import FunctionTransformer From 9fff614b93e16d379badce19fbb0968ba2da992a Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Feb 2017 16:39:40 -0500 Subject: [PATCH 101/154] add import module warning --- tpot/base.py | 5 +- tpot/operator_utils.py | 241 +++++++++++++++++++++-------------------- 2 files changed, 127 insertions(+), 119 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 34380932..eb7839bf 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -186,8 +186,9 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.arguments = [] for key in sorted(self.operator_dict.keys()): op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key]) - self.operators.append(op_class) - self.arguments += arg_types + if op_class: + self.operators.append(op_class) + self.arguments += arg_types # Schedule TPOT to run for a very long time if the user specifies a run-time # limit TPOT will automatically interrupt itself when the timer runs out diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 5669cf3c..1e490a9c 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -61,11 +61,15 @@ def source_decode(sourcecode): tmp_path = sourcecode.split('.') op_str = tmp_path.pop() import_str = '.'.join(tmp_path) - if sourcecode.startswith('tpot.'): - exec('from {} import {}'.format(import_str[4:], op_str)) - else: - exec('from {} import {}'.format(import_str, op_str)) - op_obj = eval(op_str) + try: + if sourcecode.startswith('tpot.'): + exec('from {} import {}'.format(import_str[4:], op_str)) + else: + exec('from {} import {}'.format(import_str, op_str)) + op_obj = eval(op_str) + except ImportError: + print("Operator {} is not available".format(sourcecode)) + op_obj = None return import_str, op_str, op_obj def ARGTypeClassFactory(classname, prange, BaseClass=ARGType): @@ -103,117 +107,120 @@ def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator): dep_op_list = {} import_str, op_str, op_obj = source_decode(opsourse) - # define if the operator can be the root of a pipeline - if issubclass(op_obj, ClassifierMixin) or issubclass(op_obj, RegressorMixin): - class_profile['root'] = True - optype = "Classifier or Regressor" + if not op_obj: + return None, None # nothing return else: - optype = "Preprocessor or Selector" - @classmethod - def op_type(cls): - """Returns the type of the operator, e.g: - ("Classifier", "Regressor", "Selector", "Preprocessor") - """ - return optype - - class_profile['type'] = op_type - - class_profile['sklearn_class'] = op_obj - - import_hash = {} - import_hash[import_str] = [op_str] - arg_types = [] - for pname in sorted(opdict.keys()): - prange = opdict[pname] - if not isinstance(prange, dict): - classname = '{}__{}'.format(op_str, pname) - arg_types.append(ARGTypeClassFactory(classname, prange)) + # define if the operator can be the root of a pipeline + if issubclass(op_obj, ClassifierMixin) or issubclass(op_obj, RegressorMixin): + class_profile['root'] = True + optype = "Classifier or Regressor" else: - for dkey, dval in prange.items(): - dep_import_str, dep_op_str, dep_op_obj = source_decode(dkey) - if dep_import_str in import_hash: - import_hash[import_str].append(dep_op_str) - else: - import_hash[dep_import_str] = [dep_op_str] - dep_op_list[pname]=dep_op_str - if dval: - for dpname in sorted(dval.keys()): - dprange = dval[dpname] - classname = '{}__{}__{}'.format(op_str, dep_op_str, dpname) - arg_types.append(ARGTypeClassFactory(classname, dprange)) - class_profile['arg_types'] = tuple(arg_types) - class_profile['import_hash'] = import_hash - class_profile['dep_op_list'] = dep_op_list - @classmethod - def parameter_types(cls): - """Return tuple of argument types for calling of the operator and the - return type of the operator - - Parameters - ---------- - None - - Returns - ------- - parameter_types: tuple - Tuple of the DEAP parameter types and the DEAP return type for the - operator - - """ - return ([np.ndarray] + arg_types, np.ndarray) - - - class_profile['parameter_types'] = parameter_types - @classmethod - def export(cls, *args): - """Represent the operator as a string so that it can be exported to a - file - - Parameters - ---------- - args - Arbitrary arguments to be passed to the operator - - Returns - ------- - export_string: str - String representation of the sklearn class with its parameters in - the format: - SklearnClassName(param1="val1", param2=val2) - - """ - - op_arguments = [] - if dep_op_list: - dep_op_arguments = {} - for arg_class, arg_value in zip(arg_types, args): - aname_split = arg_class.__name__.split('__') - if isinstance(arg_value, str): - arg_value = '\"{}\"'.format(arg_value) - if len(aname_split) == 2: # simple parameter - op_arguments.append("{}={}".format(aname_split[-1], arg_value)) - else: # parameter of internal operator as a parameter in the operator, usually in Selector - if not list(dep_op_list.values()).count(aname_split[1]): - raise TypeError('Warning: the {} is not in right format!'.format(self.sklearn_class.__name__)) - else: - if aname_split[1] not in dep_op_arguments: - dep_op_arguments[aname_split[1]] = [] - dep_op_arguments[aname_split[1]].append("{}={}".format(aname_split[-1], arg_value)) - tmp_op_args = [] - if dep_op_list: - # to make sure the inital operators is the first parameter just for better persentation - for dep_op_pname, dep_op_str in dep_op_list.items(): - if dep_op_str == 'f_classif': - arg_value = dep_op_str - else: - arg_value = "{}({})".format(dep_op_str, ", ".join(dep_op_arguments[dep_op_str])) - tmp_op_args.append("{}={}".format(dep_op_pname, arg_value)) - op_arguments = tmp_op_args + op_arguments - return "{}({})".format(op_obj.__name__, ", ".join(op_arguments)) - - class_profile['export'] = export - - op_classname = 'TPOT_{}'.format(op_str) - op_class = type(op_classname, (BaseClass,), class_profile) - op_class.__name__ = op_str - return op_class, arg_types + optype = "Preprocessor or Selector" + @classmethod + def op_type(cls): + """Returns the type of the operator, e.g: + ("Classifier", "Regressor", "Selector", "Preprocessor") + """ + return optype + + class_profile['type'] = op_type + + class_profile['sklearn_class'] = op_obj + + import_hash = {} + import_hash[import_str] = [op_str] + arg_types = [] + for pname in sorted(opdict.keys()): + prange = opdict[pname] + if not isinstance(prange, dict): + classname = '{}__{}'.format(op_str, pname) + arg_types.append(ARGTypeClassFactory(classname, prange)) + else: + for dkey, dval in prange.items(): + dep_import_str, dep_op_str, dep_op_obj = source_decode(dkey) + if dep_import_str in import_hash: + import_hash[import_str].append(dep_op_str) + else: + import_hash[dep_import_str] = [dep_op_str] + dep_op_list[pname]=dep_op_str + if dval: + for dpname in sorted(dval.keys()): + dprange = dval[dpname] + classname = '{}__{}__{}'.format(op_str, dep_op_str, dpname) + arg_types.append(ARGTypeClassFactory(classname, dprange)) + class_profile['arg_types'] = tuple(arg_types) + class_profile['import_hash'] = import_hash + class_profile['dep_op_list'] = dep_op_list + @classmethod + def parameter_types(cls): + """Return tuple of argument types for calling of the operator and the + return type of the operator + + Parameters + ---------- + None + + Returns + ------- + parameter_types: tuple + Tuple of the DEAP parameter types and the DEAP return type for the + operator + + """ + return ([np.ndarray] + arg_types, np.ndarray) + + + class_profile['parameter_types'] = parameter_types + @classmethod + def export(cls, *args): + """Represent the operator as a string so that it can be exported to a + file + + Parameters + ---------- + args + Arbitrary arguments to be passed to the operator + + Returns + ------- + export_string: str + String representation of the sklearn class with its parameters in + the format: + SklearnClassName(param1="val1", param2=val2) + + """ + + op_arguments = [] + if dep_op_list: + dep_op_arguments = {} + for arg_class, arg_value in zip(arg_types, args): + aname_split = arg_class.__name__.split('__') + if isinstance(arg_value, str): + arg_value = '\"{}\"'.format(arg_value) + if len(aname_split) == 2: # simple parameter + op_arguments.append("{}={}".format(aname_split[-1], arg_value)) + else: # parameter of internal operator as a parameter in the operator, usually in Selector + if not list(dep_op_list.values()).count(aname_split[1]): + raise TypeError('Warning: the {} is not in right format!'.format(self.sklearn_class.__name__)) + else: + if aname_split[1] not in dep_op_arguments: + dep_op_arguments[aname_split[1]] = [] + dep_op_arguments[aname_split[1]].append("{}={}".format(aname_split[-1], arg_value)) + tmp_op_args = [] + if dep_op_list: + # to make sure the inital operators is the first parameter just for better persentation + for dep_op_pname, dep_op_str in dep_op_list.items(): + if dep_op_str == 'f_classif': + arg_value = dep_op_str + else: + arg_value = "{}({})".format(dep_op_str, ", ".join(dep_op_arguments[dep_op_str])) + tmp_op_args.append("{}={}".format(dep_op_pname, arg_value)) + op_arguments = tmp_op_args + op_arguments + return "{}({})".format(op_obj.__name__, ", ".join(op_arguments)) + + class_profile['export'] = export + + op_classname = 'TPOT_{}'.format(op_str) + op_class = type(op_classname, (BaseClass,), class_profile) + op_class.__name__ = op_str + return op_class, arg_types From 618d480aa0f6e18b0de591859ddceed9c6fda2ae Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 23 Feb 2017 11:34:07 -0500 Subject: [PATCH 102/154] fix timeout value --- tpot/base.py | 9 +++++---- tpot_test_multi_process.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 tpot_test_multi_process.py diff --git a/tpot/base.py b/tpot/base.py index 5dc2f719..0bfe7d7e 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -715,7 +715,7 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe n_jobs=1, fit_params=sample_weight_dict) resulting_score = np.mean(cv_scores) except TimedOutExc: - resulting_score = -500000000. + resulting_score = "Timeout" except: resulting_score = -float('inf') return resulting_score @@ -730,7 +730,7 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe tmp_fitness = np.array(res_imap._items) num_job_done = len(tmp_fitness) if not self._pbar.disable and num_job_done: - timeout_index = list(np.where(tmp_fitness[:,1] == -500000000.)[0]) + timeout_index = list(np.where(tmp_fitness[:,1] == "Timeout")[0]) for idx in timeout_index: if self._pbar.n - ini_pbar_n <= idx: self._pbar.write("Skip pipeline #{0} due to time out. " @@ -740,12 +740,13 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe break else: time.sleep(0.2) - resulting_score_list = list(res_imap) + resulting_score_list = [-float('inf') if x=="Timeout" else x for x in list(res_imap)] else: resulting_score_list = [] for sklearn_pipeline in sklearn_pipeline_list: resulting_score = _wrapped_cross_val_score(sklearn_pipeline) - if resulting_score == -500000000.: + if resulting_score == "Timeout": + resulting_score = -float('inf') if not self._pbar.disable: self._pbar.write("Skip pipeline #{0} due to time out. " "Continuing to the next pipeline.".format(self._pbar.n + 1)) diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py new file mode 100644 index 00000000..31eb5790 --- /dev/null +++ b/tpot_test_multi_process.py @@ -0,0 +1,20 @@ +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split +import time + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.25, test_size=0.75) + +#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) +#time_start = time.time() +#tpot.fit(X_train, y_train) +#print(tpot.score(X_test, y_test)) +#print('\nTime used with num_cpu = 1:',time.time()-time_start) + +tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.06, n_jobs = 2, random_state = 44)#, max_time_mins=1) +time_start = time.time() +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +print('\nTime used with num_cpu = 3:',time.time()-time_start) From 3db7ed123edcb33623995b5a3d327554ac2d880f Mon Sep 17 00:00:00 2001 From: weixuanfu2016 Date: Thu, 23 Feb 2017 11:58:47 -0500 Subject: [PATCH 103/154] fix issue in windows --- TpotClassifier.ipynb.py | 87 ++++++++++++++++++++++++++++++++++++++ test.py | 11 +++++ test_export.py | 57 +++++++++++++++++++++++++ tpot/base.py | 7 +-- tpot/decorators.py | 15 +++---- tpot_test_multi_process.py | 2 +- 6 files changed, 166 insertions(+), 13 deletions(-) create mode 100644 TpotClassifier.ipynb.py create mode 100644 test.py create mode 100644 test_export.py diff --git a/TpotClassifier.ipynb.py b/TpotClassifier.ipynb.py new file mode 100644 index 00000000..29f98b88 --- /dev/null +++ b/TpotClassifier.ipynb.py @@ -0,0 +1,87 @@ + +# coding: utf-8 + +# In[1]: + +import pandas as pd +from tpot import TPOTClassifier +from sklearn.model_selection import train_test_split + + +# In[2]: + +train_data = pd.read_csv('train.csv') +test_data = pd.read_csv('test.csv') + + +# In[3]: + +combined_set = pd.concat([train_data, test_data]) +combined_set['combined_var'] = (combined_set.hair_length * .40) + (combined_set.has_soul * .40) + +# Replace categorical variables with numbers +def label_encoding(df, col): + label_map = { key: float(n) for n, key in enumerate(df[col].unique()) } + label_reverse_map = { label_map[key]: key for key in label_map } + df[col] = df[col].apply(lambda x: label_map[x]) + return df, label_map, label_reverse_map + +combined_set = pd.get_dummies(combined_set, columns=['color']) +combined_set + +train_set = combined_set[:len(train_data.index)] +test_set = combined_set[len(train_data.index):] + + +# In[4]: + +train_cols = ['combined_var', 'rotting_flesh', 'bone_length', 'has_soul', 'hair_length'] +target_var = ['type'] +selected_cols = train_cols + target_var + + +# In[5]: + +train_set, type_label_map, type_label_reverse_map = label_encoding(train_set, 'type') + + +# In[6]: + +p_train,val = train_test_split(train_set, train_size=.75, test_size=.25) + + +# In[7]: + +p_train.shape, val.shape + + +# In[8]: + +p_train[train_cols].head().values + + +# In[9]: + +tpot = TPOTClassifier(verbosity=3, generations = 5) + + + +# In[12]: +#print(pd.np.array(p_train[train_cols])) +print(pd.np.array(p_train[target_var])) +print(pd.np.array(p_train[target_var]).ravel()) + +tpot.fit(pd.np.array(p_train[train_cols]), pd.np.array(p_train[target_var]).ravel()) + + +# In[21]: + +p_train[train_cols].head() + + +# In[23]: + +p_train[target_var].head() + + +# In[ ]: diff --git a/test.py b/test.py new file mode 100644 index 00000000..fd5ebe9c --- /dev/null +++ b/test.py @@ -0,0 +1,11 @@ +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.10, test_size=0.90) + +tpot = TPOTClassifier(generations=5, population_size=20, verbosity=3) +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) diff --git a/test_export.py b/test_export.py new file mode 100644 index 00000000..71f5a821 --- /dev/null +++ b/test_export.py @@ -0,0 +1,57 @@ +from tpot import TPOTClassifier +from sklearn.model_selection import train_test_split +import pandas as pd +import numpy as np + +titanic = pd.read_csv('tutorials/data/titanic_train.csv') +titanic.head(5) +titanic.groupby('Sex').Survived.value_counts() +titanic.groupby(['Pclass','Sex']).Survived.value_counts() +id = pd.crosstab([titanic.Pclass, titanic.Sex], titanic.Survived.astype(float)) +id.div(id.sum(1).astype(float), 0) + + +titanic.rename(columns={'Survived': 'class'}, inplace=True) +titanic.dtypes + +for cat in ['Name', 'Sex', 'Ticket', 'Cabin', 'Embarked']: + print("Number of levels in category '{0}': \b {1:2.2f} ".format(cat, titanic[cat].unique().size)) + +for cat in ['Sex', 'Embarked']: + print("Levels for catgeory '{0}': {1}".format(cat, titanic[cat].unique())) + +titanic['Sex'] = titanic['Sex'].map({'male':0,'female':1}) +titanic['Embarked'] = titanic['Embarked'].map({'S':0,'C':1,'Q':2}) + +titanic = titanic.fillna(-999) +pd.isnull(titanic).any() + +from sklearn.preprocessing import MultiLabelBinarizer +mlb = MultiLabelBinarizer() +CabinTrans = mlb.fit_transform([{str(val)} for val in titanic['Cabin'].values]) + +CabinTrans + +titanic_new = titanic.drop(['Name','Ticket','Cabin','class'], axis=1) + +assert (len(titanic['Cabin'].unique()) == len(mlb.classes_)), "Not Equal" #check correct encoding done + +titanic_new = np.hstack((titanic_new.values,CabinTrans)) + +np.isnan(titanic_new).any() + +titanic_new[0].size + +titanic_class = titanic['class'].values + +training_indices, validation_indices = training_indices, testing_indices = train_test_split(titanic.index, stratify = titanic_class, train_size=0.05, test_size=0.95) +training_indices.size, validation_indices.size + +tpot = TPOTClassifier(verbosity=2, max_time_mins=2, generations= 5) +print(titanic_new[training_indices]) +print(titanic_class[training_indices]) +tpot.fit(titanic_new[training_indices], titanic_class[training_indices]) + +#tpot.score(titanic_new[validation_indices], titanic.loc[validation_indices, 'class'].values) + +#tpot.export('tpot_titanic_pipeline.py') diff --git a/tpot/base.py b/tpot/base.py index 0bfe7d7e..605bf1b8 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -46,7 +46,7 @@ from .operator_utils import TPOTOperatorClassFactory, Operator, ARGType from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code -from .decorators import _timeout, _pre_test +from .decorators import _timeout, _pre_test, TimedOutExc from .build_in_operators import CombineDFs @@ -744,8 +744,9 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe else: resulting_score_list = [] for sklearn_pipeline in sklearn_pipeline_list: - resulting_score = _wrapped_cross_val_score(sklearn_pipeline) - if resulting_score == "Timeout": + try: + resulting_score = _wrapped_cross_val_score(sklearn_pipeline) + except TimedOutExc: resulting_score = -float('inf') if not self._pbar.disable: self._pbar.write("Skip pipeline #{0} due to time out. " diff --git a/tpot/decorators.py b/tpot/decorators.py index b13a20a9..0a4afb22 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -115,15 +115,12 @@ def __init__(self, args, kwargs): def stop(self): self._stop() def run(self): - try: - # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) - # Note: Need attention if using parallel execution model of scikit-learn - current_thread().name = 'MainThread' - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self.result = func(*self.args, **self.kwargs) - except Exception: - self.result = -float('inf') + # Note: changed name of the thread to "MainThread" to avoid such warning from joblib (maybe bugs) + # Note: Need attention if using parallel execution model of scikit-learn + current_thread().name = 'MainThread' + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + self.result = func(*self.args, **self.kwargs) @wraps(func) def limitedTime(*args, **kwargs): sys.tracebacklimit = 0 diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py index 31eb5790..1d9eee78 100644 --- a/tpot_test_multi_process.py +++ b/tpot_test_multi_process.py @@ -13,7 +13,7 @@ #print(tpot.score(X_test, y_test)) #print('\nTime used with num_cpu = 1:',time.time()-time_start) -tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.06, n_jobs = 2, random_state = 44)#, max_time_mins=1) +tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.04, n_jobs = 2, random_state = 44)#, max_time_mins=1) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From c89870c77f12a4df42d735d3386476da838fb364 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 23 Feb 2017 12:04:22 -0500 Subject: [PATCH 104/154] clean codes --- TpotClassifier.ipynb.py | 87 -------------------------------------- test.py | 11 ----- test_export.py | 57 ------------------------- tpot_test_multi_process.py | 20 --------- 4 files changed, 175 deletions(-) delete mode 100644 TpotClassifier.ipynb.py delete mode 100644 test.py delete mode 100644 test_export.py delete mode 100644 tpot_test_multi_process.py diff --git a/TpotClassifier.ipynb.py b/TpotClassifier.ipynb.py deleted file mode 100644 index 29f98b88..00000000 --- a/TpotClassifier.ipynb.py +++ /dev/null @@ -1,87 +0,0 @@ - -# coding: utf-8 - -# In[1]: - -import pandas as pd -from tpot import TPOTClassifier -from sklearn.model_selection import train_test_split - - -# In[2]: - -train_data = pd.read_csv('train.csv') -test_data = pd.read_csv('test.csv') - - -# In[3]: - -combined_set = pd.concat([train_data, test_data]) -combined_set['combined_var'] = (combined_set.hair_length * .40) + (combined_set.has_soul * .40) - -# Replace categorical variables with numbers -def label_encoding(df, col): - label_map = { key: float(n) for n, key in enumerate(df[col].unique()) } - label_reverse_map = { label_map[key]: key for key in label_map } - df[col] = df[col].apply(lambda x: label_map[x]) - return df, label_map, label_reverse_map - -combined_set = pd.get_dummies(combined_set, columns=['color']) -combined_set - -train_set = combined_set[:len(train_data.index)] -test_set = combined_set[len(train_data.index):] - - -# In[4]: - -train_cols = ['combined_var', 'rotting_flesh', 'bone_length', 'has_soul', 'hair_length'] -target_var = ['type'] -selected_cols = train_cols + target_var - - -# In[5]: - -train_set, type_label_map, type_label_reverse_map = label_encoding(train_set, 'type') - - -# In[6]: - -p_train,val = train_test_split(train_set, train_size=.75, test_size=.25) - - -# In[7]: - -p_train.shape, val.shape - - -# In[8]: - -p_train[train_cols].head().values - - -# In[9]: - -tpot = TPOTClassifier(verbosity=3, generations = 5) - - - -# In[12]: -#print(pd.np.array(p_train[train_cols])) -print(pd.np.array(p_train[target_var])) -print(pd.np.array(p_train[target_var]).ravel()) - -tpot.fit(pd.np.array(p_train[train_cols]), pd.np.array(p_train[target_var]).ravel()) - - -# In[21]: - -p_train[train_cols].head() - - -# In[23]: - -p_train[target_var].head() - - -# In[ ]: diff --git a/test.py b/test.py deleted file mode 100644 index fd5ebe9c..00000000 --- a/test.py +++ /dev/null @@ -1,11 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.10, test_size=0.90) - -tpot = TPOTClassifier(generations=5, population_size=20, verbosity=3) -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) diff --git a/test_export.py b/test_export.py deleted file mode 100644 index 71f5a821..00000000 --- a/test_export.py +++ /dev/null @@ -1,57 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.model_selection import train_test_split -import pandas as pd -import numpy as np - -titanic = pd.read_csv('tutorials/data/titanic_train.csv') -titanic.head(5) -titanic.groupby('Sex').Survived.value_counts() -titanic.groupby(['Pclass','Sex']).Survived.value_counts() -id = pd.crosstab([titanic.Pclass, titanic.Sex], titanic.Survived.astype(float)) -id.div(id.sum(1).astype(float), 0) - - -titanic.rename(columns={'Survived': 'class'}, inplace=True) -titanic.dtypes - -for cat in ['Name', 'Sex', 'Ticket', 'Cabin', 'Embarked']: - print("Number of levels in category '{0}': \b {1:2.2f} ".format(cat, titanic[cat].unique().size)) - -for cat in ['Sex', 'Embarked']: - print("Levels for catgeory '{0}': {1}".format(cat, titanic[cat].unique())) - -titanic['Sex'] = titanic['Sex'].map({'male':0,'female':1}) -titanic['Embarked'] = titanic['Embarked'].map({'S':0,'C':1,'Q':2}) - -titanic = titanic.fillna(-999) -pd.isnull(titanic).any() - -from sklearn.preprocessing import MultiLabelBinarizer -mlb = MultiLabelBinarizer() -CabinTrans = mlb.fit_transform([{str(val)} for val in titanic['Cabin'].values]) - -CabinTrans - -titanic_new = titanic.drop(['Name','Ticket','Cabin','class'], axis=1) - -assert (len(titanic['Cabin'].unique()) == len(mlb.classes_)), "Not Equal" #check correct encoding done - -titanic_new = np.hstack((titanic_new.values,CabinTrans)) - -np.isnan(titanic_new).any() - -titanic_new[0].size - -titanic_class = titanic['class'].values - -training_indices, validation_indices = training_indices, testing_indices = train_test_split(titanic.index, stratify = titanic_class, train_size=0.05, test_size=0.95) -training_indices.size, validation_indices.size - -tpot = TPOTClassifier(verbosity=2, max_time_mins=2, generations= 5) -print(titanic_new[training_indices]) -print(titanic_class[training_indices]) -tpot.fit(titanic_new[training_indices], titanic_class[training_indices]) - -#tpot.score(titanic_new[validation_indices], titanic.loc[validation_indices, 'class'].values) - -#tpot.export('tpot_titanic_pipeline.py') diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py deleted file mode 100644 index 1d9eee78..00000000 --- a/tpot_test_multi_process.py +++ /dev/null @@ -1,20 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split -import time - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.25, test_size=0.75) - -#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) -#time_start = time.time() -#tpot.fit(X_train, y_train) -#print(tpot.score(X_test, y_test)) -#print('\nTime used with num_cpu = 1:',time.time()-time_start) - -tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, max_eval_time_mins=0.04, n_jobs = 2, random_state = 44)#, max_time_mins=1) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 3:',time.time()-time_start) From a2bdcfcaa00f6e6e4370d40ab06d2a0ca1fd4fc9 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Thu, 23 Feb 2017 13:39:59 -0500 Subject: [PATCH 105/154] Change Pareto front similarity criteria Instead of requiring an exact match for float value comparisons on the Pareto front, consider two solutions to be equal if they are within a reasonable tolerance level away from each other. --- tpot/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index eb7839bf..a7a053a3 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -370,7 +370,7 @@ def pareto_eq(ind1, ind2): the Pareto front """ - return np.all(ind1.fitness.values == ind2.fitness.values) + return np.allclose(ind1.fitness.values, ind2.fitness.values) # generate new pareto front if it doesn't already exist for warm start if not self.warm_start or not self._pareto_front: From dfde6d1d65d8f834958ec89378a71f84dc01127c Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 23 Feb 2017 14:31:06 -0500 Subject: [PATCH 106/154] fit parote_eq --- tpot/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index f21f2540..40f3b907 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -384,7 +384,7 @@ def pareto_eq(ind1, ind2): return np.allclose(ind1.fitness.values, ind2.fitness.values) # generate new pareto front if it doesn't already exist for warm start if not self.warm_start or not self._pareto_front: - self._pareto_front = tools.ParetoFront(similar=self._pareto_eq) + self._pareto_front = tools.ParetoFront(similar=pareto_eq) # Start the progress bar if self.max_time_mins: From e8b613554232025133c317e1186c83e4b29bb202 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 23 Feb 2017 14:43:22 -0500 Subject: [PATCH 107/154] nodes number bug fixt --- tpot/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index 40f3b907..ade2d201 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -739,7 +739,7 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe return resulting_score if not sys.platform.startswith('win'): - pool = ProcessPool(processes=self.n_jobs) + pool = ProcessPool(nodes=self.n_jobs) res_imap = pool.imap(_wrapped_cross_val_score, sklearn_pipeline_list) if not self._pbar.disable: ini_pbar_n = self._pbar.n From 45aba6c24c9cf24ede8ffee16b3c5030414210d6 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 23 Feb 2017 16:20:04 -0500 Subject: [PATCH 108/154] add -1 option --- tpot/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index ade2d201..cef52289 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -739,7 +739,10 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe return resulting_score if not sys.platform.startswith('win'): - pool = ProcessPool(nodes=self.n_jobs) + if self.n_jobs == -1: + pool = ProcessPool() + else: + pool = ProcessPool(nodes=self.n_jobs) res_imap = pool.imap(_wrapped_cross_val_score, sklearn_pipeline_list) if not self._pbar.disable: ini_pbar_n = self._pbar.n From bf1f5b57f3344bc35abca6fde1da8c41fa718775 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 24 Feb 2017 17:05:24 -0500 Subject: [PATCH 109/154] fix setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 053550e9..787673cc 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def calculate_version(): This project is hosted at https://github.com/rhiever/tpot ''', zip_safe=True, - install_requires=['numpy', 'scipy', 'scikit-learn', 'deap', 'update_checker', 'tqdm'], + install_requires=['numpy>=1.11.2', 'scipy>=0.18.1', 'scikit-learn>=0.18.1', 'deap>=1.0.2', 'update_checker>=0.12', 'tqdm>=4.9.0', 'pathos>=0.2.0'], extras_require={'xgboost': ['xgboost']}, classifiers=[ 'Intended Audience :: Science/Research', From 8f03c1067321951f0a3feb9e704037cbc5916be7 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Mar 2017 14:07:06 -0500 Subject: [PATCH 110/154] clean up decorators --- tpot/decorators.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/tpot/decorators.py b/tpot/decorators.py index 0a4afb22..cefb8c86 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -29,30 +29,11 @@ pretest_X, pretest_y = make_classification(n_samples=50, n_features=10, random_state=42) pretest_X_reg, pretest_y_reg = make_regression(n_samples=50, n_features=10, random_state=42) -def convert_mins_to_secs(time_minute): - """Convert time from minutes to seconds""" - second = int(time_minute * 60) - # time limit should be at least 1 second - return max(second, 1) - - -class TimedOutExc(RuntimeError): - """ - Raised when a timeout happens - """ - -def timeout_signal_handler(signum, frame): - """ - signal handler for _timeout function - rasie TIMEOUT exception - """ - raise TimedOutExc("Time Out!") def convert_mins_to_secs(time_minute): """Convert time from minutes to seconds""" - second = int(time_minute * 60) # time limit should be at least 1 second - return max(second, 1) + return max(int(time_minute * 60), 1) class TimedOutExc(RuntimeError): @@ -72,7 +53,7 @@ def _timeout(max_eval_time_mins=5): Parameters ---------- - time_minute: int + max_eval_time_mins: int (default = 5) Time limit in minutes func: Python function Function to run From 5e3796fa7c6338504ea1c28435ef16b8d65bd325 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Mar 2017 15:50:08 -0500 Subject: [PATCH 111/154] test timeout --- tests.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests.py b/tests.py index 74a5f2e5..7401196e 100644 --- a/tests.py +++ b/tests.py @@ -10,6 +10,7 @@ from tpot.export_utils import export_pipeline, generate_import_code, _indent, generate_pipeline_code, get_by_name from tpot.gp_types import Output_DF from tpot.gp_deap import mutNodeReplacement +from tpot.decorators import _timeout, TimedOutExc from tpot.operator_utils import TPOTOperatorClassFactory from tpot.config_classifier import classifier_config_dict @@ -18,6 +19,8 @@ import numpy as np import inspect import random +import time +import os from datetime import datetime from sklearn.datasets import load_digits, load_boston @@ -69,6 +72,23 @@ def test_init_custom_parameters(): assert not (tpot_obj._toolbox is None) +def test_timeout(): + """Assert that timeout decorator controls the currect running time of wrapped function""" + @_timeout(max_eval_time_mins=0.02) # just 1 second + def test_timeout_func(): + start_time = time.time() + try: + time.sleep(100) + return 100 + except TimedOutExc: + return time.time() - start_time + ret_timeout = int(test_timeout_func()) + assert ret_timeout == 1 + +#def test_command_line(): + + + def test_init_default_scoring(): """Assert that TPOT intitializes with the correct default scoring function""" From 537164f2d2d5ddfd8be5013bb89bc250ac58cd3f Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Mar 2017 15:54:23 -0500 Subject: [PATCH 112/154] clean codes --- setup.py | 2 +- tests.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 787673cc..8efdf7bc 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def calculate_version(): This project is hosted at https://github.com/rhiever/tpot ''', zip_safe=True, - install_requires=['numpy>=1.11.2', 'scipy>=0.18.1', 'scikit-learn>=0.18.1', 'deap>=1.0.2', 'update_checker>=0.12', 'tqdm>=4.9.0', 'pathos>=0.2.0'], + install_requires=['numpy', 'scipy', 'scikit-learn>=0.18.1', 'deap', 'update_checker', 'tqdm', 'pathos'], extras_require={'xgboost': ['xgboost']}, classifiers=[ 'Intended Audience :: Science/Research', diff --git a/tests.py b/tests.py index 7401196e..998525c6 100644 --- a/tests.py +++ b/tests.py @@ -85,9 +85,6 @@ def test_timeout_func(): ret_timeout = int(test_timeout_func()) assert ret_timeout == 1 -#def test_command_line(): - - def test_init_default_scoring(): """Assert that TPOT intitializes with the correct default scoring function""" From f96c5adbababe3c9ae1795ba283bcf2d237d024f Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Mar 2017 15:55:48 -0500 Subject: [PATCH 113/154] clean codes --- tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests.py b/tests.py index 998525c6..99ea14d3 100644 --- a/tests.py +++ b/tests.py @@ -20,7 +20,6 @@ import inspect import random import time -import os from datetime import datetime from sklearn.datasets import load_digits, load_boston From 7f00b64f3b7df78a342aafabca7f26bd6840042d Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Mar 2017 16:58:12 -0500 Subject: [PATCH 114/154] unit test for driver --- tests.csv | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests.py | 10 ++++++ 2 files changed, 111 insertions(+) create mode 100644 tests.csv diff --git a/tests.csv b/tests.csv new file mode 100644 index 00000000..70570d30 --- /dev/null +++ b/tests.csv @@ -0,0 +1,101 @@ +class,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +1,17.99,10.38,122.8,1001,0.1184,0.2776,0.3001,0.1471,0.2419,0.07871,1.095,0.9053,8.589,153.4,0.006399,0.04904,0.05373,0.01587,0.03003,0.006193,25.38,17.33,184.6,2019,0.1622,0.6656,0.7119,0.2654,0.4601,0.1189 +1,20.57,17.77,132.9,1326,0.08474,0.07864,0.0869,0.07017,0.1812,0.05667,0.5435,0.7339,3.398,74.08,0.005225,0.01308,0.0186,0.0134,0.01389,0.003532,24.99,23.41,158.8,1956,0.1238,0.1866,0.2416,0.186,0.275,0.08902 +1,19.69,21.25,130,1203,0.1096,0.1599,0.1974,0.1279,0.2069,0.05999,0.7456,0.7869,4.585,94.03,0.00615,0.04006,0.03832,0.02058,0.0225,0.004571,23.57,25.53,152.5,1709,0.1444,0.4245,0.4504,0.243,0.3613,0.08758 +1,11.42,20.38,77.58,386.1,0.1425,0.2839,0.2414,0.1052,0.2597,0.09744,0.4956,1.156,3.445,27.23,0.00911,0.07458,0.05661,0.01867,0.05963,0.009208,14.91,26.5,98.87,567.7,0.2098,0.8663,0.6869,0.2575,0.6638,0.173 +1,20.29,14.34,135.1,1297,0.1003,0.1328,0.198,0.1043,0.1809,0.05883,0.7572,0.7813,5.438,94.44,0.01149,0.02461,0.05688,0.01885,0.01756,0.005115,22.54,16.67,152.2,1575,0.1374,0.205,0.4,0.1625,0.2364,0.07678 +1,12.45,15.7,82.57,477.1,0.1278,0.17,0.1578,0.08089,0.2087,0.07613,0.3345,0.8902,2.217,27.19,0.00751,0.03345,0.03672,0.01137,0.02165,0.005082,15.47,23.75,103.4,741.6,0.1791,0.5249,0.5355,0.1741,0.3985,0.1244 +1,18.25,19.98,119.6,1040,0.09463,0.109,0.1127,0.074,0.1794,0.05742,0.4467,0.7732,3.18,53.91,0.004314,0.01382,0.02254,0.01039,0.01369,0.002179,22.88,27.66,153.2,1606,0.1442,0.2576,0.3784,0.1932,0.3063,0.08368 +1,13.71,20.83,90.2,577.9,0.1189,0.1645,0.09366,0.05985,0.2196,0.07451,0.5835,1.377,3.856,50.96,0.008805,0.03029,0.02488,0.01448,0.01486,0.005412,17.06,28.14,110.6,897,0.1654,0.3682,0.2678,0.1556,0.3196,0.1151 +1,13,21.82,87.5,519.8,0.1273,0.1932,0.1859,0.09353,0.235,0.07389,0.3063,1.002,2.406,24.32,0.005731,0.03502,0.03553,0.01226,0.02143,0.003749,15.49,30.73,106.2,739.3,0.1703,0.5401,0.539,0.206,0.4378,0.1072 +1,12.46,24.04,83.97,475.9,0.1186,0.2396,0.2273,0.08543,0.203,0.08243,0.2976,1.599,2.039,23.94,0.007149,0.07217,0.07743,0.01432,0.01789,0.01008,15.09,40.68,97.65,711.4,0.1853,1.058,1.105,0.221,0.4366,0.2075 +1,16.02,23.24,102.7,797.8,0.08206,0.06669,0.03299,0.03323,0.1528,0.05697,0.3795,1.187,2.466,40.51,0.004029,0.009269,0.01101,0.007591,0.0146,0.003042,19.19,33.88,123.8,1150,0.1181,0.1551,0.1459,0.09975,0.2948,0.08452 +1,15.78,17.89,103.6,781,0.0971,0.1292,0.09954,0.06606,0.1842,0.06082,0.5058,0.9849,3.564,54.16,0.005771,0.04061,0.02791,0.01282,0.02008,0.004144,20.42,27.28,136.5,1299,0.1396,0.5609,0.3965,0.181,0.3792,0.1048 +1,19.17,24.8,132.4,1123,0.0974,0.2458,0.2065,0.1118,0.2397,0.078,0.9555,3.568,11.07,116.2,0.003139,0.08297,0.0889,0.0409,0.04484,0.01284,20.96,29.94,151.7,1332,0.1037,0.3903,0.3639,0.1767,0.3176,0.1023 +1,15.85,23.95,103.7,782.7,0.08401,0.1002,0.09938,0.05364,0.1847,0.05338,0.4033,1.078,2.903,36.58,0.009769,0.03126,0.05051,0.01992,0.02981,0.003002,16.84,27.66,112,876.5,0.1131,0.1924,0.2322,0.1119,0.2809,0.06287 +1,13.73,22.61,93.6,578.3,0.1131,0.2293,0.2128,0.08025,0.2069,0.07682,0.2121,1.169,2.061,19.21,0.006429,0.05936,0.05501,0.01628,0.01961,0.008093,15.03,32.01,108.8,697.7,0.1651,0.7725,0.6943,0.2208,0.3596,0.1431 +1,14.54,27.54,96.73,658.8,0.1139,0.1595,0.1639,0.07364,0.2303,0.07077,0.37,1.033,2.879,32.55,0.005607,0.0424,0.04741,0.0109,0.01857,0.005466,17.46,37.13,124.1,943.2,0.1678,0.6577,0.7026,0.1712,0.4218,0.1341 +1,14.68,20.13,94.74,684.5,0.09867,0.072,0.07395,0.05259,0.1586,0.05922,0.4727,1.24,3.195,45.4,0.005718,0.01162,0.01998,0.01109,0.0141,0.002085,19.07,30.88,123.4,1138,0.1464,0.1871,0.2914,0.1609,0.3029,0.08216 +1,16.13,20.68,108.1,798.8,0.117,0.2022,0.1722,0.1028,0.2164,0.07356,0.5692,1.073,3.854,54.18,0.007026,0.02501,0.03188,0.01297,0.01689,0.004142,20.96,31.48,136.8,1315,0.1789,0.4233,0.4784,0.2073,0.3706,0.1142 +1,19.81,22.15,130,1260,0.09831,0.1027,0.1479,0.09498,0.1582,0.05395,0.7582,1.017,5.865,112.4,0.006494,0.01893,0.03391,0.01521,0.01356,0.001997,27.32,30.88,186.8,2398,0.1512,0.315,0.5372,0.2388,0.2768,0.07615 +0,13.54,14.36,87.46,566.3,0.09779,0.08129,0.06664,0.04781,0.1885,0.05766,0.2699,0.7886,2.058,23.56,0.008462,0.0146,0.02387,0.01315,0.0198,0.0023,15.11,19.26,99.7,711.2,0.144,0.1773,0.239,0.1288,0.2977,0.07259 +0,13.08,15.71,85.63,520,0.1075,0.127,0.04568,0.0311,0.1967,0.06811,0.1852,0.7477,1.383,14.67,0.004097,0.01898,0.01698,0.00649,0.01678,0.002425,14.5,20.49,96.09,630.5,0.1312,0.2776,0.189,0.07283,0.3184,0.08183 +0,9.504,12.44,60.34,273.9,0.1024,0.06492,0.02956,0.02076,0.1815,0.06905,0.2773,0.9768,1.909,15.7,0.009606,0.01432,0.01985,0.01421,0.02027,0.002968,10.23,15.66,65.13,314.9,0.1324,0.1148,0.08867,0.06227,0.245,0.07773 +1,15.34,14.26,102.5,704.4,0.1073,0.2135,0.2077,0.09756,0.2521,0.07032,0.4388,0.7096,3.384,44.91,0.006789,0.05328,0.06446,0.02252,0.03672,0.004394,18.07,19.08,125.1,980.9,0.139,0.5954,0.6305,0.2393,0.4667,0.09946 +1,21.16,23.04,137.2,1404,0.09428,0.1022,0.1097,0.08632,0.1769,0.05278,0.6917,1.127,4.303,93.99,0.004728,0.01259,0.01715,0.01038,0.01083,0.001987,29.17,35.59,188,2615,0.1401,0.26,0.3155,0.2009,0.2822,0.07526 +1,16.65,21.38,110,904.6,0.1121,0.1457,0.1525,0.0917,0.1995,0.0633,0.8068,0.9017,5.455,102.6,0.006048,0.01882,0.02741,0.0113,0.01468,0.002801,26.46,31.56,177,2215,0.1805,0.3578,0.4695,0.2095,0.3613,0.09564 +1,17.14,16.4,116,912.7,0.1186,0.2276,0.2229,0.1401,0.304,0.07413,1.046,0.976,7.276,111.4,0.008029,0.03799,0.03732,0.02397,0.02308,0.007444,22.25,21.4,152.4,1461,0.1545,0.3949,0.3853,0.255,0.4066,0.1059 +1,14.58,21.53,97.41,644.8,0.1054,0.1868,0.1425,0.08783,0.2252,0.06924,0.2545,0.9832,2.11,21.05,0.004452,0.03055,0.02681,0.01352,0.01454,0.003711,17.62,33.21,122.4,896.9,0.1525,0.6643,0.5539,0.2701,0.4264,0.1275 +1,18.61,20.25,122.1,1094,0.0944,0.1066,0.149,0.07731,0.1697,0.05699,0.8529,1.849,5.632,93.54,0.01075,0.02722,0.05081,0.01911,0.02293,0.004217,21.31,27.26,139.9,1403,0.1338,0.2117,0.3446,0.149,0.2341,0.07421 +1,15.3,25.27,102.4,732.4,0.1082,0.1697,0.1683,0.08751,0.1926,0.0654,0.439,1.012,3.498,43.5,0.005233,0.03057,0.03576,0.01083,0.01768,0.002967,20.27,36.71,149.3,1269,0.1641,0.611,0.6335,0.2024,0.4027,0.09876 +1,17.57,15.05,115,955.1,0.09847,0.1157,0.09875,0.07953,0.1739,0.06149,0.6003,0.8225,4.655,61.1,0.005627,0.03033,0.03407,0.01354,0.01925,0.003742,20.01,19.52,134.9,1227,0.1255,0.2812,0.2489,0.1456,0.2756,0.07919 +1,18.63,25.11,124.8,1088,0.1064,0.1887,0.2319,0.1244,0.2183,0.06197,0.8307,1.466,5.574,105,0.006248,0.03374,0.05196,0.01158,0.02007,0.00456,23.15,34.01,160.5,1670,0.1491,0.4257,0.6133,0.1848,0.3444,0.09782 +1,11.84,18.7,77.93,440.6,0.1109,0.1516,0.1218,0.05182,0.2301,0.07799,0.4825,1.03,3.475,41,0.005551,0.03414,0.04205,0.01044,0.02273,0.005667,16.82,28.12,119.4,888.7,0.1637,0.5775,0.6956,0.1546,0.4761,0.1402 +1,17.02,23.98,112.8,899.3,0.1197,0.1496,0.2417,0.1203,0.2248,0.06382,0.6009,1.398,3.999,67.78,0.008268,0.03082,0.05042,0.01112,0.02102,0.003854,20.88,32.09,136.1,1344,0.1634,0.3559,0.5588,0.1847,0.353,0.08482 +1,19.27,26.47,127.9,1162,0.09401,0.1719,0.1657,0.07593,0.1853,0.06261,0.5558,0.6062,3.528,68.17,0.005015,0.03318,0.03497,0.009643,0.01543,0.003896,24.15,30.9,161.4,1813,0.1509,0.659,0.6091,0.1785,0.3672,0.1123 +1,16.13,17.88,107,807.2,0.104,0.1559,0.1354,0.07752,0.1998,0.06515,0.334,0.6857,2.183,35.03,0.004185,0.02868,0.02664,0.009067,0.01703,0.003817,20.21,27.26,132.7,1261,0.1446,0.5804,0.5274,0.1864,0.427,0.1233 +1,16.74,21.59,110.1,869.5,0.0961,0.1336,0.1348,0.06018,0.1896,0.05656,0.4615,0.9197,3.008,45.19,0.005776,0.02499,0.03695,0.01195,0.02789,0.002665,20.01,29.02,133.5,1229,0.1563,0.3835,0.5409,0.1813,0.4863,0.08633 +1,14.25,21.72,93.63,633,0.09823,0.1098,0.1319,0.05598,0.1885,0.06125,0.286,1.019,2.657,24.91,0.005878,0.02995,0.04815,0.01161,0.02028,0.004022,15.89,30.36,116.2,799.6,0.1446,0.4238,0.5186,0.1447,0.3591,0.1014 +0,13.03,18.42,82.61,523.8,0.08983,0.03766,0.02562,0.02923,0.1467,0.05863,0.1839,2.342,1.17,14.16,0.004352,0.004899,0.01343,0.01164,0.02671,0.001777,13.3,22.81,84.46,545.9,0.09701,0.04619,0.04833,0.05013,0.1987,0.06169 +1,14.99,25.2,95.54,698.8,0.09387,0.05131,0.02398,0.02899,0.1565,0.05504,1.214,2.188,8.077,106,0.006883,0.01094,0.01818,0.01917,0.007882,0.001754,14.99,25.2,95.54,698.8,0.09387,0.05131,0.02398,0.02899,0.1565,0.05504 +1,13.48,20.82,88.4,559.2,0.1016,0.1255,0.1063,0.05439,0.172,0.06419,0.213,0.5914,1.545,18.52,0.005367,0.02239,0.03049,0.01262,0.01377,0.003187,15.53,26.02,107.3,740.4,0.161,0.4225,0.503,0.2258,0.2807,0.1071 +1,13.44,21.58,86.18,563,0.08162,0.06031,0.0311,0.02031,0.1784,0.05587,0.2385,0.8265,1.572,20.53,0.00328,0.01102,0.0139,0.006881,0.0138,0.001286,15.93,30.25,102.5,787.9,0.1094,0.2043,0.2085,0.1112,0.2994,0.07146 +1,10.95,21.35,71.9,371.1,0.1227,0.1218,0.1044,0.05669,0.1895,0.0687,0.2366,1.428,1.822,16.97,0.008064,0.01764,0.02595,0.01037,0.01357,0.00304,12.84,35.34,87.22,514,0.1909,0.2698,0.4023,0.1424,0.2964,0.09606 +1,19.07,24.81,128.3,1104,0.09081,0.219,0.2107,0.09961,0.231,0.06343,0.9811,1.666,8.83,104.9,0.006548,0.1006,0.09723,0.02638,0.05333,0.007646,24.09,33.17,177.4,1651,0.1247,0.7444,0.7242,0.2493,0.467,0.1038 +1,13.28,20.28,87.32,545.2,0.1041,0.1436,0.09847,0.06158,0.1974,0.06782,0.3704,0.8249,2.427,31.33,0.005072,0.02147,0.02185,0.00956,0.01719,0.003317,17.38,28,113.1,907.2,0.153,0.3724,0.3664,0.1492,0.3739,0.1027 +1,13.17,21.81,85.42,531.5,0.09714,0.1047,0.08259,0.05252,0.1746,0.06177,0.1938,0.6123,1.334,14.49,0.00335,0.01384,0.01452,0.006853,0.01113,0.00172,16.23,29.89,105.5,740.7,0.1503,0.3904,0.3728,0.1607,0.3693,0.09618 +1,18.65,17.6,123.7,1076,0.1099,0.1686,0.1974,0.1009,0.1907,0.06049,0.6289,0.6633,4.293,71.56,0.006294,0.03994,0.05554,0.01695,0.02428,0.003535,22.82,21.32,150.6,1567,0.1679,0.509,0.7345,0.2378,0.3799,0.09185 +0,8.196,16.84,51.71,201.9,0.086,0.05943,0.01588,0.005917,0.1769,0.06503,0.1563,0.9567,1.094,8.205,0.008968,0.01646,0.01588,0.005917,0.02574,0.002582,8.964,21.96,57.26,242.2,0.1297,0.1357,0.0688,0.02564,0.3105,0.07409 +1,13.17,18.66,85.98,534.6,0.1158,0.1231,0.1226,0.0734,0.2128,0.06777,0.2871,0.8937,1.897,24.25,0.006532,0.02336,0.02905,0.01215,0.01743,0.003643,15.67,27.95,102.8,759.4,0.1786,0.4166,0.5006,0.2088,0.39,0.1179 +0,12.05,14.63,78.04,449.3,0.1031,0.09092,0.06592,0.02749,0.1675,0.06043,0.2636,0.7294,1.848,19.87,0.005488,0.01427,0.02322,0.00566,0.01428,0.002422,13.76,20.7,89.88,582.6,0.1494,0.2156,0.305,0.06548,0.2747,0.08301 +0,13.49,22.3,86.91,561,0.08752,0.07698,0.04751,0.03384,0.1809,0.05718,0.2338,1.353,1.735,20.2,0.004455,0.01382,0.02095,0.01184,0.01641,0.001956,15.15,31.82,99,698.8,0.1162,0.1711,0.2282,0.1282,0.2871,0.06917 +0,11.76,21.6,74.72,427.9,0.08637,0.04966,0.01657,0.01115,0.1495,0.05888,0.4062,1.21,2.635,28.47,0.005857,0.009758,0.01168,0.007445,0.02406,0.001769,12.98,25.72,82.98,516.5,0.1085,0.08615,0.05523,0.03715,0.2433,0.06563 +0,13.64,16.34,87.21,571.8,0.07685,0.06059,0.01857,0.01723,0.1353,0.05953,0.1872,0.9234,1.449,14.55,0.004477,0.01177,0.01079,0.007956,0.01325,0.002551,14.67,23.19,96.08,656.7,0.1089,0.1582,0.105,0.08586,0.2346,0.08025 +0,11.94,18.24,75.71,437.6,0.08261,0.04751,0.01972,0.01349,0.1868,0.0611,0.2273,0.6329,1.52,17.47,0.00721,0.00838,0.01311,0.008,0.01996,0.002635,13.1,21.33,83.67,527.2,0.1144,0.08906,0.09203,0.06296,0.2785,0.07408 +1,18.22,18.7,120.3,1033,0.1148,0.1485,0.1772,0.106,0.2092,0.0631,0.8337,1.593,4.877,98.81,0.003899,0.02961,0.02817,0.009222,0.02674,0.005126,20.6,24.13,135.1,1321,0.128,0.2297,0.2623,0.1325,0.3021,0.07987 +1,15.1,22.02,97.26,712.8,0.09056,0.07081,0.05253,0.03334,0.1616,0.05684,0.3105,0.8339,2.097,29.91,0.004675,0.0103,0.01603,0.009222,0.01095,0.001629,18.1,31.69,117.7,1030,0.1389,0.2057,0.2712,0.153,0.2675,0.07873 +0,11.52,18.75,73.34,409,0.09524,0.05473,0.03036,0.02278,0.192,0.05907,0.3249,0.9591,2.183,23.47,0.008328,0.008722,0.01349,0.00867,0.03218,0.002386,12.84,22.47,81.81,506.2,0.1249,0.0872,0.09076,0.06316,0.3306,0.07036 +1,19.21,18.57,125.5,1152,0.1053,0.1267,0.1323,0.08994,0.1917,0.05961,0.7275,1.193,4.837,102.5,0.006458,0.02306,0.02945,0.01538,0.01852,0.002608,26.14,28.14,170.1,2145,0.1624,0.3511,0.3879,0.2091,0.3537,0.08294 +1,14.71,21.59,95.55,656.9,0.1137,0.1365,0.1293,0.08123,0.2027,0.06758,0.4226,1.15,2.735,40.09,0.003659,0.02855,0.02572,0.01272,0.01817,0.004108,17.87,30.7,115.7,985.5,0.1368,0.429,0.3587,0.1834,0.3698,0.1094 +0,13.05,19.31,82.61,527.2,0.0806,0.03789,0.000692,0.004167,0.1819,0.05501,0.404,1.214,2.595,32.96,0.007491,0.008593,0.000692,0.004167,0.0219,0.00299,14.23,22.25,90.24,624.1,0.1021,0.06191,0.001845,0.01111,0.2439,0.06289 +0,8.618,11.79,54.34,224.5,0.09752,0.05272,0.02061,0.007799,0.1683,0.07187,0.1559,0.5796,1.046,8.322,0.01011,0.01055,0.01981,0.005742,0.0209,0.002788,9.507,15.4,59.9,274.9,0.1733,0.1239,0.1168,0.04419,0.322,0.09026 +0,10.17,14.88,64.55,311.9,0.1134,0.08061,0.01084,0.0129,0.2743,0.0696,0.5158,1.441,3.312,34.62,0.007514,0.01099,0.007665,0.008193,0.04183,0.005953,11.02,17.45,69.86,368.6,0.1275,0.09866,0.02168,0.02579,0.3557,0.0802 +0,8.598,20.98,54.66,221.8,0.1243,0.08963,0.03,0.009259,0.1828,0.06757,0.3582,2.067,2.493,18.39,0.01193,0.03162,0.03,0.009259,0.03357,0.003048,9.565,27.04,62.06,273.9,0.1639,0.1698,0.09001,0.02778,0.2972,0.07712 +1,14.25,22.15,96.42,645.7,0.1049,0.2008,0.2135,0.08653,0.1949,0.07292,0.7036,1.268,5.373,60.78,0.009407,0.07056,0.06899,0.01848,0.017,0.006113,17.67,29.51,119.1,959.5,0.164,0.6247,0.6922,0.1785,0.2844,0.1132 +0,9.173,13.86,59.2,260.9,0.07721,0.08751,0.05988,0.0218,0.2341,0.06963,0.4098,2.265,2.608,23.52,0.008738,0.03938,0.04312,0.0156,0.04192,0.005822,10.01,19.23,65.59,310.1,0.09836,0.1678,0.1397,0.05087,0.3282,0.0849 +1,12.68,23.84,82.69,499,0.1122,0.1262,0.1128,0.06873,0.1905,0.0659,0.4255,1.178,2.927,36.46,0.007781,0.02648,0.02973,0.0129,0.01635,0.003601,17.09,33.47,111.8,888.3,0.1851,0.4061,0.4024,0.1716,0.3383,0.1031 +1,14.78,23.94,97.4,668.3,0.1172,0.1479,0.1267,0.09029,0.1953,0.06654,0.3577,1.281,2.45,35.24,0.006703,0.0231,0.02315,0.01184,0.019,0.003224,17.31,33.39,114.6,925.1,0.1648,0.3416,0.3024,0.1614,0.3321,0.08911 +0,9.465,21.01,60.11,269.4,0.1044,0.07773,0.02172,0.01504,0.1717,0.06899,0.2351,2.011,1.66,14.2,0.01052,0.01755,0.01714,0.009333,0.02279,0.004237,10.41,31.56,67.03,330.7,0.1548,0.1664,0.09412,0.06517,0.2878,0.09211 +0,11.31,19.04,71.8,394.1,0.08139,0.04701,0.03709,0.0223,0.1516,0.05667,0.2727,0.9429,1.831,18.15,0.009282,0.009216,0.02063,0.008965,0.02183,0.002146,12.33,23.84,78,466.7,0.129,0.09148,0.1444,0.06961,0.24,0.06641 +0,9.029,17.33,58.79,250.5,0.1066,0.1413,0.313,0.04375,0.2111,0.08046,0.3274,1.194,1.885,17.67,0.009549,0.08606,0.3038,0.03322,0.04197,0.009559,10.31,22.65,65.5,324.7,0.1482,0.4365,1.252,0.175,0.4228,0.1175 +0,12.78,16.49,81.37,502.5,0.09831,0.05234,0.03653,0.02864,0.159,0.05653,0.2368,0.8732,1.471,18.33,0.007962,0.005612,0.01585,0.008662,0.02254,0.001906,13.46,19.76,85.67,554.9,0.1296,0.07061,0.1039,0.05882,0.2383,0.0641 +1,18.94,21.31,123.6,1130,0.09009,0.1029,0.108,0.07951,0.1582,0.05461,0.7888,0.7975,5.486,96.05,0.004444,0.01652,0.02269,0.0137,0.01386,0.001698,24.86,26.58,165.9,1866,0.1193,0.2336,0.2687,0.1789,0.2551,0.06589 +0,8.888,14.64,58.79,244,0.09783,0.1531,0.08606,0.02872,0.1902,0.0898,0.5262,0.8522,3.168,25.44,0.01721,0.09368,0.05671,0.01766,0.02541,0.02193,9.733,15.67,62.56,284.4,0.1207,0.2436,0.1434,0.04786,0.2254,0.1084 +1,17.2,24.52,114.2,929.4,0.1071,0.183,0.1692,0.07944,0.1927,0.06487,0.5907,1.041,3.705,69.47,0.00582,0.05616,0.04252,0.01127,0.01527,0.006299,23.32,33.82,151.6,1681,0.1585,0.7394,0.6566,0.1899,0.3313,0.1339 +1,13.8,15.79,90.43,584.1,0.1007,0.128,0.07789,0.05069,0.1662,0.06566,0.2787,0.6205,1.957,23.35,0.004717,0.02065,0.01759,0.009206,0.0122,0.00313,16.57,20.86,110.3,812.4,0.1411,0.3542,0.2779,0.1383,0.2589,0.103 +0,12.31,16.52,79.19,470.9,0.09172,0.06829,0.03372,0.02272,0.172,0.05914,0.2505,1.025,1.74,19.68,0.004854,0.01819,0.01826,0.007965,0.01386,0.002304,14.11,23.21,89.71,611.1,0.1176,0.1843,0.1703,0.0866,0.2618,0.07609 +1,16.07,19.65,104.1,817.7,0.09168,0.08424,0.09769,0.06638,0.1798,0.05391,0.7474,1.016,5.029,79.25,0.01082,0.02203,0.035,0.01809,0.0155,0.001948,19.77,24.56,128.8,1223,0.15,0.2045,0.2829,0.152,0.265,0.06387 +0,13.53,10.94,87.91,559.2,0.1291,0.1047,0.06877,0.06556,0.2403,0.06641,0.4101,1.014,2.652,32.65,0.0134,0.02839,0.01162,0.008239,0.02572,0.006164,14.08,12.49,91.36,605.5,0.1451,0.1379,0.08539,0.07407,0.271,0.07191 +1,18.05,16.15,120.2,1006,0.1065,0.2146,0.1684,0.108,0.2152,0.06673,0.9806,0.5505,6.311,134.8,0.00794,0.05839,0.04658,0.0207,0.02591,0.007054,22.39,18.91,150.1,1610,0.1478,0.5634,0.3786,0.2102,0.3751,0.1108 +1,20.18,23.97,143.7,1245,0.1286,0.3454,0.3754,0.1604,0.2906,0.08142,0.9317,1.885,8.649,116.4,0.01038,0.06835,0.1091,0.02593,0.07895,0.005987,23.37,31.72,170.3,1623,0.1639,0.6164,0.7681,0.2508,0.544,0.09964 +0,12.86,18,83.19,506.3,0.09934,0.09546,0.03889,0.02315,0.1718,0.05997,0.2655,1.095,1.778,20.35,0.005293,0.01661,0.02071,0.008179,0.01748,0.002848,14.24,24.82,91.88,622.1,0.1289,0.2141,0.1731,0.07926,0.2779,0.07918 +0,11.45,20.97,73.81,401.5,0.1102,0.09362,0.04591,0.02233,0.1842,0.07005,0.3251,2.174,2.077,24.62,0.01037,0.01706,0.02586,0.007506,0.01816,0.003976,13.11,32.16,84.53,525.1,0.1557,0.1676,0.1755,0.06127,0.2762,0.08851 +0,13.34,15.86,86.49,520,0.1078,0.1535,0.1169,0.06987,0.1942,0.06902,0.286,1.016,1.535,12.96,0.006794,0.03575,0.0398,0.01383,0.02134,0.004603,15.53,23.19,96.66,614.9,0.1536,0.4791,0.4858,0.1708,0.3527,0.1016 +1,25.22,24.91,171.5,1878,0.1063,0.2665,0.3339,0.1845,0.1829,0.06782,0.8973,1.474,7.382,120,0.008166,0.05693,0.0573,0.0203,0.01065,0.005893,30,33.62,211.7,2562,0.1573,0.6076,0.6476,0.2867,0.2355,0.1051 +1,19.1,26.29,129.1,1132,0.1215,0.1791,0.1937,0.1469,0.1634,0.07224,0.519,2.91,5.801,67.1,0.007545,0.0605,0.02134,0.01843,0.03056,0.01039,20.33,32.72,141.3,1298,0.1392,0.2817,0.2432,0.1841,0.2311,0.09203 +0,12,15.65,76.95,443.3,0.09723,0.07165,0.04151,0.01863,0.2079,0.05968,0.2271,1.255,1.441,16.16,0.005969,0.01812,0.02007,0.007027,0.01972,0.002607,13.67,24.9,87.78,567.9,0.1377,0.2003,0.2267,0.07632,0.3379,0.07924 +1,18.46,18.52,121.1,1075,0.09874,0.1053,0.1335,0.08795,0.2132,0.06022,0.6997,1.475,4.782,80.6,0.006471,0.01649,0.02806,0.0142,0.0237,0.003755,22.93,27.68,152.2,1603,0.1398,0.2089,0.3157,0.1642,0.3695,0.08579 +1,14.48,21.46,94.25,648.2,0.09444,0.09947,0.1204,0.04938,0.2075,0.05636,0.4204,2.22,3.301,38.87,0.009369,0.02983,0.05371,0.01761,0.02418,0.003249,16.21,29.25,108.4,808.9,0.1306,0.1976,0.3349,0.1225,0.302,0.06846 +1,19.02,24.59,122,1076,0.09029,0.1206,0.1468,0.08271,0.1953,0.05629,0.5495,0.6636,3.055,57.65,0.003872,0.01842,0.0371,0.012,0.01964,0.003337,24.56,30.41,152.9,1623,0.1249,0.3206,0.5755,0.1956,0.3956,0.09288 +0,12.36,21.8,79.78,466.1,0.08772,0.09445,0.06015,0.03745,0.193,0.06404,0.2978,1.502,2.203,20.95,0.007112,0.02493,0.02703,0.01293,0.01958,0.004463,13.83,30.5,91.46,574.7,0.1304,0.2463,0.2434,0.1205,0.2972,0.09261 +0,14.64,15.24,95.77,651.9,0.1132,0.1339,0.09966,0.07064,0.2116,0.06346,0.5115,0.7372,3.814,42.76,0.005508,0.04412,0.04436,0.01623,0.02427,0.004841,16.34,18.24,109.4,803.6,0.1277,0.3089,0.2604,0.1397,0.3151,0.08473 +0,14.62,24.02,94.57,662.7,0.08974,0.08606,0.03102,0.02957,0.1685,0.05866,0.3721,1.111,2.279,33.76,0.004868,0.01818,0.01121,0.008606,0.02085,0.002893,16.11,29.11,102.9,803.7,0.1115,0.1766,0.09189,0.06946,0.2522,0.07246 +1,15.37,22.76,100.2,728.2,0.092,0.1036,0.1122,0.07483,0.1717,0.06097,0.3129,0.8413,2.075,29.44,0.009882,0.02444,0.04531,0.01763,0.02471,0.002142,16.43,25.84,107.5,830.9,0.1257,0.1997,0.2846,0.1476,0.2556,0.06828 +0,13.27,14.76,84.74,551.7,0.07355,0.05055,0.03261,0.02648,0.1386,0.05318,0.4057,1.153,2.701,36.35,0.004481,0.01038,0.01358,0.01082,0.01069,0.001435,16.36,22.35,104.5,830.6,0.1006,0.1238,0.135,0.1001,0.2027,0.06206 +0,13.45,18.3,86.6,555.1,0.1022,0.08165,0.03974,0.0278,0.1638,0.0571,0.295,1.373,2.099,25.22,0.005884,0.01491,0.01872,0.009366,0.01884,0.001817,15.1,25.94,97.59,699.4,0.1339,0.1751,0.1381,0.07911,0.2678,0.06603 +1,15.06,19.83,100.3,705.6,0.1039,0.1553,0.17,0.08815,0.1855,0.06284,0.4768,0.9644,3.706,47.14,0.00925,0.03715,0.04867,0.01851,0.01498,0.00352,18.23,24.23,123.5,1025,0.1551,0.4203,0.5203,0.2115,0.2834,0.08234 +1,20.26,23.03,132.4,1264,0.09078,0.1313,0.1465,0.08683,0.2095,0.05649,0.7576,1.509,4.554,87.87,0.006016,0.03482,0.04232,0.01269,0.02657,0.004411,24.22,31.59,156.1,1750,0.119,0.3539,0.4098,0.1573,0.3689,0.08368 +0,12.18,17.84,77.79,451.1,0.1045,0.07057,0.0249,0.02941,0.19,0.06635,0.3661,1.511,2.41,24.44,0.005433,0.01179,0.01131,0.01519,0.0222,0.003408,12.83,20.92,82.14,495.2,0.114,0.09358,0.0498,0.05882,0.2227,0.07376 +0,9.787,19.94,62.11,294.5,0.1024,0.05301,0.006829,0.007937,0.135,0.0689,0.335,2.043,2.132,20.05,0.01113,0.01463,0.005308,0.00525,0.01801,0.005667,10.92,26.29,68.81,366.1,0.1316,0.09473,0.02049,0.02381,0.1934,0.08988 +0,11.6,12.84,74.34,412.6,0.08983,0.07525,0.04196,0.0335,0.162,0.06582,0.2315,0.5391,1.475,15.75,0.006153,0.0133,0.01693,0.006884,0.01651,0.002551,13.06,17.16,82.96,512.5,0.1431,0.1851,0.1922,0.08449,0.2772,0.08756 +1,14.42,19.77,94.48,642.5,0.09752,0.1141,0.09388,0.05839,0.1879,0.0639,0.2895,1.851,2.376,26.85,0.008005,0.02895,0.03321,0.01424,0.01462,0.004452,16.33,30.86,109.5,826.4,0.1431,0.3026,0.3194,0.1565,0.2718,0.09353 \ No newline at end of file diff --git a/tests.py b/tests.py index 74a5f2e5..2c85f385 100644 --- a/tests.py +++ b/tests.py @@ -19,6 +19,7 @@ import inspect import random from datetime import datetime +import subprocess from sklearn.datasets import load_digits, load_boston from sklearn.model_selection import train_test_split @@ -42,6 +43,15 @@ test_operator_key = 'sklearn.feature_selection.SelectKBest' TPOTSelectKBest,TPOTSelectKBest_args = TPOTOperatorClassFactory(test_operator_key, classifier_config_dict[test_operator_key]) +def test_driver(): + """Assert that the TPOT driver output normal result""" + batcmd = "python -m tpot.driver tests.csv -is , -target class -g 2 -p 2 -os 4 -cv 5 -s 45 -v 1" + ret_stdout = subprocess.check_output(batcmd, shell=True) + try: + ret_val = float(ret_stdout.decode("utf-8").split('\n')[-2].split(': ')[-1]) + except: + ret_val = -float('inf') + assert ret_val > 0.0 def test_init_custom_parameters(): From 75539d52f3383b62354b74b5d31c25c3a72860b5 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 1 Mar 2017 17:00:00 -0500 Subject: [PATCH 115/154] clean codes --- tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests.py b/tests.py index 2c85f385..5ab3e01f 100644 --- a/tests.py +++ b/tests.py @@ -43,6 +43,7 @@ test_operator_key = 'sklearn.feature_selection.SelectKBest' TPOTSelectKBest,TPOTSelectKBest_args = TPOTOperatorClassFactory(test_operator_key, classifier_config_dict[test_operator_key]) + def test_driver(): """Assert that the TPOT driver output normal result""" batcmd = "python -m tpot.driver tests.csv -is , -target class -g 2 -p 2 -os 4 -cv 5 -s 45 -v 1" From b3e611bda8b716852698abdf84329cb0ccd9aeb2 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Sat, 4 Mar 2017 19:47:26 -0500 Subject: [PATCH 116/154] fix bug --- tpot/gp_deap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 9dde3eca..7f363fcf 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -57,7 +57,7 @@ def varOr(population, toolbox, lambda_, cxpb, mutpb): op_choice = np.random.random() if op_choice < cxpb: # Apply crossover idxs = np.random.randint(0, len(population),size=2) - ind1, ind2 = population[idxs[0]],population[idxs[1]] + ind1, ind2 = toolbox.clone(population[idxs[0]]), toolbox.clone(population[idxs[1]]) ind_str = str(ind1) ind1, ind2 = toolbox.mate(ind1, ind2) if ind_str != str(ind1): # check if crossover generated a new pipeline @@ -65,7 +65,7 @@ def varOr(population, toolbox, lambda_, cxpb, mutpb): offspring.append(ind1) elif op_choice < cxpb + mutpb: # Apply mutation idx = np.random.randint(0, len(population)) - ind = population[idx] + ind = toolbox.clone(population[idx]) ind_str = str(ind) ind, = toolbox.mutate(ind) if ind_str != str(ind): # check if mutation happend @@ -73,7 +73,7 @@ def varOr(population, toolbox, lambda_, cxpb, mutpb): offspring.append(ind) else: # Apply reproduction idx = np.random.randint(0, len(population)) - offspring.append(population[idx]) + offspring.append(toolbox.clone(population[idx])) return offspring @@ -151,7 +151,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar, pbar.update(len(offspring)-len(invalid_ind)) if not (max_time_mins is None) and pbar.n >= pbar.total: pbar.total += lambda_ - + fitnesses = toolbox.evaluate(invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit From a458c5d6d2ff375f466cb684d58c094f6adad9d3 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 9 Mar 2017 16:01:24 -0500 Subject: [PATCH 117/154] check mut and co --- tpot/base.py | 5 +---- tpot/gp_deap.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index cef52289..e721a7ff 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -806,15 +806,12 @@ def _random_mutation_operator(self, individual): """ # debug usage #print(str(individual)) - old_ind = str(individual) - mut_ind = (str(individual),) mutation_techniques = [ partial(gp.mutInsert, pset=self._pset), partial(mutNodeReplacement, pset=self._pset), partial(gp.mutShrink) ] - mut_ind = np.random.choice(mutation_techniques)(individual) - return mut_ind + return np.random.choice(mutation_techniques)(individual) def _gen_grow_safe(self, pset, min_, max_, type_=None): diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py index 7f363fcf..396cb3d6 100644 --- a/tpot/gp_deap.py +++ b/tpot/gp_deap.py @@ -59,16 +59,22 @@ def varOr(population, toolbox, lambda_, cxpb, mutpb): idxs = np.random.randint(0, len(population),size=2) ind1, ind2 = toolbox.clone(population[idxs[0]]), toolbox.clone(population[idxs[1]]) ind_str = str(ind1) - ind1, ind2 = toolbox.mate(ind1, ind2) - if ind_str != str(ind1): # check if crossover generated a new pipeline + num_loop = 0 + while ind_str == str(ind1) and num_loop < 50 : # 50 loops at most to generate a different individual by crossover + ind1, ind2 = toolbox.mate(ind1, ind2) + num_loop += 1 + if ind_str != str(ind1): # check if crossover happened del ind1.fitness.values offspring.append(ind1) elif op_choice < cxpb + mutpb: # Apply mutation idx = np.random.randint(0, len(population)) ind = toolbox.clone(population[idx]) ind_str = str(ind) - ind, = toolbox.mutate(ind) - if ind_str != str(ind): # check if mutation happend + num_loop = 0 + while ind_str == str(ind) and num_loop < 50 : # 50 loops at most to generate a different individual by mutation + ind, = toolbox.mutate(ind) + num_loop += 1 + if ind_str != str(ind): # check if mutation happened del ind.fitness.values offspring.append(ind) else: # Apply reproduction From 0407b1db1776e5b265e227be0c62fb5fdf526ff5 Mon Sep 17 00:00:00 2001 From: Michal Ficek Date: Fri, 10 Mar 2017 13:02:00 +0100 Subject: [PATCH 118/154] =?UTF-8?q?Decorate=20class=20functions,=20create?= =?UTF-8?q?=20=E2=80=9Cmate=E2=80=9D=20operator=20wrapper.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tpot/base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index cef52289..2795c527 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -320,9 +320,9 @@ def _setup_toolbox(self): self._toolbox.register('population', tools.initRepeat, list, self._toolbox.individual) self._toolbox.register('compile', self._compile_to_sklearn) self._toolbox.register('select', tools.selNSGA2) - self._toolbox.register('mate', _pre_test(gp.cxOnePoint)) + self._toolbox.register('mate', self._mate_operator) self._toolbox.register('expr_mut', self._gen_grow_safe, min_=1, max_=4) - self._toolbox.register('mutate', _pre_test(self._random_mutation_operator)) + self._toolbox.register('mutate', self._random_mutation_operator) def fit(self, features, classes, sample_weight=None): """Fits a machine learning pipeline that maximizes classification score @@ -788,7 +788,11 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe fitnesses_ordered.append(fitnesses_dict[key]) return fitnesses_ordered + @_pre_test + def _mate_operator(self, ind1, ind2): + return gp.cxOnePoint(ind1, ind2) + @_pre_test def _random_mutation_operator(self, individual): """Perform a replacement, insert, or shrink mutation on an individual From 5512284548634f22f4ea1bef4ab38a15f4831b71 Mon Sep 17 00:00:00 2001 From: Michal Ficek Date: Fri, 10 Mar 2017 13:06:47 +0100 Subject: [PATCH 119/154] FIX: Ensure that pipeline gets evaluated when function result is not an individual but a tuple. --- tpot/decorators.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tpot/decorators.py b/tpot/decorators.py index cefb8c86..acfe943f 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -145,13 +145,16 @@ def check_pipeline(self, *args, **kwargs): with warnings.catch_warnings(): warnings.simplefilter('ignore') expr = func(self, *args, **kwargs) - #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) # debug - sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(expr), self.operators), self.operators_context) - if self.classification: - sklearn_pipeline.fit(pretest_X, pretest_y) - else: - sklearn_pipeline.fit(pretest_X_reg, pretest_y_reg) - bad_pipeline = False + # mutation operator returns tuple (ind,); crossover operator returns tuple (ind1, ind2) + expr_tuple = expr if isinstance(expr, tuple) else (expr,) + for expr_test in expr_tuple: + #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) # debug + sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(expr_test), self.operators), self.operators_context) + if self.classification: + sklearn_pipeline.fit(pretest_X, pretest_y) + else: + sklearn_pipeline.fit(pretest_X_reg, pretest_y_reg) + bad_pipeline = False except: pass finally: From dd0fc8b3bd87bb38bfb71a57144d0bf4377e0d5a Mon Sep 17 00:00:00 2001 From: Michal Ficek Date: Fri, 10 Mar 2017 13:08:04 +0100 Subject: [PATCH 120/154] ADD: Debug printout to know if pipeline is wrong or something else is happening inside the _pre_test decorator. --- tpot/decorators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tpot/decorators.py b/tpot/decorators.py index acfe943f..81f261df 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -155,7 +155,9 @@ def check_pipeline(self, *args, **kwargs): else: sklearn_pipeline.fit(pretest_X_reg, pretest_y_reg) bad_pipeline = False - except: + except BaseException as e: + if self.verbosity == 3: + print('_pre_test decorator: {fname}: num_test={n} {e}'.format(n=num_test, fname=func.__name__, e=e)) pass finally: num_test += 1 From 231d82382e1b9357469f599ee8c37b5ad70df519 Mon Sep 17 00:00:00 2001 From: Michal Ficek Date: Tue, 14 Mar 2017 09:55:51 +0100 Subject: [PATCH 121/154] FIX: Clone individuals before func evaluation. --- tpot/decorators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tpot/decorators.py b/tpot/decorators.py index 81f261df..f0731da1 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -24,6 +24,7 @@ import warnings from sklearn.datasets import make_classification, make_regression from .export_utils import expr_to_tree, generate_pipeline_code +from deap import creator # generate a small data set for a new pipeline, in order to check if the pipeline # has unsuppported combinations in params pretest_X, pretest_y = make_classification(n_samples=50, n_features=10, random_state=42) @@ -141,6 +142,8 @@ def check_pipeline(self, *args, **kwargs): bad_pipeline = True num_test = 0 # number of tests while bad_pipeline and num_test < 10: # a pool for workable pipeline + # clone individual before each func call so it is not altered for the possible next cycle loop + args = [self._toolbox.clone(arg) if isinstance(arg, creator.Individual) else arg for arg in args] try: with warnings.catch_warnings(): warnings.simplefilter('ignore') From b0cb35ca726b07947f7498155f8f80b0c69bcbf6 Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Thu, 16 Mar 2017 15:49:51 +0100 Subject: [PATCH 122/154] One example of how to unify pre_test for generate, mutate and mate. --- tpot/decorators.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tpot/decorators.py b/tpot/decorators.py index cefb8c86..cfa7cd89 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -145,16 +145,25 @@ def check_pipeline(self, *args, **kwargs): with warnings.catch_warnings(): warnings.simplefilter('ignore') expr = func(self, *args, **kwargs) - #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) # debug - sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(expr), self.operators), self.operators_context) - if self.classification: - sklearn_pipeline.fit(pretest_X, pretest_y) + # unify output of gen, mutate and mate -> always a tuple + if type(expr) == list: + exprs = (expr,) else: - sklearn_pipeline.fit(pretest_X_reg, pretest_y_reg) + exprs = expr + + for ind in exprs: + #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) # debug + sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(ind), self.operators), self.operators_context) + if self.classification: + sklearn_pipeline.fit(pretest_X, pretest_y) + else: + sklearn_pipeline.fit(pretest_X_reg, pretest_y_reg) bad_pipeline = False - except: - pass + except Exception as e: + print(e) finally: num_test += 1 + return expr + return check_pipeline From 8abe2a40f4a7e220381870bc19d12acba73b3375 Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Thu, 16 Mar 2017 15:53:07 +0100 Subject: [PATCH 123/154] pre_test decorator for mate and mutate now not inline, but through @ syntax, making sure self gets passed. --- tpot/base.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index e721a7ff..377f4eda 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -303,7 +303,8 @@ def _setup_pset(self): # Terminals for _type in self.arguments: for val in _type.values: - self._pset.addTerminal(val, _type) + terminal_name = _type.__name__ + "=" + str(val) + self._pset.addTerminal(val, _type, name=terminal_name) if self.verbosity > 2: print('{} operators are imported.'.format(len(self.operators))) @@ -320,9 +321,9 @@ def _setup_toolbox(self): self._toolbox.register('population', tools.initRepeat, list, self._toolbox.individual) self._toolbox.register('compile', self._compile_to_sklearn) self._toolbox.register('select', tools.selNSGA2) - self._toolbox.register('mate', _pre_test(gp.cxOnePoint)) + self._toolbox.register('mate', self._gpCxOnePoint) self._toolbox.register('expr_mut', self._gen_grow_safe, min_=1, max_=4) - self._toolbox.register('mutate', _pre_test(self._random_mutation_operator)) + self._toolbox.register('mutate', self._random_mutation_operator) def fit(self, features, classes, sample_weight=None): """Fits a machine learning pipeline that maximizes classification score @@ -788,7 +789,11 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe fitnesses_ordered.append(fitnesses_dict[key]) return fitnesses_ordered + @_pre_test + def _gpCxOnePoint(self, *args): + return gp.cxOnePoint(*args) + @_pre_test def _random_mutation_operator(self, individual): """Perform a replacement, insert, or shrink mutation on an individual From 0a258e9408a03ee972005169a72ee2e4d4d19ec2 Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Thu, 16 Mar 2017 15:58:24 +0100 Subject: [PATCH 124/154] expr_to_tree now also requires pset as an argument, so symbolic links can be used (for terminals with a name different from its value) --- tpot/base.py | 4 ++-- tpot/decorators.py | 4 ++-- tpot/export_utils.py | 11 +++++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 377f4eda..028cf109 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -572,7 +572,7 @@ def export(self, output_file_name): raise ValueError('A pipeline has not yet been optimized. Please call fit() first.') with open(output_file_name, 'w') as output_file: - output_file.write(export_pipeline(self._optimized_pipeline, self.operators)) + output_file.write(export_pipeline(self._optimized_pipeline, self.operators, self._pset)) def _compile_to_sklearn(self, expr): """Compiles a DEAP pipeline into a sklearn pipeline @@ -586,7 +586,7 @@ def _compile_to_sklearn(self, expr): ------- sklearn_pipeline: sklearn.pipeline.Pipeline """ - sklearn_pipeline = generate_pipeline_code(expr_to_tree(expr), self.operators) + sklearn_pipeline = generate_pipeline_code(expr_to_tree(expr, self._pset), self.operators) return eval(sklearn_pipeline, self.operators_context) def _set_param_recursive(self, pipeline_steps, parameter, value): diff --git a/tpot/decorators.py b/tpot/decorators.py index cfa7cd89..faa18ef2 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -150,10 +150,10 @@ def check_pipeline(self, *args, **kwargs): exprs = (expr,) else: exprs = expr - + for ind in exprs: #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) # debug - sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(ind), self.operators), self.operators_context) + sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(ind, self._pset), self.operators), self.operators_context) if self.classification: sklearn_pipeline.fit(pretest_X, pretest_y) else: diff --git a/tpot/export_utils.py b/tpot/export_utils.py index bf996fb8..02708317 100644 --- a/tpot/export_utils.py +++ b/tpot/export_utils.py @@ -45,7 +45,7 @@ def get_by_name(opname, operators): ret_op_class = ret_op_classes[0] return ret_op_class -def export_pipeline(exported_pipeline, operators): +def export_pipeline(exported_pipeline, operators, pset): """Generates the source code of a TPOT Pipeline Parameters @@ -62,7 +62,7 @@ def export_pipeline(exported_pipeline, operators): """ # Unroll the nested function calls into serial code - pipeline_tree = expr_to_tree(exported_pipeline) + pipeline_tree = expr_to_tree(exported_pipeline, pset) # Have the exported code import all of the necessary modules and functions pipeline_text = generate_import_code(exported_pipeline, operators) @@ -73,7 +73,7 @@ def export_pipeline(exported_pipeline, operators): return pipeline_text -def expr_to_tree(ind): +def expr_to_tree(ind, pset): """Convert the unstructured DEAP pipeline into a tree data-structure Parameters @@ -95,7 +95,10 @@ def expr_to_tree(ind): """ def prim_to_list(prim, args): if isinstance(prim, deap.gp.Terminal): - return prim.value + if prim.name in pset.context: + return pset.context[prim.name] + else: + return prim.value return [prim.name] + args From a9383208d47eda9bdade6e6340756c24c685ce31 Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Thu, 16 Mar 2017 16:06:20 +0100 Subject: [PATCH 125/154] Allow for missing values --- tpot/base.py | 3 ++- tpot/operator_utils.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index 028cf109..1a12f77b 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -302,7 +302,8 @@ def _setup_pset(self): # Terminals for _type in self.arguments: - for val in _type.values: + type_values = list(_type.values) + ['MISSING'] + for val in type_values: terminal_name = _type.__name__ + "=" + str(val) self._pset.addTerminal(val, _type, name=terminal_name) diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 7820ae86..aa6604e5 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -194,6 +194,8 @@ def export(cls, *args): if dep_op_list: dep_op_arguments = {} for arg_class, arg_value in zip(arg_types, args): + if arg_value == "MISSING": + continue aname_split = arg_class.__name__.split('__') if isinstance(arg_value, str): arg_value = '\"{}\"'.format(arg_value) From 471ca39ad5daccb4faf24280c869153d9c049926 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 16 Mar 2017 11:34:58 -0400 Subject: [PATCH 126/154] fix export_pipeline in unit tests --- tests.py | 8 ++++---- tpot/decorators.py | 2 +- tpot_test_multi_process.py | 8 +------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tests.py b/tests.py index ed56383c..225402e4 100644 --- a/tests.py +++ b/tests.py @@ -183,7 +183,7 @@ def test_random_ind_2(): exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) """ - assert expected_code == export_pipeline(pipeline, tpot_obj.operators) + assert expected_code == export_pipeline(pipeline, tpot_obj.operators, tpot_obj._pset) def test_score(): """Assert that the TPOT score function raises a ValueError when no optimized pipeline exists""" @@ -503,7 +503,7 @@ def test_export_pipeline(): exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) """ - assert expected_code == export_pipeline(pipeline,tpot_obj.operators) + assert expected_code == export_pipeline(pipeline, tpot_obj.operators, tpot_obj._pset) def test_export_pipeline_2(): @@ -527,7 +527,7 @@ def test_export_pipeline_2(): exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) """ - assert expected_code == export_pipeline(pipeline, tpot_obj.operators) + assert expected_code == export_pipeline(pipeline, tpot_obj.operators, tpot_obj._pset) def test_export_pipeline_3(): """Assert that exported_pipeline() generated a compile source file as expected given a fixed simple pipeline with a preprocessor""" @@ -556,7 +556,7 @@ def test_export_pipeline_3(): exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) """ - assert expected_code == export_pipeline(pipeline, tpot_obj.operators) + assert expected_code == export_pipeline(pipeline, tpot_obj.operators, tpot_obj._pset) def test_operator_export(): diff --git a/tpot/decorators.py b/tpot/decorators.py index 29d860b0..3295be91 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -152,7 +152,7 @@ def check_pipeline(self, *args, **kwargs): expr_tuple = expr if isinstance(expr, tuple) else (expr,) for expr_test in expr_tuple: #print(num_test, generate_pipeline_code(expr_to_tree(expr), self.operators)) # debug - sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(expr_test), self.operators), self.operators_context) + sklearn_pipeline = eval(generate_pipeline_code(expr_to_tree(expr_test, self._pset), self.operators), self.operators_context) if self.classification: sklearn_pipeline.fit(pretest_X, pretest_y) else: diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py index 832d34c2..aa48ec2d 100644 --- a/tpot_test_multi_process.py +++ b/tpot_test_multi_process.py @@ -7,13 +7,7 @@ X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, train_size=0.25, test_size=0.75) -#tpot = TPOTClassifier(generations=3, population_size=10, verbosity=2, num_cpu=1, random_state = 42) -#time_start = time.time() -#tpot.fit(X_train, y_train) -#print(tpot.score(X_test, y_test)) -#print('\nTime used with num_cpu = 1:',time.time()-time_start) - -tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2, n_jobs = 2, random_state = 44)#, max_time_mins=1) +tpot = TPOTClassifier(generations=3, population_size=5, offspring_size=10, verbosity=3, n_jobs = 2, random_state = 44)#, max_time_mins=1) time_start = time.time() tpot.fit(X_train, y_train) print(tpot.score(X_test, y_test)) From d3432d8ce79496611715a6835b13bedd89725a81 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 16 Mar 2017 13:58:40 -0400 Subject: [PATCH 127/154] use fix pipelines instead random pipelines to test scores in unit tests --- tests.py | 109 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 37 deletions(-) diff --git a/tests.py b/tests.py index 225402e4..6c226acb 100644 --- a/tests.py +++ b/tests.py @@ -198,17 +198,17 @@ def test_score(): def test_score_2(): - """Assert that the TPOTClassifier score function outputs a known score for a ramdom pipeline""" + """Assert that the TPOTClassifier score function outputs a known score for a fix pipeline""" - tpot_obj = TPOTClassifier(random_state=43) - tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 0.96710588996037627 # Assumes use of the TPOT balanced_accuracy function + tpot_obj = TPOTClassifier() + known_score = 0.987691257357 # Assumes use of the TPOT balanced_accuracy function # Reify pipeline with known score - tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() + pipeline_string= ('KNeighborsClassifier(input_matrix, KNeighborsClassifier__n_neighbors=10, ' + 'KNeighborsClassifier__p=1,KNeighborsClassifier__weights=uniform)') + tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) - # Get score from TPOT score = tpot_obj.score(testing_features, testing_classes) @@ -219,14 +219,22 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): assert isclose(known_score, score) def test_score_3(): - """Assert that the TPOTRegressor score function outputs a known score for a random pipeline""" + """Assert that the TPOTRegressor score function outputs a known score for a fix pipeline""" - tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error', random_state=53) - tpot_obj._pbar = tqdm(total=1, disable=True) - known_score = 15.724128278216726 # Assumes use of mse + tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') + known_score = 11.2010824752 # Assumes use of mse # Reify pipeline with known score - tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() + + pipeline_string = ("ExtraTreesRegressor(" + "GradientBoostingRegressor(input_matrix, GradientBoostingRegressor__alpha=0.8," + "GradientBoostingRegressor__learning_rate=0.1,GradientBoostingRegressor__loss=huber," + "GradientBoostingRegressor__max_depth=5, GradientBoostingRegressor__max_features=0.5," + "GradientBoostingRegressor__min_samples_leaf=5, GradientBoostingRegressor__min_samples_split=5," + "GradientBoostingRegressor__subsample=0.25)," + "ExtraTreesRegressor__bootstrap=True,ExtraTreesRegressor__max_features=0.5," + "ExtraTreesRegressor__min_samples_leaf=5,ExtraTreesRegressor__min_samples_split=5)") + tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r) @@ -256,8 +264,11 @@ def test_predict(): def test_predict_2(): """Assert that the TPOT predict function returns a numpy matrix of shape (num_testing_rows,)""" - tpot_obj = TPOTClassifier(random_state=49) - tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() + tpot_obj = TPOTClassifier() + pipeline_string= ('DecisionTreeClassifier(input_matrix, DecisionTreeClassifier__criterion=gini' + ', DecisionTreeClassifier__max_depth=8,DecisionTreeClassifier__min_samples_leaf=5,' + 'DecisionTreeClassifier__min_samples_split=5)') + tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -269,8 +280,11 @@ def test_predict_2(): def test_predict_proba(): """Assert that the TPOT predict_proba function returns a numpy matrix of shape (num_testing_rows, num_testing_classes)""" - tpot_obj = TPOTClassifier(random_state=51) - tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() + tpot_obj = TPOTClassifier() + pipeline_string= ('DecisionTreeClassifier(input_matrix, DecisionTreeClassifier__criterion=gini' + ', DecisionTreeClassifier__max_depth=8,DecisionTreeClassifier__min_samples_leaf=5,' + 'DecisionTreeClassifier__min_samples_split=5)') + tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -283,8 +297,11 @@ def test_predict_proba(): def test_predict_proba2(): """Assert that the TPOT predict_proba function returns a numpy matrix filled with probabilities (float)""" - tpot_obj = TPOTClassifier(random_state=53) - tpot_obj._optimized_pipeline = tpot_obj._toolbox.individual() + tpot_obj = TPOTClassifier() + pipeline_string= ('DecisionTreeClassifier(input_matrix, DecisionTreeClassifier__criterion=gini' + ', DecisionTreeClassifier__max_depth=8,DecisionTreeClassifier__min_samples_leaf=5,' + 'DecisionTreeClassifier__min_samples_split=5)') + tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) tpot_obj._fitted_pipeline.fit(training_features, training_classes) @@ -456,8 +473,15 @@ def test_generate_import_code(): def test_mutNodeReplacement(): """Assert that mutNodeReplacement() returns the correct type of mutation node in a fixed pipeline""" - tpot_obj = TPOTClassifier(random_state=42) - pipeline = tpot_obj._toolbox.individual() + tpot_obj = TPOTClassifier() + pipeline_string= ('KNeighborsClassifier(CombineDFs(' + 'DecisionTreeClassifier(input_matrix, DecisionTreeClassifier__criterion=gini' + ', DecisionTreeClassifier__max_depth=8,DecisionTreeClassifier__min_samples_leaf=5,' + 'DecisionTreeClassifier__min_samples_split=5),SelectKBest(input_matrix, SelectKBest__k=20)' + 'KNeighborsClassifier__n_neighbors=10, ' + 'KNeighborsClassifier__p=1,KNeighborsClassifier__weights=uniform') + pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) + pipeline[0].ret = Output_DF old_ret_type_list = [node.ret for node in pipeline] old_prims_list = [node for node in pipeline if node.arity != 0] mut_ind = mutNodeReplacement(pipeline, pset = tpot_obj._pset) @@ -474,17 +498,23 @@ def test_mutNodeReplacement(): def test_export_pipeline(): """Assert that exported_pipeline() generated a compile source file as expected given a fixed complex pipeline""" tpot_obj = TPOTClassifier() - pipeline = creator.Individual.\ - from_string("GaussianNB(CombineDFs(ZeroCount(input_matrix), RobustScaler(input_matrix)))", tpot_obj._pset) + pipeline_string= ('KNeighborsClassifier(CombineDFs(' + 'DecisionTreeClassifier(input_matrix, DecisionTreeClassifier__criterion=gini' + ', DecisionTreeClassifier__max_depth=8,DecisionTreeClassifier__min_samples_leaf=5,' + 'DecisionTreeClassifier__min_samples_split=5),SelectKBest(input_matrix, SelectKBest__k=20)' + 'KNeighborsClassifier__n_neighbors=10, ' + 'KNeighborsClassifier__p=1,KNeighborsClassifier__weights=uniform') + pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) expected_code = """import numpy as np from sklearn.ensemble import VotingClassifier +from sklearn.feature_selection import SelectKBest, f_classif from sklearn.model_selection import train_test_split -from sklearn.naive_bayes import GaussianNB +from sklearn.neighbors import KNeighborsClassifier from sklearn.pipeline import make_pipeline, make_union -from sklearn.preprocessing import FunctionTransformer, RobustScaler -from tpot.build_in_operators import ZeroCount +from sklearn.preprocessing import FunctionTransformer +from sklearn.tree import DecisionTreeClassifier # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -494,10 +524,12 @@ def test_export_pipeline(): exported_pipeline = make_pipeline( make_union( - ZeroCount(), - RobustScaler() + make_union(VotingClassifier([('branch', + DecisionTreeClassifier(criterion="gini", max_depth=8, min_samples_leaf=5, min_samples_split=5) + )]), FunctionTransformer(lambda X: X)), + SelectKBest(score_func=f_classif, k=20) ), - GaussianNB() + KNeighborsClassifier(n_neighbors=10, p=1, weights="uniform") ) exported_pipeline.fit(training_features, training_classes) @@ -509,12 +541,13 @@ def test_export_pipeline(): def test_export_pipeline_2(): """Assert that exported_pipeline() generated a compile source file as expected given a fixed simple pipeline (only one classifier)""" tpot_obj = TPOTClassifier() - pipeline = creator.Individual.\ - from_string("GaussianNB(input_matrix)", tpot_obj._pset) + pipeline_string= ('KNeighborsClassifier(input_matrix, KNeighborsClassifier__n_neighbors=10, ' + 'KNeighborsClassifier__p=1,KNeighborsClassifier__weights=uniform)') + pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) expected_code = """import numpy as np from sklearn.model_selection import train_test_split -from sklearn.naive_bayes import GaussianNB +from sklearn.neighbors import KNeighborsClassifier # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -522,7 +555,7 @@ def test_export_pipeline_2(): training_features, testing_features, training_classes, testing_classes = \\ train_test_split(features, tpot_data['class'], random_state=42) -exported_pipeline = GaussianNB() +exported_pipeline = KNeighborsClassifier(n_neighbors=10, p=1, weights="uniform") exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) @@ -532,15 +565,17 @@ def test_export_pipeline_2(): def test_export_pipeline_3(): """Assert that exported_pipeline() generated a compile source file as expected given a fixed simple pipeline with a preprocessor""" tpot_obj = TPOTClassifier() - pipeline = creator.Individual.\ - from_string("GaussianNB(MaxAbsScaler(input_matrix))", tpot_obj._pset) + pipeline_string= ('DecisionTreeClassifier(SelectKBest(input_matrix, SelectKBest__k=20),' + 'DecisionTreeClassifier__criterion=gini, DecisionTreeClassifier__max_depth=8,' + 'DecisionTreeClassifier__min_samples_leaf=5, DecisionTreeClassifier__min_samples_split=5)') + pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) expected_code = """import numpy as np +from sklearn.feature_selection import SelectKBest, f_classif from sklearn.model_selection import train_test_split -from sklearn.naive_bayes import GaussianNB from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import MaxAbsScaler +from sklearn.tree import DecisionTreeClassifier # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -549,8 +584,8 @@ def test_export_pipeline_3(): train_test_split(features, tpot_data['class'], random_state=42) exported_pipeline = make_pipeline( - MaxAbsScaler(), - GaussianNB() + SelectKBest(score_func=f_classif, k=20), + DecisionTreeClassifier(criterion="gini", max_depth=8, min_samples_leaf=5, min_samples_split=5) ) exported_pipeline.fit(training_features, training_classes) From 8ce8bbb5eb730b18249041e5a93c0007580474d7 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 16 Mar 2017 14:05:41 -0400 Subject: [PATCH 128/154] clean codes --- tpot_test_multi_process.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 tpot_test_multi_process.py diff --git a/tpot_test_multi_process.py b/tpot_test_multi_process.py deleted file mode 100644 index aa48ec2d..00000000 --- a/tpot_test_multi_process.py +++ /dev/null @@ -1,14 +0,0 @@ -from tpot import TPOTClassifier -from sklearn.datasets import load_digits -from sklearn.model_selection import train_test_split -import time - -digits = load_digits() -X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, - train_size=0.25, test_size=0.75) - -tpot = TPOTClassifier(generations=3, population_size=5, offspring_size=10, verbosity=3, n_jobs = 2, random_state = 44)#, max_time_mins=1) -time_start = time.time() -tpot.fit(X_train, y_train) -print(tpot.score(X_test, y_test)) -print('\nTime used with num_cpu = 2:',time.time()-time_start) From 6b9245544ee138f1dfa04131e0d2e3e46b594192 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Thu, 16 Mar 2017 14:51:13 -0400 Subject: [PATCH 129/154] Change default TPOT scoring metric to accuracy Too many users were getting confused about balanced accuracy as the default metric. Let's use the same default metric as sklearn: accuracy. --- tpot/tpot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpot/tpot.py b/tpot/tpot.py index 384e96ca..e6496e10 100644 --- a/tpot/tpot.py +++ b/tpot/tpot.py @@ -26,7 +26,7 @@ class TPOTClassifier(TPOTBase): """TPOT estimator for classification problems""" - scoring_function = 'balanced_accuracy' # Classification scoring + scoring_function = 'accuracy' # Classification scoring default_operator_dict = classifier_config_dict # Classification dictionary classification = True regression = False From 094b8ee37396d18017b9ee6ee1de47fb2499243f Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Thu, 16 Mar 2017 15:04:34 -0400 Subject: [PATCH 130/154] fix a unit test --- tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests.py b/tests.py index 6c226acb..4e58e5ff 100644 --- a/tests.py +++ b/tests.py @@ -201,7 +201,7 @@ def test_score_2(): """Assert that the TPOTClassifier score function outputs a known score for a fix pipeline""" tpot_obj = TPOTClassifier() - known_score = 0.987691257357 # Assumes use of the TPOT balanced_accuracy function + known_score = 0.977777777778 # Assumes use of the TPOT balanced_accuracy function # Reify pipeline with known score pipeline_string= ('KNeighborsClassifier(input_matrix, KNeighborsClassifier__n_neighbors=10, ' @@ -215,7 +215,7 @@ def test_score_2(): # http://stackoverflow.com/questions/5595425/ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) - + assert isclose(known_score, score) def test_score_3(): From 409d47f454fbdaac68ceb1e190355ee6d662c72b Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 17 Mar 2017 13:25:32 -0400 Subject: [PATCH 131/154] add unit test for sample weight --- docs_sources/releases.md | 19 ++++++++++++++ docs_sources/using.md | 11 ++++++++ tests.py | 54 ++++++++++++++++++++++++++++++++++++---- tpot/base.py | 31 ++--------------------- tpot/driver.py | 4 +-- tpot/operator_utils.py | 27 ++++++++++++++++++++ 6 files changed, 110 insertions(+), 36 deletions(-) diff --git a/docs_sources/releases.md b/docs_sources/releases.md index 73006e5c..8b69e421 100644 --- a/docs_sources/releases.md +++ b/docs_sources/releases.md @@ -1,3 +1,22 @@ +# Version 0.7 + +* **TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only)** TPOT allows you to use multiple processes for pipeline optimization in TPOT with the `n_jobs` parameter in both TPOTClassifier and TPOTRegressor. The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-njobs` parameter. + +* **TPOT now support customized dictionary of operators and parameters** TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the `operator_dict` parameter. The format of this customized dictionary can be found in [online manual](/using/#tpot-with-code). The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-operator` parameter but take a file including the dictionary instead. + +* TPOT now allows you to **specify a time limit (default time limit is 5 minutes)** for evaluating a single pipeline in optimization process with the `max_eval_time_mins` parameter, so TPOT can skip these time-consuming pipelines. + +* The [evolutionary algorithm] is replaced by the (mu + lambda) evolutionary algorithm. TPOT allows you to set offspring size (lambda) for pipeline optimization in TPOT with a new `offspring_size` parameter. The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-c` parameter. + +* Fixed issue about reproducing results with same random seed + +* Default operators and their parameters in TPOT were refined. + +* The TPOT point mutation operator was refined + +* The default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn. + + # Version 0.6 * **TPOT now supports regression problems!** We have created two separate `TPOTClassifier` and `TPOTRegressor` classes to support classification and regression problems, respectively. The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-mode` parameter. diff --git a/docs_sources/using.md b/docs_sources/using.md index 9781760a..8150728b 100644 --- a/docs_sources/using.md +++ b/docs_sources/using.md @@ -52,6 +52,12 @@ TPOT offers several arguments that can be provided at the command line: Number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. +-c +OFFSPRING_SIZE +Any positive integer +The number of children to produce in each generation. + + -mr MUTATION_RATE [0.0, 1.0] @@ -188,6 +194,11 @@ Note that you can pass several parameters to the TPOT instantiation call: The number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. +offspring_size +Any positive integer +The number of children to produce in each generation. + + mutation_rate [0.0, 1.0] The mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing. diff --git a/tests.py b/tests.py index 4e58e5ff..8bf6ec03 100644 --- a/tests.py +++ b/tests.py @@ -12,7 +12,7 @@ from tpot.gp_deap import mutNodeReplacement from tpot.decorators import _timeout, TimedOutExc -from tpot.operator_utils import TPOTOperatorClassFactory +from tpot.operator_utils import TPOTOperatorClassFactory, set_sample_weight from tpot.config_classifier import classifier_config_dict @@ -24,8 +24,7 @@ import subprocess from sklearn.datasets import load_digits, load_boston -from sklearn.model_selection import train_test_split - +from sklearn.model_selection import train_test_split, cross_val_score from deap import creator from tqdm import tqdm @@ -48,7 +47,7 @@ def test_driver(): """Assert that the TPOT driver output normal result""" - batcmd = "python -m tpot.driver tests.csv -is , -target class -g 2 -p 2 -os 4 -cv 5 -s 45 -v 1" + batcmd = "python -m tpot.driver tests.csv -is , -target class -g 2 -p 2 -c 4 -cv 5 -s 45 -v 1" ret_stdout = subprocess.check_output(batcmd, shell=True) try: ret_val = float(ret_stdout.decode("utf-8").split('\n')[-2].split(': ')[-1]) @@ -215,7 +214,7 @@ def test_score_2(): # http://stackoverflow.com/questions/5595425/ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) - + assert isclose(known_score, score) def test_score_3(): @@ -247,6 +246,51 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): assert isclose(known_score, score) +def test_sample_weight_func(): + """Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline with sample weights""" + + tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error') + + # Reify pipeline with known scor + + pipeline_string = ("ExtraTreesRegressor(" + "GradientBoostingRegressor(input_matrix, GradientBoostingRegressor__alpha=0.8," + "GradientBoostingRegressor__learning_rate=0.1,GradientBoostingRegressor__loss=huber," + "GradientBoostingRegressor__max_depth=5, GradientBoostingRegressor__max_features=0.5," + "GradientBoostingRegressor__min_samples_leaf=5, GradientBoostingRegressor__min_samples_split=5," + "GradientBoostingRegressor__subsample=0.25)," + "ExtraTreesRegressor__bootstrap=True,ExtraTreesRegressor__max_features=0.5," + "ExtraTreesRegressor__min_samples_leaf=5,ExtraTreesRegressor__min_samples_split=5)") + tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset) + tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline) + + # make up a sample weight + training_classes_r_weight = np.array(range(1, len(training_classes_r)+1)) + training_classes_r_weight_dict = set_sample_weight(tpot_obj._fitted_pipeline.steps, training_classes_r_weight) + + np.random.seed(42) + cv_score1 = cross_val_score(tpot_obj._fitted_pipeline, training_features_r, training_classes_r, cv=3, scoring='neg_mean_squared_error') + + np.random.seed(42) + cv_score2 = cross_val_score(tpot_obj._fitted_pipeline, training_features_r, training_classes_r, cv=3, scoring='neg_mean_squared_error') + + np.random.seed(42) + cv_score_weight = cross_val_score(tpot_obj._fitted_pipeline, training_features_r, training_classes_r, cv=3, scoring='neg_mean_squared_error', fit_params=training_classes_r_weight_dict) + + np.random.seed(42) + tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r, **training_classes_r_weight_dict) + # Get score from TPOT + known_score = 14.1377471426 # Assumes use of mse + score = tpot_obj.score(testing_features_r, testing_classes_r) + + # http://stackoverflow.com/questions/5595425/ + def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + assert np.allclose(cv_score1, cv_score2) + assert not np.allclose(cv_score1, cv_score_weight) + assert isclose(known_score, score) + + def test_predict(): diff --git a/tpot/base.py b/tpot/base.py index 99ca6145..0570493c 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -43,7 +43,7 @@ from update_checker import update_check from ._version import __version__ -from .operator_utils import TPOTOperatorClassFactory, Operator, ARGType +from .operator_utils import TPOTOperatorClassFactory, Operator, ARGType, set_sample_weight from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code from .decorators import _timeout, _pre_test, TimedOutExc @@ -643,8 +643,6 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = total_mins_elapsed = (datetime.now() - self._start_datetime).total_seconds() / 60. if total_mins_elapsed >= self.max_time_mins: raise KeyboardInterrupt('{} minutes have elapsed. TPOT will close down.'.format(total_mins_elapsed)) - if not sample_weight: - sample_weight_dict = None # return fitness scores fitnesses_dict = {} @@ -697,35 +695,10 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = sklearn_pipeline_list.append(sklearn_pipeline) test_idx_list.append(indidx) - def _set_sample_weight(pipeline_steps, sample_weight): - """Recursively iterates through all objects in the pipeline and sets the given parameter to the specified value - - Parameters - ---------- - pipeline_steps: array-like - List of (str, obj) tuples from a scikit-learn pipeline or related object - sample_weight: array-like - List of sample weight - Returns - ------- - sample_weight_dict: - A dictionary of sample_weight - - """ - sample_weight_dict = {} - for (pname, obj) in pipeline_steps: - if inspect.getargspec(obj.fit).args.count('sample_weight') and sample_weight: - step_sw = pname + '__sample_weight' - sample_weight_dict[step_sw] = sample_weight - if sample_weight_dict: - return sample_weight_dict - else: - return None - @_timeout(max_eval_time_mins=self.max_eval_time_mins) def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classes, cv=self.cv, scoring_function=self.scoring_function,sample_weight=sample_weight): - sample_weight_dict = _set_sample_weight(sklearn_pipeline.steps, sample_weight) + sample_weight_dict = set_sample_weight(sklearn_pipeline.steps, sample_weight) from .decorators import TimedOutExc try: with warnings.catch_warnings(): diff --git a/tpot/driver.py b/tpot/driver.py index db4bbb86..679d1f9c 100644 --- a/tpot/driver.py +++ b/tpot/driver.py @@ -106,9 +106,9 @@ def main(): '(and therefore time) to optimize over. TPOT will evaluate ' 'GENERATIONS x POPULATION_SIZE number of pipelines in total.') - parser.add_argument('-os', action='store', dest='OFFSPRING_SIZE', default=100, + parser.add_argument('-c', action='store', dest='OFFSPRING_SIZE', default=100, type=positive_integer, help='The number of children to produce ' - 'at each generation.') + 'in each generation.') parser.add_argument('-mr', action='store', dest='MUTATION_RATE', default=0.9, type=float_range, help='GP mutation rate in the range [0.0, 1.0]. We ' diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index aa6604e5..d4c9c959 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -21,6 +21,7 @@ import numpy as np from sklearn.base import ClassifierMixin from sklearn.base import RegressorMixin +import inspect class Operator(object): @@ -72,6 +73,32 @@ def source_decode(sourcecode): op_obj = None return import_str, op_str, op_obj +def set_sample_weight(pipeline_steps, sample_weight=None): + """Recursively iterates through all objects in the pipeline and sets sample weight + + Parameters + ---------- + pipeline_steps: array-like + List of (str, obj) tuples from a scikit-learn pipeline or related object + sample_weight: array-like + List of sample weight + Returns + ------- + sample_weight_dict: + A dictionary of sample_weight + + """ + sample_weight_dict = {} + if not isinstance(sample_weight, type(None)): + for (pname, obj) in pipeline_steps: + if inspect.getargspec(obj.fit).args.count('sample_weight'): + step_sw = pname + '__sample_weight' + sample_weight_dict[step_sw] = sample_weight + if sample_weight_dict: + return sample_weight_dict + else: + return None + def ARGTypeClassFactory(classname, prange, BaseClass=ARGType): """ Dynamically create parameter type class From a4d05a7b439116070ef0fceac8f56bd3201046c2 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 17 Mar 2017 14:42:13 -0400 Subject: [PATCH 132/154] add release note --- docs_sources/releases.md | 6 +++++- tests.py | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs_sources/releases.md b/docs_sources/releases.md index 8b69e421..24f8b485 100644 --- a/docs_sources/releases.md +++ b/docs_sources/releases.md @@ -1,6 +1,6 @@ # Version 0.7 -* **TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only)** TPOT allows you to use multiple processes for pipeline optimization in TPOT with the `n_jobs` parameter in both TPOTClassifier and TPOTRegressor. The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-njobs` parameter. +* **TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only)** TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the `n_jobs` parameter in both TPOTClassifier and TPOTRegressor. The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-njobs` parameter. * **TPOT now support customized dictionary of operators and parameters** TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the `operator_dict` parameter. The format of this customized dictionary can be found in [online manual](/using/#tpot-with-code). The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-operator` parameter but take a file including the dictionary instead. @@ -14,6 +14,10 @@ * The TPOT point mutation operator was refined +* TPOT now supports sample weights to be used like `TPOTRegressor.fit(x_train, y_train, sample_weights=sample_weights)` + +* TPOT now checks duplicated pipeline to accelerate optimization process. + * The default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn. diff --git a/tests.py b/tests.py index 8bf6ec03..eb136695 100644 --- a/tests.py +++ b/tests.py @@ -291,8 +291,6 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0): assert isclose(known_score, score) - - def test_predict(): """Assert that the TPOT predict function raises a ValueError when no optimized pipeline exists""" From 5ca593116c64fb9a449f73062c4f3df4dd94d22c Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Fri, 17 Mar 2017 14:50:59 -0400 Subject: [PATCH 133/154] clean up --- docs_sources/releases.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_sources/releases.md b/docs_sources/releases.md index 24f8b485..0cfea94e 100644 --- a/docs_sources/releases.md +++ b/docs_sources/releases.md @@ -2,7 +2,7 @@ * **TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only)** TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the `n_jobs` parameter in both TPOTClassifier and TPOTRegressor. The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-njobs` parameter. -* **TPOT now support customized dictionary of operators and parameters** TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the `operator_dict` parameter. The format of this customized dictionary can be found in [online manual](/using/#tpot-with-code). The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-operator` parameter but take a file including the dictionary instead. +* **TPOT now support customized dictionary of operators and parameters** TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the `operator_dict` parameter. The format of this customized dictionary can be found in [online manual](/using/#tpot-with-code). The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-operator` parameter but only takes a file including the dictionary instead. * TPOT now allows you to **specify a time limit (default time limit is 5 minutes)** for evaluating a single pipeline in optimization process with the `max_eval_time_mins` parameter, so TPOT can skip these time-consuming pipelines. From d07ba32abe432fe8bc68a4ee4edb453f0edff500 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 20 Mar 2017 12:05:57 -0400 Subject: [PATCH 134/154] add doc for installing pywin32 in windows OS --- docs_sources/installing.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs_sources/installing.md b/docs_sources/installing.md index eebe4396..7f09a3dc 100644 --- a/docs_sources/installing.md +++ b/docs_sources/installing.md @@ -26,6 +26,12 @@ DEAP, update_checker, and tqdm (used for verbose TPOT runs) can be installed wit pip install deap update_checker tqdm ``` +**For Windows OS,**, the pywin32 module is required if the Python is NOT installed via [Anaconda Python distribution](https://www.continuum.io/downloads) and can be installed with `pip` via the command: + +```Shell +pip install pywin32 +``` + Optionally, install XGBoost if you would like TPOT to use XGBoost. XGBoost is entirely optional, and TPOT will still function normally without XGBoost if you do not have it installed. ```Shell From bb71fa0267fe5db9f4328c698d32dc3579a71924 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Mon, 20 Mar 2017 16:59:00 -0400 Subject: [PATCH 135/154] Tons of code cleanup and docstrings updates --- .gitignore | 2 + tests.py | 14 +- tpot/base.py | 197 ++++++++---------- ..._in_operators.py => built_in_operators.py} | 0 tpot/config_classifier.py | 2 +- tpot/config_regressor.py | 2 +- tpot/decorators.py | 11 +- tpot/driver.py | 115 +++++----- tpot/operator_utils.py | 2 +- tpot/tpot.py | 4 +- 10 files changed, 170 insertions(+), 179 deletions(-) rename tpot/{build_in_operators.py => built_in_operators.py} (100%) diff --git a/.gitignore b/.gitignore index 332f1f28..d437602b 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,5 @@ docs/sources/examples/.Rhistory analyze-oj2-tpot-mdr.ipynb tpot-mdr-demo.ipynb + +github.com/ diff --git a/tests.py b/tests.py index eb136695..52fde637 100644 --- a/tests.py +++ b/tests.py @@ -47,10 +47,10 @@ def test_driver(): """Assert that the TPOT driver output normal result""" - batcmd = "python -m tpot.driver tests.csv -is , -target class -g 2 -p 2 -c 4 -cv 5 -s 45 -v 1" + batcmd = "python -m tpot.driver tests.csv -is , -target class -g 2 -p 2 -os 4 -cv 5 -s 45 -v 1" ret_stdout = subprocess.check_output(batcmd, shell=True) try: - ret_val = float(ret_stdout.decode("utf-8").split('\n')[-2].split(': ')[-1]) + ret_val = float(ret_stdout.decode('UTF-8').split('\n')[-2].split(': ')[-1]) except: ret_val = -float('inf') assert ret_val > 0.0 @@ -118,7 +118,7 @@ def test_get_params(): 'population_size': 500, 'generations': 1000, 'offspring_size': 2000, - 'operator_dict': classifier_config_dict, + 'config_dict': classifier_config_dict, 'verbosity': 1 } @@ -166,7 +166,7 @@ def test_random_ind_2(): from sklearn.linear_model import LogisticRegression from sklearn.model_selection import train_test_split from sklearn.pipeline import make_pipeline -from tpot.build_in_operators import ZeroCount +from tpot.built_in_operators import ZeroCount # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -388,7 +388,7 @@ def test_fit(): def testTPOTOperatorClassFactory(): """Assert that the TPOT operators class factory""" - test_operator_dict = { + test_config_dict = { 'sklearn.svm.LinearSVC': { 'penalty': ["l1", "l2"], 'loss': ["hinge", "squared_hinge"], @@ -409,8 +409,8 @@ def testTPOTOperatorClassFactory(): } tpot_operator_list = [] tpot_argument_list = [] - for key in sorted(test_operator_dict.keys()): - op,args = TPOTOperatorClassFactory(key,test_operator_dict[key]) + for key in sorted(test_config_dict.keys()): + op,args = TPOTOperatorClassFactory(key, test_config_dict[key]) tpot_operator_list.append(op) tpot_argument_list += args assert len(tpot_operator_list) == 3 diff --git a/tpot/base.py b/tpot/base.py index 0570493c..6ad0bdb3 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -47,19 +47,12 @@ from .export_utils import export_pipeline, expr_to_tree, generate_pipeline_code from .decorators import _timeout, _pre_test, TimedOutExc -from .build_in_operators import CombineDFs +from .built_in_operators import CombineDFs - -from .gp_types import Bool, Output_DF from .metrics import SCORERS +from .gp_types import Bool, Output_DF from .gp_deap import eaMuPlusLambda, mutNodeReplacement - - - - - - # hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS if sys.platform.startswith('win'): import win32api @@ -73,48 +66,47 @@ def handler(dwCtrlType, hook_sigint=_thread.interrupt_main): return 1 # don't chain to the next handler return 0 win32api.SetConsoleCtrlHandler(handler, 1) -# add time limit for imported function -#cross_val_score = _timeout(cross_val_score) - class TPOTBase(BaseEstimator): """TPOT automatically creates and optimizes machine learning pipelines using genetic programming""" - def __init__(self, population_size=100, generations=100, offspring_size=None, - mutation_rate=0.9, crossover_rate=0.05, + def __init__(self, generations=100, population_size=100, offspring_size=None, + mutation_rate=0.9, crossover_rate=0.1, scoring=None, cv=5, n_jobs=1, max_time_mins=None, max_eval_time_mins=5, - random_state=None, operator_dict=None, verbosity=0, - disable_update_check=False, warm_start=False): + random_state=None, config_dict=None, warm_start=False, + verbosity=0, disable_update_check=False): """Sets up the genetic programming algorithm for pipeline optimization. Parameters ---------- + generations: int (default: 100) + Number of iterations to the run pipeline optimization process. + Generally, TPOT will work better when you give it more generations (and + therefore time) to optimize the pipeline. TPOT will evaluate + POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. population_size: int (default: 100) - The number of pipelines in the genetic algorithm population. Must - be > 0.The more pipelines in the population, the slower TPOT will - run, but it's also more likely to find better pipelines. + Number of individuals to retain in the GP population every generation. + Generally, TPOT will work better when you give it more individuals + (and therefore time) to optimize the pipeline. TPOT will evaluate + POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. offspring_size: int (default: None) - The number of children to produce at each generation. - generations: int (default: 100) - The number of generations to run pipeline optimization for. Must - be > 0. The more generations you give TPOT to run, the longer it - takes, but it's also more likely to find better pipelines. + Number of offspring to produce in each GP generation. + By default, offspring_size = population_size. mutation_rate: float (default: 0.9) - The mutation rate for the genetic programming algorithm in the range - [0.0, 1.0]. This tells the genetic programming algorithm how many - pipelines to apply random changes to every generation. We don't - recommend that you tweak this parameter unless you know what you're - doing. - crossover_rate: float (default: 0.05) - The crossover rate for the genetic programming algorithm in the - range [0.0, 1.0]. This tells the genetic programming algorithm how - many pipelines to "breed" every generation. We don't recommend that - you tweak this parameter unless you know what you're doing. + Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. + This parameter tells the GP algorithm how many pipelines to apply random + changes to every generation. We recommend using the default parameter unless + you understand how the mutation rate affects GP algorithms + crossover_rate: float (default: 0.1) + Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. + This parameter tells the genetic programming algorithm how many pipelines to + "breed" every generation. We recommend using the default parameter unless you + understand how the mutation rate affects GP algorithms. scoring: function or str Function used to evaluate the quality of a given pipeline for the - problem. By default, balanced class accuracy is used for - classification problems, mean squared error for regression problems. + problem. By default, accuracy is used for classification problems and + mean squared error (mse) for regression problems. TPOT assumes that this scoring function should be maximized, i.e., higher is better. @@ -126,33 +118,35 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'] cv: int (default: 5) - The number of folds to evaluate each pipeline over in k-fold - cross-validation during the TPOT pipeline optimization process + Number of folds to evaluate each pipeline over in k-fold cross-validation + during the TPOT optimization process. n_jobs: int (default: 1) - The number of CPUs for evaluating each pipeline over - cross-validation during the TPOT pipeline optimization process + Number of CPUs for evaluating pipelines in parallel during the TPOT + optimization process. max_time_mins: int (default: None) - How many minutes TPOT has to optimize the pipeline. If not None, - this setting will override the `generations` parameter. + How many minutes TPOT has to optimize the pipeline. + If not None, this setting will override the "generations" parameter and allow + TPOT to run until it runs out of time. max_eval_time_mins: int (default: 5) How many minutes TPOT has to optimize a single pipeline. - Setting this parameter to higher values will allow TPOT to explore more complex - pipelines but will also allow TPOT to run longer. + Setting this parameter to higher values will allow TPOT to explore more + complex pipelines, but will also allow TPOT to run longer. random_state: int (default: None) - The random number generator seed for TPOT. Use this to make sure + Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. - operator_dict: a customized python dictionary (default: None) - The customized python dictionary to specify the list of operators and - their arguments. Format examples: config_regressor.py and config_classifier.py + config_dict: dictionary (default: None) + Configuration dictionary for customizing the operators and parameters that + TPOT uses in the optimization process. + For examples, see config_regressor.py and config_classifier.py + warm_start: bool (default: False) + Flag indicating whether the TPOT instance will reuse the population from + previous calls to fit(). verbosity: int (default: 0) How much information TPOT communicates while it's running. - 0 = none, 1 = minimal, 2 = all + 0 = none, 1 = minimal, 2 = high, 3 = all. disable_update_check: bool (default: False) Flag indicating whether the TPOT version checker should be disabled. - warm_start: bool (default: False) - Flag indicating whether TPOT will reuse models from previous calls to - fit() for faster operation Returns ------- @@ -160,8 +154,7 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, """ if self.__class__.__name__ == 'TPOTBase': - raise RuntimeError('Do not instantiate the TPOTBase class directly; ' - 'use TPOTRegressor or TPOTClassifier instead.') + raise RuntimeError('Do not instantiate the TPOTBase class directly; use TPOTRegressor or TPOTClassifier instead.') # Prompt the user if their version is out of date self.disable_update_check = disable_update_check @@ -178,41 +171,37 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.max_time_mins = max_time_mins self.max_eval_time_mins = max_eval_time_mins - - # set offspring_size equal to population_size by default + # Set offspring_size equal to population_size by default if offspring_size: self.offspring_size = offspring_size else: self.offspring_size = population_size - # define operator dictionary based on files - if operator_dict: - self.operator_dict = operator_dict + # Define the configuration dictionary based on files + if config_dict: + self.config_dict = config_dict else: - self.operator_dict = self.default_operator_dict + self.config_dict = self.default_config_dict self.operators = [] self.arguments = [] - for key in sorted(self.operator_dict.keys()): - op_class, arg_types = TPOTOperatorClassFactory(key, self.operator_dict[key], + for key in sorted(self.config_dict.keys()): + op_class, arg_types = TPOTOperatorClassFactory(key, self.config_dict[key], BaseClass=Operator, ArgBaseClass=ARGType) if op_class: self.operators.append(op_class) self.arguments += arg_types - - # Schedule TPOT to run for a very long time if the user specifies a run-time - # limit TPOT will automatically interrupt itself when the timer runs out + # Schedule TPOT to run for many generations if the user specifies a run-time limit + # TPOT will automatically interrupt itself when the timer runs out if not (max_time_mins is None): self.generations = 1000000 self.mutation_rate = mutation_rate self.crossover_rate = crossover_rate - # check if mutation_rate + crossover_rate > 1 if self.mutation_rate + self.crossover_rate > 1: - raise TypeError('The sum of the crossover and mutation probabilities must be smaller ' - 'or equal to 1.0.') + raise TypeError('The sum of the crossover and mutation probabilities must be <= 1.0.') self.verbosity = verbosity self.operators_context = { @@ -222,15 +211,13 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, 'FunctionTransformer': FunctionTransformer } - self._pbar = None - # a dictionary of individual which has already evaluated in previous generation. - self.eval_ind = {} + # Dictionary of individuals that have already been evaluated in previous generations + self._evaluated_individuals = {} self.random_state = random_state - # If the user passed a custom scoring function, store it in the sklearn SCORERS dictionary if scoring: if hasattr(scoring, '__call__'): @@ -249,8 +236,8 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self.cv = cv # If the OS is windows, reset cpu number to 1 since the OS did not have multiprocessing module if sys.platform.startswith('win') and n_jobs > 1: - print('Warning: Parallelizing cross validation is not supported in Windows OS.', - 'Reset number of cpu to 1 during TPOT pipeline optimization process') + print('Warning: Parallelization is not currently supported in TPOT for Windows.', + 'Setting n_jobs to 1 during the TPOT optimization process.') self.n_jobs = 1 else: self.n_jobs = n_jobs @@ -259,14 +246,10 @@ def __init__(self, population_size=100, generations=100, offspring_size=None, self._setup_toolbox() def _setup_pset(self): - - # creating dynamically create operator class - if self.random_state is not None: random.seed(self.random_state) np.random.seed(self.random_state) - self._pset = gp.PrimitiveSetTyped('MAIN', [np.ndarray], Output_DF) # Rename pipeline input to "input_df" @@ -308,8 +291,7 @@ def _setup_pset(self): self._pset.addTerminal(val, _type, name=terminal_name) if self.verbosity > 2: - print('{} operators are imported.'.format(len(self.operators))) - + print('{} operators have been imported by TPOT.'.format(len(self.operators))) def _setup_toolbox(self): @@ -413,23 +395,22 @@ def pareto_eq(ind1, ind2): # Allow for certain exceptions to signal a premature fit() cancellation except (KeyboardInterrupt, SystemExit): if self.verbosity > 0: - print('') # just for better interface - print('GP closed prematurely - will use current best pipeline') + self._pbar.write('') # just for better interface + self._pbar.write('GP closed prematurely - will use current best pipeline') finally: # Close the progress bar # Standard truthiness checks won't work for tqdm if not isinstance(self._pbar, type(None)): self._pbar.close() - # Store the pipeline with the highest internal testing score if self._pareto_front: top_score = -float('inf') for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)): if pipeline_scores.wvalues[1] > top_score: self._optimized_pipeline = pipeline - top_score = pipeline_scores.wvalues[1] + # It won't raise error for a small test like in a unit test because a few pipeline sometimes # may fail due to the training data does not fit the operator's requirement. if not self._optimized_pipeline: @@ -657,15 +638,19 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = individual = individuals[indidx] individual_str = str(individual) if (individual_str.count('PolynomialFeatures') > 1): - print('Invalid pipeline -- skipping its evaluation') + if self.verbosity > 2: + self._pbar.write('Invalid pipeline encountered. Skipping its evaluation.') fitnesses_dict[indidx] = (5000., -float('inf')) if not self._pbar.disable: self._pbar.update(1) - # check if the individual are evaluated before - elif individual_str in self.eval_ind: - # get fitness score from previous evaluation - fitnesses_dict[indidx] = self.eval_ind[individual_str] + # Check if the individual was evaluated before + elif individual_str in self._evaluated_individuals: + # Get fitness score from previous evaluation + fitnesses_dict[indidx] = self._evaluated_individuals[individual_str] + if self.verbosity > 2: + self._pbar.write('Pipeline encountered that has previously been evaluated during the ' + 'optimization process. Using the score from the previous evaluation.') if not self._pbar.disable: self._pbar.update(1) else: @@ -678,7 +663,6 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = # Count the number of pipeline operators as a measure of pipeline complexity operator_count = 0 - # add time limit for evaluation of pipeline for i in range(len(individual)): node = individual[i] if ((type(node) is deap.gp.Terminal) or @@ -697,7 +681,8 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight = @_timeout(max_eval_time_mins=self.max_eval_time_mins) def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classes, - cv=self.cv, scoring_function=self.scoring_function,sample_weight=sample_weight): + cv=self.cv, scoring_function=self.scoring_function, + sample_weight=sample_weight): sample_weight_dict = set_sample_weight(sklearn_pipeline.steps, sample_weight) from .decorators import TimedOutExc try: @@ -708,7 +693,7 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe n_jobs=1, fit_params=sample_weight_dict) resulting_score = np.mean(cv_scores) except TimedOutExc: - resulting_score = "Timeout" + resulting_score = 'Timeout' except: resulting_score = -float('inf') return resulting_score @@ -721,22 +706,22 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe res_imap = pool.imap(_wrapped_cross_val_score, sklearn_pipeline_list) if not self._pbar.disable: ini_pbar_n = self._pbar.n - # hacky way for pbar update by using imap in pathos.multiprocessing.ProcessPool + # Hacky way for pbar update by using imap in pathos.multiprocessing.ProcessPool while not self._pbar.disable: tmp_fitness = np.array(res_imap._items) num_job_done = len(tmp_fitness) if not self._pbar.disable and num_job_done: - timeout_index = list(np.where(tmp_fitness[:,1] == "Timeout")[0]) + timeout_index = list(np.where(tmp_fitness[:, 1] == 'Timeout')[0]) for idx in timeout_index: - if self._pbar.n - ini_pbar_n <= idx: - self._pbar.write("Skip pipeline #{0} due to time out. " - "Continuing to the next pipeline.".format(ini_pbar_n + idx + 1)) + if self.verbosity > 2 and self._pbar.n - ini_pbar_n <= idx: + self._pbar.write('Skipped pipeline #{0} due to time out. ' + 'Continuing to the next pipeline.'.format(ini_pbar_n + idx + 1)) self._pbar.update(ini_pbar_n + num_job_done - self._pbar.n) if num_job_done >= len(sklearn_pipeline_list): break else: time.sleep(0.2) - resulting_score_list = [-float('inf') if x=="Timeout" else x for x in list(res_imap)] + resulting_score_list = [-float('inf') if x == 'Timeout' else x for x in list(res_imap)] else: resulting_score_list = [] for sklearn_pipeline in sklearn_pipeline_list: @@ -744,19 +729,19 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe resulting_score = _wrapped_cross_val_score(sklearn_pipeline) except TimedOutExc: resulting_score = -float('inf') - if not self._pbar.disable: - self._pbar.write("Skip pipeline #{0} due to time out. " - "Continuing to the next pipeline.".format(self._pbar.n + 1)) + if self.verbosity > 2 and not self._pbar.disable: + self._pbar.write('Skipped pipeline #{0} due to time out. ' + 'Continuing to the next pipeline.'.format(self._pbar.n + 1)) resulting_score_list.append(resulting_score) if not self._pbar.disable: self._pbar.update(1) for resulting_score, operator_count, individual_str, test_idx in zip(resulting_score_list, operator_count_list, eval_individuals_str, test_idx_list): if type(resulting_score) in [float, np.float64, np.float32]: - self.eval_ind[individual_str] = (max(1, operator_count), resulting_score) - fitnesses_dict[test_idx] = self.eval_ind[individual_str] + self._evaluated_individuals[individual_str] = (max(1, operator_count), resulting_score) + fitnesses_dict[test_idx] = self._evaluated_individuals[individual_str] else: - raise ValueError('Scoring function does not return a float') + raise ValueError('Scoring function does not return a float.') fitnesses_ordered = [] for key in sorted(fitnesses_dict.keys()): @@ -770,7 +755,7 @@ def _mate_operator(self, ind1, ind2): @_pre_test def _random_mutation_operator(self, individual): - """Perform a replacement, insert, or shrink mutation on an individual + """Perform a replacement, insertion, or shrink mutation on an individual Parameters ---------- @@ -796,7 +781,7 @@ def _random_mutation_operator(self, individual): def _gen_grow_safe(self, pset, min_, max_, type_=None): """Generate an expression where each leaf might have a different depth - between *min* and *max*. + between min_ and max_. Parameters ---------- diff --git a/tpot/build_in_operators.py b/tpot/built_in_operators.py similarity index 100% rename from tpot/build_in_operators.py rename to tpot/built_in_operators.py diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py index 94ab9fe7..0e77216d 100644 --- a/tpot/config_classifier.py +++ b/tpot/config_classifier.py @@ -155,7 +155,7 @@ 'sklearn.preprocessing.StandardScaler': { }, - 'tpot.build_in_operators.ZeroCount': { + 'tpot.built_in_operators.ZeroCount': { }, # Selectors diff --git a/tpot/config_regressor.py b/tpot/config_regressor.py index 7d6190d9..8701152c 100644 --- a/tpot/config_regressor.py +++ b/tpot/config_regressor.py @@ -152,7 +152,7 @@ 'sklearn.preprocessing.StandardScaler': { }, - 'tpot.build_in_operators.ZeroCount': { + 'tpot.built_in_operators.ZeroCount': { }, # Selectors diff --git a/tpot/decorators.py b/tpot/decorators.py index 3295be91..9ea3f3ae 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -81,7 +81,7 @@ def limitedTime(*args, **kw): warnings.simplefilter('ignore') ret = func(*args, **kw) except: - raise TimedOutExc("Time Out!") + raise TimedOutExc('Time Out!') finally: signal.signal(signal.SIGALRM, old_signal_hander) # Old signal handler is restored signal.alarm(0) # Alarm removed @@ -159,9 +159,12 @@ def check_pipeline(self, *args, **kwargs): sklearn_pipeline.fit(pretest_X_reg, pretest_y_reg) bad_pipeline = False except BaseException as e: - if self.verbosity == 3: - print('_pre_test decorator: {fname}: num_test={n} {e}'.format(n=num_test, fname=func.__name__, e=e)) - pass + if self.verbosity > 2: + print_function = print + # Use the pbar output stream if it's active + if not isinstance(self._pbar, type(None)): + print_function = self._pbar.write + print_function('_pre_test decorator: {fname}: num_test={n} {e}'.format(n=num_test, fname=func.__name__, e=e)) finally: num_test += 1 diff --git a/tpot/driver.py b/tpot/driver.py index 679d1f9c..f6a475b8 100644 --- a/tpot/driver.py +++ b/tpot/driver.py @@ -76,10 +76,11 @@ def main(): 'automatically creates and optimizes machine learning pipelines using ' 'genetic programming.', add_help=False) - parser.add_argument('INPUT_FILE', type=str, help='Data file to optimize the ' - 'pipeline on; ensure that the class label column is labeled as "class".') + parser.add_argument('INPUT_FILE', type=str, help='Data file to use in the TPOT ' + 'optimization process. Ensure that the class label column is labeled as "class".') - parser.add_argument('-h', '--help', action='help', help='Show this help message and exit.') + parser.add_argument('-h', '--help', action='help', + help='Show this help message and exit.') parser.add_argument('-is', action='store', dest='INPUT_SEPARATOR', default='\t', type=str, help='Character used to separate columns in the input file.') @@ -89,49 +90,41 @@ def main(): parser.add_argument('-mode', action='store', dest='TPOT_MODE', choices=['classification', 'regression'], default='classification', type=str, - help='Whether TPOT is being used for a classification or regression problem.') + help='Whether TPOT is being used for a supervised classification or regression problem.') parser.add_argument('-o', action='store', dest='OUTPUT_FILE', default='', type=str, help='File to export the final optimized pipeline.') parser.add_argument('-g', action='store', dest='GENERATIONS', default=100, - type=positive_integer, help='Number of generations to run pipeline ' - 'optimization over.\nGenerally, TPOT will work better when ' - 'you give it more generations (and therefore time) to optimize over. ' - 'TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.') + type=positive_integer, help='Number of iterations to run the pipeline optimization process.\n' + 'Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. ' + 'TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.') parser.add_argument('-p', action='store', dest='POPULATION_SIZE', default=100, - type=positive_integer, help='Number of individuals in the GP population.\n' - 'Generally, TPOT will work better when you give it more individuals ' - '(and therefore time) to optimize over. TPOT will evaluate ' - 'GENERATIONS x POPULATION_SIZE number of pipelines in total.') + type=positive_integer, help='Number of individuals to retain in the GP population every generation.\n' + 'Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. ' + 'TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.') - parser.add_argument('-c', action='store', dest='OFFSPRING_SIZE', default=100, - type=positive_integer, help='The number of children to produce ' - 'in each generation.') + parser.add_argument('-os', action='store', dest='OFFSPRING_SIZE', default=None, + type=positive_integer, help='Number of offspring to produce in each GP generation. ' + 'By default, OFFSPRING_SIZE = POPULATION_SIZE.') parser.add_argument('-mr', action='store', dest='MUTATION_RATE', default=0.9, - type=float_range, help='GP mutation rate in the range [0.0, 1.0]. We ' - 'recommend using the default parameter unless you ' - 'understand how the mutation rate affects GP algorithms.') + type=float_range, help='GP mutation rate in the range [0.0, 1.0]. This tells the ' + 'GP algorithm how many pipelines to apply random changes to every generation. ' + 'We recommend using the default parameter unless you understand how the mutation ' + 'rate affects GP algorithms.') - parser.add_argument('-xr', action='store', dest='CROSSOVER_RATE', default=0.05, - type=float_range, help='GP crossover rate in the range [0.0, 1.0]. We ' - 'recommend using the default parameter unless you ' - 'understand how the crossover rate affects GP algorithms.') - - parser.add_argument('-cv', action='store', dest='NUM_CV_FOLDS', default=5, - type=int, help='The number of folds to evaluate each pipeline over in ' - 'k-fold cross-validation during the TPOT pipeline optimization process.') - - parser.add_argument('-njobs', action='store', dest='NUM_JOBS', default=1, - type=int, help='The number of CPUs for evaluating each pipeline in ' - ' cross-validation during the TPOT pipeline optimization process.') + parser.add_argument('-xr', action='store', dest='CROSSOVER_RATE', default=0.1, + type=float_range, help='GP crossover rate in the range [0.0, 1.0]. This tells the ' + 'GP algorithm how many pipelines to "breed" every generation. ' + 'We recommend using the default parameter unless you understand how the crossover ' + 'rate affects GP algorithms.') parser.add_argument('-scoring', action='store', dest='SCORING_FN', default=None, type=str, help='Function used to evaluate the quality of a given pipeline for ' - 'the problem. By default, balanced accuracy is used for classification and mean ' - 'squared error is used for regression. ' + 'the problem. By default, accuracy is used for classification problems and mean ' + 'squared error (mse) is used for regression problems. ' 'TPOT assumes that any function with "error" or "loss" in the name is meant to ' 'be minimized, whereas any other functions will be maximized. ' 'Offers the same options as cross_val_score: ' @@ -141,6 +134,14 @@ def main(): '"precision_micro", "precision_samples", "precision_weighted", "r2", "recall", ' '"recall_macro", "recall_micro", "recall_samples", "recall_weighted", "roc_auc"') + parser.add_argument('-cv', action='store', dest='NUM_CV_FOLDS', default=5, + type=int, help='Number of folds to evaluate each pipeline over in ' + 'k-fold cross-validation during the TPOT optimization process.') + + parser.add_argument('-njobs', action='store', dest='NUM_JOBS', default=1, + type=int, help='Number of CPUs for evaluating pipelines in parallel ' + ' during the TPOT optimization process.') + parser.add_argument('-maxtime', action='store', dest='MAX_TIME_MINS', default=None, type=int, help='How many minutes TPOT has to optimize the pipeline. This ' 'setting will override the GENERATIONS parameter ' @@ -156,13 +157,13 @@ def main(): 'this seed if you want your TPOT run to be reproducible with the same ' 'seed and data set in the future.') - parser.add_argument('-operator', action='store', dest='OPERATOR', default='', - type=str, help='File including a customized python dictionary to specify ' - 'operators and their arguments') + parser.add_argument('-config', action='store', dest='CONFIG_FILE', default='', + type=str, help='Configuration file for customizing the operators and parameters ' + 'that TPOT uses in the optimization process.') parser.add_argument('-v', action='store', dest='VERBOSITY', default=1, - choices=[0, 1, 2, 3], type=int, help='How much information TPOT ' - 'communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all.') + choices=[0, 1, 2, 3], type=int, help='How much information TPOT communicates ' + 'while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all.') parser.add_argument('--no-update-check', action='store_true', dest='DISABLE_UPDATE_CHECK', default=False, @@ -170,10 +171,7 @@ def main(): parser.add_argument('--version', action='version', version='TPOT {version}'.format(version=__version__), - help='Show TPOT\'s version number and exit.') - - - + help='Show the TPOT version number and exit.') args = parser.parse_args() @@ -183,11 +181,13 @@ def main(): arg_val = args.__dict__[arg] if arg == 'DISABLE_UPDATE_CHECK': continue - elif arg == 'SCORING_FN' and args.__dict__[arg] is None: + elif arg == 'SCORING_FN' and arg_val is None: if args.TPOT_MODE == 'classification': - arg_val = 'balanced_accuracy' + arg_val = 'accuracy' else: arg_val = 'mean_squared_error' + elif arg == 'OFFSPRING_SIZE' and arg_val is None: + arg_val = args.__dict__['POPULATION_SIZE'] print('{}\t=\t{}'.format(arg, arg_val)) print('') @@ -207,24 +207,25 @@ def main(): else: tpot_type = TPOTRegressor - if args.OPERATOR: + operator_dict = None + if args.CONFIG_FILE: try: - with open(args.OPERATOR,'r') as inf: - file_string = inf.read() - operator_dict = eval(file_string[file_string.find('{'):(file_string.rfind('}')+1)]) + with open(args.CONFIG_FILE, 'r') as input_file: + file_string = input_file.read() + operator_dict = eval(file_string[file_string.find('{'):(file_string.rfind('}') + 1)]) except: - raise TypeError('The operator dictionary file is in bad format or not available! ' - 'Please check the dictionary file') - else: - operator_dict = None + raise TypeError('The operator configuration file is in a bad format or not available. ' + 'Please check the configuration file before running TPOT.') tpot = tpot_type(generations=args.GENERATIONS, population_size=args.POPULATION_SIZE, - offspring_size=args.OFFSPRING_SIZE, mutation_rate=args.MUTATION_RATE, crossover_rate=args.CROSSOVER_RATE, - cv=args.NUM_CV_FOLDS, n_jobs=args.NUM_JOBS, - scoring=args.SCORING_FN, - max_time_mins=args.MAX_TIME_MINS, max_eval_time_mins=args.MAX_EVAL_MINS, - random_state=args.RANDOM_STATE, operator_dict=operator_dict, verbosity=args.VERBOSITY, - disable_update_check=args.DISABLE_UPDATE_CHECK) + offspring_size=args.OFFSPRING_SIZE, mutation_rate=args.MUTATION_RATE, crossover_rate=args.CROSSOVER_RATE, + cv=args.NUM_CV_FOLDS, n_jobs=args.NUM_JOBS, + scoring=args.SCORING_FN, + max_time_mins=args.MAX_TIME_MINS, max_eval_time_mins=args.MAX_EVAL_MINS, + random_state=args.RANDOM_STATE, config_dict=operator_dict, + verbosity=args.VERBOSITY, disable_update_check=args.DISABLE_UPDATE_CHECK) + + print('') tpot.fit(training_features, training_classes) diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index d4c9c959..4f69e1fe 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -69,7 +69,7 @@ def source_decode(sourcecode): exec('from {} import {}'.format(import_str, op_str)) op_obj = eval(op_str) except ImportError: - print("Operator {} is not available".format(sourcecode)) + print('Warning: {} is not available and will not be used by TPOT.'.format(sourcecode)) op_obj = None return import_str, op_str, op_obj diff --git a/tpot/tpot.py b/tpot/tpot.py index e6496e10..3fe3f43e 100644 --- a/tpot/tpot.py +++ b/tpot/tpot.py @@ -27,7 +27,7 @@ class TPOTClassifier(TPOTBase): """TPOT estimator for classification problems""" scoring_function = 'accuracy' # Classification scoring - default_operator_dict = classifier_config_dict # Classification dictionary + default_config_dict = classifier_config_dict # Classification dictionary classification = True regression = False @@ -36,6 +36,6 @@ class TPOTRegressor(TPOTBase): """TPOT estimator for regression problems""" scoring_function = 'neg_mean_squared_error' # Regression scoring - default_operator_dict = regressor_config_dict # Regression dictionary + default_config_dict = regressor_config_dict # Regression dictionary classification = False regression = True From d3f57c2e287e8c33dd718c95051764685ae0dcde Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Mon, 20 Mar 2017 17:11:31 -0400 Subject: [PATCH 136/154] Minor code organization changes --- tpot/base.py | 15 ++++++++------- tpot/decorators.py | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index 6ad0bdb3..e7583a9c 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -366,7 +366,8 @@ def pareto_eq(ind1, ind2): """ return np.allclose(ind1.fitness.values, ind2.fitness.values) - # generate new pareto front if it doesn't already exist for warm start + + # Generate new pareto front if it doesn't already exist for warm start if not self.warm_start or not self._pareto_front: self._pareto_front = tools.ParetoFront(similar=pareto_eq) @@ -396,7 +397,7 @@ def pareto_eq(ind1, ind2): except (KeyboardInterrupt, SystemExit): if self.verbosity > 0: self._pbar.write('') # just for better interface - self._pbar.write('GP closed prematurely - will use current best pipeline') + self._pbar.write('TPOT closed prematurely. Will use the current best pipeline.') finally: # Close the progress bar # Standard truthiness checks won't work for tqdm @@ -415,11 +416,11 @@ def pareto_eq(ind1, ind2): # may fail due to the training data does not fit the operator's requirement. if not self._optimized_pipeline: print('There was an error in the TPOT optimization ' - 'process. This could be because the data was ' - 'not formatted properly, or because data for ' - 'a regression problem was provided to the ' - 'TPOTClassifier object. Please make sure you ' - 'passed the data to TPOT correctly.') + 'process. This could be because the data was ' + 'not formatted properly, or because data for ' + 'a regression problem was provided to the ' + 'TPOTClassifier object. Please make sure you ' + 'passed the data to TPOT correctly.') else: self._fitted_pipeline = self._toolbox.compile(expr=self._optimized_pipeline) diff --git a/tpot/decorators.py b/tpot/decorators.py index 9ea3f3ae..516fe528 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -113,8 +113,8 @@ def limitedTime(*args, **kwargs): #timer = Timer(max_time_seconds, interrupt_main) tmp_it.join(max_time_seconds) if tmp_it.isAlive(): - raise TimedOutExc("Time Out!") - sys.tracebacklimit=1000 + raise TimedOutExc('Time Out!') + sys.tracebacklimit = 1000 return tmp_it.result tmp_it.stop() return limitedTime From f639a0226c0e58946242136ef93192e7b6145028 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Tue, 21 Mar 2017 13:58:47 -0400 Subject: [PATCH 137/154] Change "MISSING" param name to "DEFAULT" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was rather confusing what “MISSING” meant in the context of parameter values, and I believe that “DEFAULT” more clearly indicates what that parameter value means. --- tpot/base.py | 2 +- tpot/operator_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tpot/base.py b/tpot/base.py index e7583a9c..0316c84c 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -285,7 +285,7 @@ def _setup_pset(self): # Terminals for _type in self.arguments: - type_values = list(_type.values) + ['MISSING'] + type_values = list(_type.values) + ['DEFAULT'] for val in type_values: terminal_name = _type.__name__ + "=" + str(val) self._pset.addTerminal(val, _type, name=terminal_name) diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py index 4f69e1fe..e2a482a3 100644 --- a/tpot/operator_utils.py +++ b/tpot/operator_utils.py @@ -221,7 +221,7 @@ def export(cls, *args): if dep_op_list: dep_op_arguments = {} for arg_class, arg_value in zip(arg_types, args): - if arg_value == "MISSING": + if arg_value == "DEFAULT": continue aname_split = arg_class.__name__.split('__') if isinstance(arg_value, str): From 1c6e942f2887271bc200b36ccf4b3fa381c8557d Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Tue, 21 Mar 2017 14:09:07 -0400 Subject: [PATCH 138/154] Import the print function for Python 2 --- tpot/decorators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tpot/decorators.py b/tpot/decorators.py index 516fe528..5670b63c 100644 --- a/tpot/decorators.py +++ b/tpot/decorators.py @@ -18,6 +18,7 @@ """ +from __future__ import print_function from threading import Thread, current_thread from functools import wraps import sys From 1dbb7b146f235640776610b5d5c5f73917e718de Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Tue, 21 Mar 2017 15:21:45 -0400 Subject: [PATCH 139/154] Update tutorial examples for latest version --- tutorials/IRIS.ipynb | 281 +++++++++++++++-- tutorials/MNIST.ipynb | 137 ++++++--- tutorials/Titanic_Kaggle.ipynb | 463 +++++++++++++++++++++++------ tutorials/data/submission.csv | 124 ++++---- tutorials/tpot_iris_pipeline.py | 11 +- tutorials/tpot_mnist_pipeline.py | 26 +- tutorials/tpot_titanic_pipeline.py | 8 +- 7 files changed, 789 insertions(+), 261 deletions(-) diff --git a/tutorials/IRIS.ipynb b/tutorials/IRIS.ipynb index d1556ec8..2287b090 100644 --- a/tutorials/IRIS.ipynb +++ b/tutorials/IRIS.ipynb @@ -2,9 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -15,7 +17,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Load the IRIS data set and explore its contents. " ] @@ -24,7 +29,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -56,7 +63,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Split the data set in train and test. " ] @@ -65,7 +75,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -89,64 +101,274 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Warning: xgboost.XGBClassifier is not available and will not be used by TPOT.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 200/200 [00:18<00:00, 9.33pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 1 - Current best internal CV score: 0.9825757575757577\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 300/300 [00:28<00:00, 12.80pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 2 - Current best internal CV score: 0.9825757575757577\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 400/400 [00:39<00:00, 11.16pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 3 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 500/500 [00:45<00:00, 17.87pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 4 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 600/600 [00:52<00:00, 16.63pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 5 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 700/700 [00:58<00:00, 17.42pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 6 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 800/800 [01:06<00:00, 14.45pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 7 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 900/900 [01:11<00:00, 12.45pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 8 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 1000/1000 [01:17<00:00, 8.87pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 9 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 1100/1100 [01:23<00:00, 11.12pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 10 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 1200/1200 [01:29<00:00, 9.37pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 11 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 1300/1300 [01:34<00:00, 9.55pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 12 - Current best internal CV score: 0.990909090909091\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 1400/1400 [01:40<00:00, 16.09pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 13 - Current best internal CV score: 0.990909090909091\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "Optimization Progress: 100%|██████████| 100/100 [00:00<00:00, 1.96pipeline/s]" + "Optimization Progress: 100%|██████████| 1500/1500 [01:46<00:00, 9.96pipeline/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation 1 - Current best internal CV score: 0.9870085470085469\n" + "Generation 14 - Current best internal CV score: 0.990909090909091\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Optimization Progress: 95%|█████████▌| 190/200 [00:00<00:14, 1.41s/pipeline]" + "Optimization Progress: 100%|██████████| 1600/1600 [01:52<00:00, 10.02pipeline/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation 2 - Current best internal CV score: 0.9870085470085469\n" + "Generation 15 - Current best internal CV score: 0.990909090909091\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Optimization Progress: 96%|█████████▌| 288/300 [00:00<00:04, 2.56pipeline/s]" + "Optimization Progress: 100%|██████████| 1700/1700 [01:59<00:00, 7.28pipeline/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation 3 - Current best internal CV score: 0.9870085470085469\n" + "Generation 16 - Current best internal CV score: 0.990909090909091\n" ] }, { "name": "stderr", "output_type": "stream", - "text": [] + "text": [ + "\r", + " \r", + "\r", + " \r", + "Optimization Progress: 95%|█████████▍| 1703/1800 [02:01<00:13, 7.28pipeline/s]\r", + " \r", + "\r", + " \r", + "Optimization Progress: 95%|█████████▍| 1703/1800 [02:01<00:13, 7.28pipeline/s]\r", + " " + ] }, { "name": "stdout", "output_type": "stream", "text": [ - "GP closed prematurely - will use current best pipeline\n", "\n", - "Best pipeline: ExtraTreesClassifier(Nystroem(input_matrix, 50, 0.01, 8), 35, 3.0)\n", - "0.979700854701\n" + "TPOT closed prematurely. Will use the current best pipeline.\n", + "\n", + "Best pipeline: DecisionTreeClassifier(RBFSampler(input_matrix, RBFSampler__gamma=0.85), DecisionTreeClassifier__criterion=entropy, DecisionTreeClassifier__max_depth=3, DecisionTreeClassifier__min_samples_leaf=4, DecisionTreeClassifier__min_samples_split=9)\n", + "0.973684210526\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" ] } ], @@ -160,7 +382,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -171,18 +395,19 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "# %load tpot_iris_pipeline.py\n", "import numpy as np\n", "\n", - "from sklearn.ensemble import ExtraTreesClassifier, VotingClassifier\n", - "from sklearn.kernel_approximation import Nystroem\n", + "from sklearn.kernel_approximation import RBFSampler\n", "from sklearn.model_selection import train_test_split\n", - "from sklearn.pipeline import make_pipeline, make_union\n", - "from sklearn.preprocessing import FunctionTransformer\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.tree import DecisionTreeClassifier\n", "\n", "# NOTE: Make sure that the class is labeled 'class' in the data file\n", "tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\n", @@ -191,8 +416,8 @@ " train_test_split(features, tpot_data['class'], random_state=42)\n", "\n", "exported_pipeline = make_pipeline(\n", - " Nystroem(gamma=0.01, kernel=\"poly\", n_components=8),\n", - " ExtraTreesClassifier(criterion=\"entropy\", max_features=1.0, n_estimators=500)\n", + " RBFSampler(gamma=0.8500000000000001),\n", + " DecisionTreeClassifier(criterion=\"entropy\", max_depth=3, min_samples_leaf=4, min_samples_split=9)\n", ")\n", "\n", "exported_pipeline.fit(training_features, training_classes)\n", @@ -203,7 +428,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -226,7 +453,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.3" } }, "nbformat": 4, diff --git a/tutorials/MNIST.ipynb b/tutorials/MNIST.ipynb index 58431993..a02bfef7 100644 --- a/tutorials/MNIST.ipynb +++ b/tutorials/MNIST.ipynb @@ -2,9 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -17,7 +19,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -42,78 +46,139 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Warning: xgboost.XGBClassifier is not available and will not be used by TPOT.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 80/80 [01:08<00:00, 1.39pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 1 - Current best internal CV score: 0.9628970273793229\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 120/120 [01:34<00:00, 1.56pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 2 - Current best internal CV score: 0.9829041339667848\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 160/160 [02:13<00:00, 1.26pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 3 - Current best internal CV score: 0.9829041339667848\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "Optimization Progress: 51%|█████▏ | 41/80 [01:14<00:45, 1.16s/pipeline]" + "Optimization Progress: 100%|██████████| 200/200 [02:39<00:00, 1.42pipeline/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation 1 - Current best internal CV score: 0.9891007793284805\n" + "Generation 4 - Current best internal CV score: 0.9829041339667848\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Optimization Progress: 68%|██████▊ | 81/120 [02:08<00:33, 1.18pipeline/s]" + "Optimization Progress: 100%|██████████| 240/240 [03:37<00:00, 3.76s/pipeline]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation 2 - Current best internal CV score: 0.9898883509262202\n" + "Generation 5 - Current best internal CV score: 0.9829288814415318\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Optimization Progress: 96%|█████████▌| 115/120 [00:00<00:02, 2.04pipeline/s]" + "Optimization Progress: 100%|██████████| 280/280 [04:55<00:00, 1.22s/pipeline]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation 3 - Current best internal CV score: 0.9898883509262202\n" + "Generation 6 - Current best internal CV score: 0.9851265982875169\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Optimization Progress: 99%|█████████▉| 158/160 [00:00<00:01, 1.48pipeline/s]" + "Optimization Progress: 100%|██████████| 320/320 [06:34<00:00, 3.85s/pipeline]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation 4 - Current best internal CV score: 0.9898883509262202\n" + "Generation 7 - Current best internal CV score: 0.9851265982875169\n" ] }, { "name": "stderr", "output_type": "stream", - "text": [] + "text": [ + " 8, 3.01s/pipeline]" + ] }, { "name": "stdout", "output_type": "stream", "text": [ - "GP closed prematurely - will use current best pipeline\n", "\n", - "Best pipeline: KNeighborsClassifier(CombineDFs(XGBClassifier(Binarizer(input_matrix, 0.62), 15, 80, 20.0, 15.0), ZeroCount(RFE(input_matrix, 28.0))), 90, 9)\n", - "0.993419911157\n" + "TPOT closed prematurely. Will use the current best pipeline.\n", + "\n", + "Best pipeline: KNeighborsClassifier(input_matrix, KNeighborsClassifier__n_neighbors=4, KNeighborsClassifier__p=2, KNeighborsClassifier__weights=distance)\n", + "0.984444444444\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" ] } ], @@ -127,7 +192,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -138,22 +205,17 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "# %load tpot_mnist_pipeline.py\n", "import numpy as np\n", "\n", - "from sklearn.ensemble import VotingClassifier\n", - "from sklearn.feature_selection import RFE\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.neighbors import KNeighborsClassifier\n", - "from sklearn.pipeline import make_pipeline, make_union\n", - "from sklearn.preprocessing import Binarizer, FunctionTransformer\n", - "from sklearn.svm import SVC\n", - "from tpot.operators.preprocessors import ZeroCount\n", - "from xgboost import XGBClassifier\n", "\n", "# NOTE: Make sure that the class is labeled 'class' in the data file\n", "tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\n", @@ -161,24 +223,7 @@ "training_features, testing_features, training_classes, testing_classes = \\\n", " train_test_split(features, tpot_data['class'], random_state=42)\n", "\n", - "exported_pipeline = make_pipeline(\n", - " make_union(\n", - " make_union(VotingClassifier([('branch',\n", - " make_pipeline(\n", - " Binarizer(threshold=0.62),\n", - " XGBClassifier(learning_rate=1.0, max_depth=10, min_child_weight=20, n_estimators=500, subsample=1.0)\n", - " )\n", - " )]), FunctionTransformer(lambda X: X)),\n", - " make_pipeline(\n", - " RFE(estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n", - " decision_function_shape=None, degree=3, gamma='auto', kernel='linear',\n", - " max_iter=-1, probability=False, random_state=42, shrinking=True,\n", - " tol=0.001, verbose=False), step=0.99),\n", - " ZeroCount()\n", - " )\n", - " ),\n", - " KNeighborsClassifier(n_neighbors=5, weights=\"distance\")\n", - ")\n", + "exported_pipeline = KNeighborsClassifier(n_neighbors=4, p=2, weights=\"distance\")\n", "\n", "exported_pipeline.fit(training_features, training_classes)\n", "results = exported_pipeline.predict(testing_features)\n" @@ -188,7 +233,9 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [] @@ -211,7 +258,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.3" } }, "nbformat": 4, diff --git a/tutorials/Titanic_Kaggle.ipynb b/tutorials/Titanic_Kaggle.ipynb index 5a94b912..2a418e76 100644 --- a/tutorials/Titanic_Kaggle.ipynb +++ b/tutorials/Titanic_Kaggle.ipynb @@ -2,23 +2,31 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "# TPOT tutorial on the Titanic dataset " ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The Titanic machine learning competition on [Kaggle](https://www.kaggle.com/c/titanic) is one of the most popular beginner's competitions on the platform. We will use that competition here to demonstrate the implementation of TPOT. " ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -33,7 +41,9 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -174,7 +184,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Data Exploration " ] @@ -183,7 +196,9 @@ "cell_type": "code", "execution_count": 3, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -210,7 +225,9 @@ "cell_type": "code", "execution_count": 4, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -245,7 +262,9 @@ "cell_type": "code", "execution_count": 5, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -328,14 +347,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Data Munging " ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The first and most important step in using TPOT on any data set is to rename the target class/response variable to `class`." ] @@ -344,7 +369,9 @@ "cell_type": "code", "execution_count": 6, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -353,7 +380,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "At present, TPOT requires all the data to be in numerical format. As we can see below, our data set has 5 categorical variables which contain non-numerical values: `Name`, `Sex`, `Ticket`, `Cabin` and `Embarked`." ] @@ -362,7 +392,9 @@ "cell_type": "code", "execution_count": 7, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -394,7 +426,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "We then check the number of levels that each of the five categorical variables have. " ] @@ -403,7 +438,9 @@ "cell_type": "code", "execution_count": 8, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -425,7 +462,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "As we can see, `Sex` and `Embarked` have few levels. Let's find out what they are." ] @@ -434,7 +474,9 @@ "cell_type": "code", "execution_count": 9, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -453,7 +495,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "We then code these levels manually into numerical values. For `nan` i.e. the missing values, we simply replace them with a placeholder value (-999). In fact, we perform this replacement for the entire data set." ] @@ -462,7 +507,9 @@ "cell_type": "code", "execution_count": 10, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -474,7 +521,9 @@ "cell_type": "code", "execution_count": 11, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -507,7 +556,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Since `Name` and `Ticket` have so many levels, we drop them from our analysis for the sake of simplicity. For `Cabin`, we encode the levels as digits using Scikit-learn's [`MultiLabelBinarizer`](http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MultiLabelBinarizer.html) and treat them as new features. " ] @@ -516,7 +568,9 @@ "cell_type": "code", "execution_count": 12, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -529,7 +583,9 @@ "cell_type": "code", "execution_count": 13, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -555,7 +611,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Drop the unused features from the dataset. " ] @@ -564,7 +623,9 @@ "cell_type": "code", "execution_count": 14, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -575,7 +636,9 @@ "cell_type": "code", "execution_count": 15, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -584,7 +647,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "We then add the encoded features to form the final dataset to be used with TPOT. " ] @@ -593,7 +659,9 @@ "cell_type": "code", "execution_count": 16, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -604,7 +672,9 @@ "cell_type": "code", "execution_count": 17, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -624,7 +694,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Keeping in mind that the final dataset is in the form of a numpy array, we can check the number of features in the final dataset as follows." ] @@ -633,7 +706,9 @@ "cell_type": "code", "execution_count": 18, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -653,7 +728,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Finally we store the class labels, which we need to predict, in a separate variable. " ] @@ -662,7 +740,9 @@ "cell_type": "code", "execution_count": 19, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -671,14 +751,20 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "## Data Analysis using TPOT" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "To begin our analysis, we need to divide our training data into training and validation sets. The validation set is just to give us an idea of the test set error. The model selection and tuning is entirely taken care of by TPOT, so if we want to, we can skip creating this validation set." ] @@ -687,7 +773,9 @@ "cell_type": "code", "execution_count": 20, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -708,7 +796,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "After that, we proceed to calling the `fit`, `score` and `export` functions on our training dataset. To get a better idea of how these functions work, refer the TPOT documentation [here](http://rhiever.github.io/tpot/using/).\n", "\n", @@ -719,40 +810,199 @@ "cell_type": "code", "execution_count": 21, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Warning: xgboost.XGBClassifier is not available and will not be used by TPOT.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 80/80 [00:12<00:00, 6.43pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 1 - Current best internal CV score: 0.8203972618112445\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 120/120 [00:19<00:00, 5.50pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 2 - Current best internal CV score: 0.8203972618112445\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 160/160 [00:32<00:00, 2.99pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 3 - Current best internal CV score: 0.8203972618112445\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 200/200 [00:43<00:00, 6.08pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 4 - Current best internal CV score: 0.8203972618112445\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 240/240 [00:52<00:00, 4.20pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 5 - Current best internal CV score: 0.8203972618112445\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 280/280 [01:03<00:00, 3.62pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 6 - Current best internal CV score: 0.8219341887055946\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 320/320 [01:15<00:00, 3.21pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 7 - Current best internal CV score: 0.8249418737556994\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 360/360 [01:26<00:00, 3.95pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 8 - Current best internal CV score: 0.8249418737556994\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 400/400 [01:38<00:00, 4.84pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 9 - Current best internal CV score: 0.8249418737556994\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimization Progress: 100%|██████████| 440/440 [01:48<00:00, 3.66pipeline/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation 10 - Current best internal CV score: 0.8249418737556994\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "Optimization Progress: 100%|██████████| 100/100 [00:00<00:00, 1.11s/pipeline]" + "Optimization Progress: 100%|██████████| 480/480 [02:51<00:00, 2.06s/pipeline]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation 1 - Current best internal CV score: 0.7952653536499413\n" + "Generation 11 - Current best internal CV score: 0.8249418737556994\n" ] }, { "name": "stderr", "output_type": "stream", - "text": [] + "text": [ + " 9, 2.05s/pipeline]" + ] }, { "name": "stdout", "output_type": "stream", "text": [ - "GP closed prematurely - will use current best pipeline\n", "\n", - "Best pipeline: RandomForestClassifier(input_matrix)\n" + "TPOT closed prematurely. Will use the current best pipeline.\n", + "\n", + "Best pipeline: RandomForestClassifier(input_matrix, RandomForestClassifier__bootstrap=False, RandomForestClassifier__criterion=DEFAULT, RandomForestClassifier__max_features=0.4, RandomForestClassifier__min_samples_leaf=1, RandomForestClassifier__min_samples_split=9)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" ] } ], "source": [ - "tpot = TPOTClassifier(verbosity=2, max_time_mins=2)\n", + "tpot = TPOTClassifier(verbosity=2, max_time_mins=2, max_eval_time_mins=0.04, population_size=40)\n", "tpot.fit(titanic_new[training_indices], titanic_class[training_indices])" ] }, @@ -760,13 +1010,15 @@ "cell_type": "code", "execution_count": 22, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { "data": { "text/plain": [ - "0.78679341368188771" + "0.8340807174887892" ] }, "execution_count": 22, @@ -782,7 +1034,9 @@ "cell_type": "code", "execution_count": 23, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -791,7 +1045,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "Let's have a look at the generated code. As we can see, the random forest classifier performed the best on the given dataset out of all the other models that TPOT currently evaluates on. If we ran TPOT for more generations, then the score should improve further." ] @@ -800,17 +1057,17 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ "# %load tpot_titanic_pipeline.py\n", "import numpy as np\n", "\n", - "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n", + "from sklearn.ensemble import RandomForestClassifier\n", "from sklearn.model_selection import train_test_split\n", - "from sklearn.pipeline import make_pipeline, make_union\n", - "from sklearn.preprocessing import FunctionTransformer\n", "\n", "# NOTE: Make sure that the class is labeled 'class' in the data file\n", "tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\n", @@ -818,9 +1075,7 @@ "training_features, testing_features, training_classes, testing_classes = \\\n", " train_test_split(features, tpot_data['class'], random_state=42)\n", "\n", - "exported_pipeline = make_pipeline(\n", - " RandomForestClassifier(n_estimators=500)\n", - ")\n", + "exported_pipeline = RandomForestClassifier(bootstrap=False, max_features=0.4, min_samples_leaf=1, min_samples_split=9)\n", "\n", "exported_pipeline.fit(training_features, training_classes)\n", "results = exported_pipeline.predict(testing_features)\n" @@ -829,7 +1084,9 @@ { "cell_type": "markdown", "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "source": [ "### Make predictions on the submission data " @@ -839,17 +1096,11 @@ "cell_type": "code", "execution_count": 25, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/olsonran/anaconda/lib/python3.5/site-packages/numpy/lib/function_base.py:3834: RuntimeWarning: Invalid value encountered in percentile\n", - " RuntimeWarning)\n" - ] - }, { "data": { "text/html": [ @@ -907,28 +1158,28 @@ " 25%\n", " 996.250000\n", " 1.000000\n", - " NaN\n", + " 21.000000\n", " 0.000000\n", " 0.000000\n", - " NaN\n", + " 7.895800\n", " \n", " \n", " 50%\n", " 1100.500000\n", " 3.000000\n", - " NaN\n", + " 27.000000\n", " 0.000000\n", " 0.000000\n", - " NaN\n", + " 14.454200\n", " \n", " \n", " 75%\n", " 1204.750000\n", " 3.000000\n", - " NaN\n", + " 39.000000\n", " 1.000000\n", " 0.000000\n", - " NaN\n", + " 31.500000\n", " \n", " \n", " max\n", @@ -949,9 +1200,9 @@ "mean 1100.500000 2.265550 30.272590 0.447368 0.392344 35.627188\n", "std 120.810458 0.841838 14.181209 0.896760 0.981429 55.907576\n", "min 892.000000 1.000000 0.170000 0.000000 0.000000 0.000000\n", - "25% 996.250000 1.000000 NaN 0.000000 0.000000 NaN\n", - "50% 1100.500000 3.000000 NaN 0.000000 0.000000 NaN\n", - "75% 1204.750000 3.000000 NaN 1.000000 0.000000 NaN\n", + "25% 996.250000 1.000000 21.000000 0.000000 0.000000 7.895800\n", + "50% 1100.500000 3.000000 27.000000 0.000000 0.000000 14.454200\n", + "75% 1204.750000 3.000000 39.000000 1.000000 0.000000 31.500000\n", "max 1309.000000 3.000000 76.000000 8.000000 9.000000 512.329200" ] }, @@ -968,7 +1219,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "The most important step here is to check for new levels in the categorical variables of the submission dataset that are absent in the training set. We identify them and set them to our placeholder value of '-999', i.e., we treat them as missing values. This ensures training consistency, as otherwise the model does not know what to do with the new levels in the submission dataset. " ] @@ -977,7 +1231,9 @@ "cell_type": "code", "execution_count": 26, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -988,7 +1244,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "We then carry out the data munging steps as done earlier for the training dataset." ] @@ -997,7 +1256,9 @@ "cell_type": "code", "execution_count": 27, "metadata": { - "collapsed": true + "collapsed": true, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1009,7 +1270,9 @@ "cell_type": "code", "execution_count": 28, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -1041,7 +1304,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "While calling `MultiLabelBinarizer` for the submission data set, we first fit on the training set again to learn the levels and then transform the submission dataset values. This further ensures that only those levels that were present in the training dataset are transformed. If new levels are still found in the submission dataset then it will return an error and we need to go back and check our earlier step of replacing new levels with the placeholder value. " ] @@ -1050,7 +1316,9 @@ "cell_type": "code", "execution_count": 29, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1064,7 +1332,9 @@ "cell_type": "code", "execution_count": 30, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1076,7 +1346,9 @@ "cell_type": "code", "execution_count": 31, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -1098,7 +1370,9 @@ "cell_type": "code", "execution_count": 32, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1110,7 +1384,9 @@ "cell_type": "code", "execution_count": 33, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1122,7 +1398,9 @@ "cell_type": "code", "execution_count": 34, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [], "source": [ @@ -1135,7 +1413,9 @@ "cell_type": "code", "execution_count": 35, "metadata": { - "collapsed": false + "collapsed": false, + "deletable": true, + "editable": true }, "outputs": [ { @@ -1155,7 +1435,10 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "deletable": true, + "editable": true + }, "source": [ "There we go! We have successfully generated the predictions for the 418 data points in the submission dataset, and we're good to go ahead to submit these predictions on Kaggle. " ] @@ -1178,7 +1461,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.5.3" } }, "nbformat": 4, diff --git a/tutorials/data/submission.csv b/tutorials/data/submission.csv index 29ac81da..64dffda2 100644 --- a/tutorials/data/submission.csv +++ b/tutorials/data/submission.csv @@ -2,8 +2,8 @@ PassengerId,Survived 892,0 893,0 894,0 -895,0 -896,0 +895,1 +896,1 897,0 898,0 899,0 @@ -17,43 +17,43 @@ PassengerId,Survived 907,1 908,0 909,0 -910,0 +910,1 911,0 912,0 913,1 914,1 -915,1 +915,0 916,1 917,0 918,1 919,0 920,0 921,0 -922,0 +922,1 923,0 924,0 -925,0 +925,1 926,1 -927,1 +927,0 928,0 -929,0 +929,1 930,0 931,0 932,0 -933,0 +933,1 934,0 935,1 936,1 937,0 938,0 -939,1 +939,0 940,1 941,1 942,0 943,0 944,1 945,1 -946,0 +946,1 947,0 948,0 949,0 @@ -62,34 +62,34 @@ PassengerId,Survived 952,0 953,0 954,0 -955,0 +955,1 956,1 957,1 958,1 959,0 -960,1 +960,0 961,1 -962,0 +962,1 963,0 964,0 -965,1 +965,0 966,1 -967,1 +967,0 968,0 969,1 970,0 -971,0 +971,1 972,1 973,0 974,0 975,0 976,0 977,0 -978,0 -979,0 +978,1 +979,1 980,1 981,1 -982,0 +982,1 983,0 984,1 985,0 @@ -97,33 +97,33 @@ PassengerId,Survived 987,0 988,1 989,0 -990,0 +990,1 991,0 992,1 993,0 -994,1 +994,0 995,0 996,1 997,0 998,0 -999,1 +999,0 1000,0 1001,0 1002,0 1003,1 1004,1 -1005,0 +1005,1 1006,1 1007,0 1008,0 1009,1 -1010,1 +1010,0 1011,1 1012,1 1013,0 1014,1 1015,0 -1016,1 +1016,0 1017,1 1018,0 1019,1 @@ -131,13 +131,13 @@ PassengerId,Survived 1021,0 1022,0 1023,0 -1024,1 +1024,0 1025,0 1026,0 1027,0 1028,0 1029,0 -1030,0 +1030,1 1031,0 1032,0 1033,1 @@ -145,9 +145,9 @@ PassengerId,Survived 1035,0 1036,0 1037,0 -1038,0 +1038,1 1039,0 -1040,0 +1040,1 1041,0 1042,1 1043,0 @@ -156,7 +156,7 @@ PassengerId,Survived 1046,0 1047,0 1048,1 -1049,0 +1049,1 1050,0 1051,1 1052,1 @@ -168,7 +168,7 @@ PassengerId,Survived 1058,0 1059,0 1060,1 -1061,0 +1061,1 1062,0 1063,0 1064,0 @@ -176,35 +176,35 @@ PassengerId,Survived 1066,0 1067,1 1068,1 -1069,0 +1069,1 1070,1 1071,1 1072,0 -1073,1 +1073,0 1074,1 -1075,1 +1075,0 1076,1 -1077,0 +1077,1 1078,1 1079,0 1080,0 1081,0 -1082,0 -1083,0 +1082,1 +1083,1 1084,1 1085,0 1086,1 1087,0 1088,1 -1089,0 +1089,1 1090,0 1091,0 1092,1 1093,1 -1094,1 +1094,0 1095,1 1096,0 -1097,0 +1097,1 1098,0 1099,0 1100,1 @@ -212,7 +212,7 @@ PassengerId,Survived 1102,0 1103,0 1104,0 -1105,0 +1105,1 1106,0 1107,0 1108,1 @@ -224,7 +224,7 @@ PassengerId,Survived 1114,1 1115,0 1116,1 -1117,1 +1117,0 1118,0 1119,1 1120,0 @@ -235,15 +235,15 @@ PassengerId,Survived 1125,0 1126,1 1127,0 -1128,1 +1128,0 1129,0 1130,1 1131,1 1132,1 1133,1 -1134,1 +1134,0 1135,0 -1136,0 +1136,1 1137,1 1138,1 1139,0 @@ -251,11 +251,11 @@ PassengerId,Survived 1141,1 1142,1 1143,0 -1144,1 +1144,0 1145,0 1146,0 1147,0 -1148,1 +1148,0 1149,0 1150,1 1151,0 @@ -270,19 +270,19 @@ PassengerId,Survived 1160,0 1161,0 1162,0 -1163,1 +1163,0 1164,1 1165,1 1166,0 1167,1 1168,0 -1169,0 +1169,1 1170,0 1171,0 -1172,0 +1172,1 1173,1 1174,1 -1175,0 +1175,1 1176,1 1177,0 1178,0 @@ -321,7 +321,7 @@ PassengerId,Survived 1211,0 1212,0 1213,0 -1214,0 +1214,1 1215,0 1216,1 1217,0 @@ -330,9 +330,9 @@ PassengerId,Survived 1220,0 1221,0 1222,1 -1223,1 +1223,0 1224,0 -1225,0 +1225,1 1226,0 1227,1 1228,0 @@ -346,7 +346,7 @@ PassengerId,Survived 1236,1 1237,0 1238,0 -1239,1 +1239,0 1240,0 1241,1 1242,1 @@ -357,12 +357,12 @@ PassengerId,Survived 1247,0 1248,1 1249,0 -1250,1 +1250,0 1251,0 1252,0 1253,1 1254,1 -1255,0 +1255,1 1256,1 1257,0 1258,0 @@ -375,11 +375,11 @@ PassengerId,Survived 1265,0 1266,1 1267,1 -1268,0 +1268,1 1269,0 1270,0 1271,0 -1272,1 +1272,0 1273,0 1274,1 1275,1 @@ -391,7 +391,7 @@ PassengerId,Survived 1281,0 1282,0 1283,1 -1284,1 +1284,0 1285,0 1286,0 1287,1 @@ -406,7 +406,7 @@ PassengerId,Survived 1296,0 1297,0 1298,0 -1299,1 +1299,0 1300,1 1301,1 1302,1 diff --git a/tutorials/tpot_iris_pipeline.py b/tutorials/tpot_iris_pipeline.py index 4b13b671..50815b3c 100644 --- a/tutorials/tpot_iris_pipeline.py +++ b/tutorials/tpot_iris_pipeline.py @@ -1,10 +1,9 @@ import numpy as np -from sklearn.ensemble import ExtraTreesClassifier, VotingClassifier -from sklearn.kernel_approximation import Nystroem +from sklearn.kernel_approximation import RBFSampler from sklearn.model_selection import train_test_split -from sklearn.pipeline import make_pipeline, make_union -from sklearn.preprocessing import FunctionTransformer +from sklearn.pipeline import make_pipeline +from sklearn.tree import DecisionTreeClassifier # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -13,8 +12,8 @@ train_test_split(features, tpot_data['class'], random_state=42) exported_pipeline = make_pipeline( - Nystroem(gamma=0.01, kernel="poly", n_components=8), - ExtraTreesClassifier(criterion="entropy", max_features=1.0, n_estimators=500) + RBFSampler(gamma=0.8500000000000001), + DecisionTreeClassifier(criterion="entropy", max_depth=3, min_samples_leaf=4, min_samples_split=9) ) exported_pipeline.fit(training_features, training_classes) diff --git a/tutorials/tpot_mnist_pipeline.py b/tutorials/tpot_mnist_pipeline.py index 3ea0ab99..72c07ab8 100644 --- a/tutorials/tpot_mnist_pipeline.py +++ b/tutorials/tpot_mnist_pipeline.py @@ -1,14 +1,7 @@ import numpy as np -from sklearn.ensemble import VotingClassifier -from sklearn.feature_selection import RFE from sklearn.model_selection import train_test_split from sklearn.neighbors import KNeighborsClassifier -from sklearn.pipeline import make_pipeline, make_union -from sklearn.preprocessing import Binarizer, FunctionTransformer -from sklearn.svm import SVC -from tpot.operators.preprocessors import ZeroCount -from xgboost import XGBClassifier # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -16,24 +9,7 @@ training_features, testing_features, training_classes, testing_classes = \ train_test_split(features, tpot_data['class'], random_state=42) -exported_pipeline = make_pipeline( - make_union( - make_union(VotingClassifier([('branch', - make_pipeline( - Binarizer(threshold=0.62), - XGBClassifier(learning_rate=1.0, max_depth=10, min_child_weight=20, n_estimators=500, subsample=1.0) - ) - )]), FunctionTransformer(lambda X: X)), - make_pipeline( - RFE(estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0, - decision_function_shape=None, degree=3, gamma='auto', kernel='linear', - max_iter=-1, probability=False, random_state=42, shrinking=True, - tol=0.001, verbose=False), step=0.99), - ZeroCount() - ) - ), - KNeighborsClassifier(n_neighbors=5, weights="distance") -) +exported_pipeline = KNeighborsClassifier(n_neighbors=4, p=2, weights="distance") exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) diff --git a/tutorials/tpot_titanic_pipeline.py b/tutorials/tpot_titanic_pipeline.py index 09085eac..57bf5130 100644 --- a/tutorials/tpot_titanic_pipeline.py +++ b/tutorials/tpot_titanic_pipeline.py @@ -1,9 +1,7 @@ import numpy as np -from sklearn.ensemble import RandomForestClassifier, VotingClassifier +from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split -from sklearn.pipeline import make_pipeline, make_union -from sklearn.preprocessing import FunctionTransformer # NOTE: Make sure that the class is labeled 'class' in the data file tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64) @@ -11,9 +9,7 @@ training_features, testing_features, training_classes, testing_classes = \ train_test_split(features, tpot_data['class'], random_state=42) -exported_pipeline = make_pipeline( - RandomForestClassifier(n_estimators=500) -) +exported_pipeline = RandomForestClassifier(bootstrap=False, max_features=0.4, min_samples_leaf=1, min_samples_split=9) exported_pipeline.fit(training_features, training_classes) results = exported_pipeline.predict(testing_features) From 03e7c035f5259b87db0b45cdd61f976584d3b995 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Tue, 21 Mar 2017 15:55:55 -0400 Subject: [PATCH 140/154] Update the docs with the latest API changes --- docs/index.html | 4 +- docs/installing/index.html | 4 + docs/mkdocs/search_index.json | 15 ++-- docs/releases/index.html | 38 +++++++++- docs/sitemap.xml | 22 +++--- docs/using/index.html | 139 +++++++++++++++++++++++----------- docs_sources/installing.md | 2 +- docs_sources/using.md | 103 +++++++++++++------------ tpot/base.py | 8 +- tpot/driver.py | 10 ++- 10 files changed, 222 insertions(+), 123 deletions(-) diff --git a/docs/index.html b/docs/index.html index cc25acda..ab353276 100644 --- a/docs/index.html +++ b/docs/index.html @@ -244,6 +244,6 @@ diff --git a/docs/installing/index.html b/docs/installing/index.html index 49cad5b9..febd3e78 100644 --- a/docs/installing/index.html +++ b/docs/installing/index.html @@ -200,6 +200,10 @@ 
pip install deap update_checker tqdm
+

For the Windows OS, the pywin32 module is required if the Python is NOT installed via Anaconda Python distribution and can be installed with pip via the command:

+
pip install pywin32
+
+

Optionally, install XGBoost if you would like TPOT to use XGBoost. XGBoost is entirely optional, and TPOT will still function normally without XGBoost if you do not have it installed.

pip install xgboost
diff --git a/docs/mkdocs/search_index.json b/docs/mkdocs/search_index.json index fc742ba0..45e3b3cf 100644 --- a/docs/mkdocs/search_index.json +++ b/docs/mkdocs/search_index.json @@ -7,22 +7,22 @@ }, { "location": "/installing/", - "text": "TPOT is built on top of several existing Python libraries, including:\n\n\n\n\n\n\nNumPy\n\n\n\n\n\n\nSciPy\n\n\n\n\n\n\nscikit-learn\n\n\n\n\n\n\nDEAP\n\n\n\n\n\n\nupdate_checker\n\n\n\n\n\n\ntqdm\n\n\n\n\n\n\nMost of the necessary Python packages can be installed via the \nAnaconda Python distribution\n, which we strongly recommend that you use. We also strongly recommend that you use of Python 3 over Python 2 if you're given the choice.\n\n\nNumPy, SciPy, and scikit-learn can be installed in Anaconda via the command:\n\n\nconda install numpy scipy scikit-learn\n\n\n\n\nDEAP, update_checker, and tqdm (used for verbose TPOT runs) can be installed with \npip\n via the command:\n\n\npip install deap update_checker tqdm\n\n\n\n\nOptionally, install XGBoost if you would like TPOT to use XGBoost. XGBoost is entirely optional, and TPOT will still function normally without XGBoost if you do not have it installed.\n\n\npip install xgboost\n\n\n\n\nIf you have issues installing XGBoost, check the \nXGBoost installation documentation\n.\n\n\nFinally to install TPOT itself, run the following command:\n\n\npip install tpot\n\n\n\n\nPlease \nfile a new issue\n if you run into installation problems.", + "text": "TPOT is built on top of several existing Python libraries, including:\n\n\n\n\n\n\nNumPy\n\n\n\n\n\n\nSciPy\n\n\n\n\n\n\nscikit-learn\n\n\n\n\n\n\nDEAP\n\n\n\n\n\n\nupdate_checker\n\n\n\n\n\n\ntqdm\n\n\n\n\n\n\nMost of the necessary Python packages can be installed via the \nAnaconda Python distribution\n, which we strongly recommend that you use. We also strongly recommend that you use of Python 3 over Python 2 if you're given the choice.\n\n\nNumPy, SciPy, and scikit-learn can be installed in Anaconda via the command:\n\n\nconda install numpy scipy scikit-learn\n\n\n\n\nDEAP, update_checker, and tqdm (used for verbose TPOT runs) can be installed with \npip\n via the command:\n\n\npip install deap update_checker tqdm\n\n\n\n\nFor the Windows OS\n, the pywin32 module is required if the Python is NOT installed via \nAnaconda Python distribution\n and can be installed with \npip\n via the command:\n\n\npip install pywin32\n\n\n\n\nOptionally, install XGBoost if you would like TPOT to use XGBoost. XGBoost is entirely optional, and TPOT will still function normally without XGBoost if you do not have it installed.\n\n\npip install xgboost\n\n\n\n\nIf you have issues installing XGBoost, check the \nXGBoost installation documentation\n.\n\n\nFinally to install TPOT itself, run the following command:\n\n\npip install tpot\n\n\n\n\nPlease \nfile a new issue\n if you run into installation problems.", "title": "Installation" }, { "location": "/using/", - "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nThe number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nThe number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. This setting will override the GENERATIONS parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngeneration\n\n\nAny positive integer\n\n\nThe number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nThe number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nThe mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nThe crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We don't recommend that you tweak this parameter unless you know what you're doing.\n\n\n\n\n\n\nnum_cv_folds\n\n\n[2, 10]\n\n\nThe number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nThe number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. This setting will override the generations parameter.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nThe random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit().\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether TPOT will reuse models from previous calls\nto fit() for faster operation\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)", + "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like:\n\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\n\n\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example:\n\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)", "title": "Using TPOT" }, { "location": "/using/#tpot-on-the-command-line", - "text": "To use TPOT via the command line, enter the following command with a path to the data file: tpot /path_to/data_file.csv TPOT offers several arguments that can be provided at the command line: Argument Parameter Valid values Effect -is INPUT_SEPARATOR Any string Character used to separate columns in the input file. -target TARGET_NAME Any string Name of the target column in the input file. -mode TPOT_MODE ['classification', 'regression'] Whether TPOT is being used for a classification or regression problem. -o OUTPUT_FILE String path to a file File to export the code for the final optimized pipeline. -g GENERATIONS Any positive integer Number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. -p POPULATION_SIZE Any positive integer Number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. -mr MUTATION_RATE [0.0, 1.0] GP mutation rate. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. -xr CROSSOVER_RATE [0.0, 1.0] GP crossover rate in the range [0.0, 1.0]. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms. -cv NUM_CV_FOLDS Any integer >1 The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. -njobs NUM_JOBS Any positive integer or -1 The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation. -scoring SCORING_FN 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. -maxtime MAX_TIME_MINS Any positive integer How many minutes TPOT has to optimize the pipeline. This setting will override the GENERATIONS parameter and allow TPOT to run until it runs out of time. -maxeval MAX_EVAL_MINS Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. -s RANDOM_STATE Any positive integer Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future. -v VERBOSITY {0, 1, 2, 3} How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. --no-update-check N/A Flag indicating whether the TPOT version checker should be disabled. --version N/A Show TPOT's version number and exit. --help N/A Show TPOT's help documentation and exit. An example command-line call to TPOT may look like: tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2", + "text": "To use TPOT via the command line, enter the following command with a path to the data file: tpot /path_to/data_file.csv TPOT offers several arguments that can be provided at the command line: Argument Parameter Valid values Effect -is INPUT_SEPARATOR Any string Character used to separate columns in the input file. -target TARGET_NAME Any string Name of the target column in the input file. -mode TPOT_MODE ['classification', 'regression'] Whether TPOT is being used for a supervised classification or regression problem. -o OUTPUT_FILE String path to a file File to export the code for the final optimized pipeline. -g GENERATIONS Any positive integer Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. -p POPULATION_SIZE Any positive integer Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. -os OFFSPRING_SIZE Any positive integer Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE. -mr MUTATION_RATE [0.0, 1.0] GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. -xr CROSSOVER_RATE [0.0, 1.0] GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms. -scoring SCORING_FN 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. -cv NUM_CV_FOLDS Any integer >1 Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process. -njobs NUM_JOBS Any positive integer or -1 Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. -maxtime MAX_TIME_MINS Any positive integer How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time. -maxeval MAX_EVAL_MINS Any positive integer How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. -s RANDOM_STATE Any positive integer Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future. -config CONFIG_FILE String path to a file Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like: \nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n} -v VERBOSITY {0, 1, 2, 3} How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. --no-update-check N/A Flag indicating whether the TPOT version checker should be disabled. --version N/A Show TPOT's version number and exit. --help N/A Show TPOT's help documentation and exit. An example command-line call to TPOT may look like: tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2", "title": "TPOT on the command line" }, { "location": "/using/#tpot-with-code", - "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn. TPOT can be imported just like any regular Python module. To import TPOT, type: from tpot import TPOTClassifier then create an instance of TPOT as follows: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier() It's also possible to use TPOT for regression problems with the TPOTRegressor class. Other than the class name, a TPOTRegressor is used the same way as a TPOTClassifier . Note that you can pass several parameters to the TPOT instantiation call: Parameter Valid values Effect generation Any positive integer The number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. population_size Any positive integer The number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. mutation_rate [0.0, 1.0] The mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing. crossover_rate [0.0, 1.0] The crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We don't recommend that you tweak this parameter unless you know what you're doing. num_cv_folds [2, 10] The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. n_jobs Any positive integer or -1 The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation. scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. max_time_mins Any positive integer How many minutes TPOT has to optimize the pipeline. This setting will override the generations parameter. max_eval_time_mins Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. random_state Any positive integer The random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit(). disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. warm_start [True, False] Flag indicating whether TPOT will reuse models from previous calls\nto fit() for faster operation Some example code with custom TPOT parameters might look like: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2) Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes) The fit() function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score. You can then proceed to evaluate the final pipeline on the testing set with the score() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes)) Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py') Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline. Check our examples to see TPOT applied to some specific data sets.", + "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn. TPOT can be imported just like any regular Python module. To import TPOT, type: from tpot import TPOTClassifier then create an instance of TPOT as follows: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier() It's also possible to use TPOT for regression problems with the TPOTRegressor class. Other than the class name, a TPOTRegressor is used the same way as a TPOTClassifier . Note that you can pass several parameters to the TPOT instantiation call: Parameter Valid values Effect generations Any positive integer Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. population_size Any positive integer Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. offspring_size Any positive integer Number of offspring to produce in each GP generation. By default, offspring_size = population_size. mutation_rate [0.0, 1.0] Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. crossover_rate [0.0, 1.0] Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. cv Any integer >1 Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process. n_jobs Any positive integer or -1 Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. max_time_mins Any positive integer How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time. max_eval_time_mins Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer. random_state Any positive integer Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. config_dict Python dictionary Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example: \nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n} warm_start [True, False] Flag indicating whether the TPOT instance will reuse the population from previous calls to fit(). verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. Some example code with custom TPOT parameters might look like: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2) Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes) The fit() function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score. You can then proceed to evaluate the final pipeline on the testing set with the score() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes)) Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py') Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline. Check our examples to see TPOT applied to some specific data sets.", "title": "TPOT with code" }, { @@ -82,9 +82,14 @@ }, { "location": "/releases/", - "text": "Version 0.6\n\n\n\n\n\n\nTPOT now supports regression problems!\n We have created two separate \nTPOTClassifier\n and \nTPOTRegressor\n classes to support classification and regression problems, respectively. The \ncommand-line interface\n also supports this feature through the \n-mode\n parameter.\n\n\n\n\n\n\nTPOT now allows you to \nspecify a time limit\n for the optimization process with the \nmax_time_mins\n parameter, so you don't need to guess how long TPOT will take any more to recommend a pipeline to you.\n\n\n\n\n\n\nAdded a new operator that performs feature selection using \nExtraTrees\n feature importance scores.\n\n\n\n\n\n\nXGBoost\n has been added as an optional dependency to TPOT.\n If you have XGBoost installed, TPOT will automatically detect your installation and use the \nXGBoostClassifier\n and \nXGBoostRegressor\n in its pipelines.\n\n\n\n\n\n\nTPOT now offers a verbosity level of 3 (\"science mode\"), which outputs the entire Pareto front instead of only the current best score. This feature may be useful for users looking to make a trade-off between pipeline complexity and score.\n\n\n\n\n\n\nVersion 0.5\n\n\n\n\nMajor refactor: Each operator is defined in a separate class file. Hooray for easier-to-maintain code!\n\n\nTPOT now \nexports directly to scikit-learn Pipelines\n instead of hacky code.\n\n\nInternal representation of individuals now uses scikit-learn pipelines.\n\n\nParameters for each operator have been optimized so TPOT spends less time exploring useless parameters.\n\n\nWe have removed pandas as a dependency and instead use numpy matrices to store the data.\n\n\nTPOT now uses \nk-fold cross-validation\n when evaluating pipelines, with a default k = 3. This k parameter can be tuned when creating a new TPOT instance.\n\n\nImproved \nscoring function support\n: Even though TPOT uses balanced accuracy by default, you can now have TPOT use \nany of the scoring functions\n that \ncross_val_score\n supports.\n\n\nAdded the scikit-learn \nNormalizer\n preprocessor.\n\n\nMinor text fixes.\n\n\n\n\nVersion 0.4\n\n\nIn TPOT 0.4, we've made some major changes to the internals of TPOT and added some convenience functions. We've summarized the changes below.\n\n\n\n\nAdded new sklearn models and preprocessors\n\n\n\n\nAdaBoostClassifier\n\n\nBernoulliNB\n\n\nExtraTreesClassifier\n\n\nGaussianNB\n\n\nMultinomialNB\n\n\nLinearSVC\n\n\nPassiveAggressiveClassifier\n\n\nGradientBoostingClassifier\n\n\nRBFSampler\n\n\nFastICA\n\n\nFeatureAgglomeration\n\n\nNystroem\n\n\n\n\nAdded operator that inserts virtual features for the count of features with values of zero\n\n\nReworked parameterization of TPOT operators\n\n\n\nReduced parameter search space with information from a scikit-learn benchmark\n\n\nTPOT no longer generates arbitrary parameter values, but uses a fixed parameter set instead\n\n\n\n\nRemoved XGBoost as a dependency\n\n\n\nToo many users were having install issues with XGBoost\n\n\nReplaced with scikit-learn's GradientBoostingClassifier\n\n\n\n\nImproved descriptiveness of TPOT command line parameter documentation\n\n\nRemoved min/max/avg details during fit() when verbosity > 1\n\n\n\n\nReplaced with tqdm progress bar\n\n\nAdded tqdm as a dependency\n\n\n\n\nAdded \nfit_predict()\n convenience function\n\n\nAdded \nget_params()\n function so TPOT can operate in scikit-learn's \ncross_val_score\n & related functions\n\n\n\n\n\nVersion 0.3\n\n\n\n\nWe revised the internal optimization process of TPOT to make it more efficient, in particular in regards to the model parameters that TPOT optimizes over.\n\n\n\n\nVersion 0.2\n\n\n\n\n\n\nTPOT now has the ability to export the optimized pipelines to sklearn code.\n\n\n\n\n\n\nLogistic regression, SVM, and k-nearest neighbors classifiers were added as pipeline operators. Previously, TPOT only included decision tree and random forest classifiers.\n\n\n\n\n\n\nTPOT can now use arbitrary scoring functions for the optimization process.\n\n\n\n\n\n\nTPOT now performs multi-objective Pareto optimization to balance model complexity (i.e., # of pipeline operators) and the score of the pipeline.\n\n\n\n\n\n\nVersion 0.1\n\n\n\n\n\n\nFirst public release of TPOT.\n\n\n\n\n\n\nOptimizes pipelines with decision trees and random forest classifiers as the model, and uses a handful of feature preprocessors.", + "text": "Version 0.7\n\n\n\n\n\n\nTPOT now supports parallel computing for pipeline optimization (Linux and MacOS only)\n TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the \nn_jobs\n parameter in both TPOTClassifier and TPOTRegressor. The \ncommand-line interface\n also supports this feature through the \n-njobs\n parameter.\n\n\n\n\n\n\nTPOT now support customized dictionary of operators and parameters\n TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the \noperator_dict\n parameter. The format of this customized dictionary can be found in \nonline manual\n. The \ncommand-line interface\n also supports this feature through the \n-operator\n parameter but only takes a file including the dictionary instead.\n\n\n\n\n\n\nTPOT now allows you to \nspecify a time limit (default time limit is 5 minutes)\n for evaluating a single pipeline in optimization process with the \nmax_eval_time_mins\n parameter, so TPOT can skip these time-consuming pipelines.\n\n\n\n\n\n\nThe [evolutionary algorithm] is replaced by the (mu + lambda) evolutionary algorithm. TPOT allows you to set offspring size (lambda) for pipeline optimization in TPOT with a new \noffspring_size\n parameter. The \ncommand-line interface\n also supports this feature through the \n-c\n parameter.\n\n\n\n\n\n\nFixed issue about reproducing results with same random seed\n\n\n\n\n\n\nDefault operators and their parameters in TPOT were refined.\n\n\n\n\n\n\nThe TPOT point mutation operator was refined\n\n\n\n\n\n\nTPOT now supports sample weights to be used like \nTPOTRegressor.fit(x_train, y_train, sample_weights=sample_weights)\n\n\n\n\n\n\nTPOT now checks duplicated pipeline to accelerate optimization process.\n\n\n\n\n\n\nThe default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn.\n\n\n\n\n\n\nVersion 0.6\n\n\n\n\n\n\nTPOT now supports regression problems!\n We have created two separate \nTPOTClassifier\n and \nTPOTRegressor\n classes to support classification and regression problems, respectively. The \ncommand-line interface\n also supports this feature through the \n-mode\n parameter.\n\n\n\n\n\n\nTPOT now allows you to \nspecify a time limit\n for the optimization process with the \nmax_time_mins\n parameter, so you don't need to guess how long TPOT will take any more to recommend a pipeline to you.\n\n\n\n\n\n\nAdded a new operator that performs feature selection using \nExtraTrees\n feature importance scores.\n\n\n\n\n\n\nXGBoost\n has been added as an optional dependency to TPOT.\n If you have XGBoost installed, TPOT will automatically detect your installation and use the \nXGBoostClassifier\n and \nXGBoostRegressor\n in its pipelines.\n\n\n\n\n\n\nTPOT now offers a verbosity level of 3 (\"science mode\"), which outputs the entire Pareto front instead of only the current best score. This feature may be useful for users looking to make a trade-off between pipeline complexity and score.\n\n\n\n\n\n\nVersion 0.5\n\n\n\n\nMajor refactor: Each operator is defined in a separate class file. Hooray for easier-to-maintain code!\n\n\nTPOT now \nexports directly to scikit-learn Pipelines\n instead of hacky code.\n\n\nInternal representation of individuals now uses scikit-learn pipelines.\n\n\nParameters for each operator have been optimized so TPOT spends less time exploring useless parameters.\n\n\nWe have removed pandas as a dependency and instead use numpy matrices to store the data.\n\n\nTPOT now uses \nk-fold cross-validation\n when evaluating pipelines, with a default k = 3. This k parameter can be tuned when creating a new TPOT instance.\n\n\nImproved \nscoring function support\n: Even though TPOT uses balanced accuracy by default, you can now have TPOT use \nany of the scoring functions\n that \ncross_val_score\n supports.\n\n\nAdded the scikit-learn \nNormalizer\n preprocessor.\n\n\nMinor text fixes.\n\n\n\n\nVersion 0.4\n\n\nIn TPOT 0.4, we've made some major changes to the internals of TPOT and added some convenience functions. We've summarized the changes below.\n\n\n\n\nAdded new sklearn models and preprocessors\n\n\n\n\nAdaBoostClassifier\n\n\nBernoulliNB\n\n\nExtraTreesClassifier\n\n\nGaussianNB\n\n\nMultinomialNB\n\n\nLinearSVC\n\n\nPassiveAggressiveClassifier\n\n\nGradientBoostingClassifier\n\n\nRBFSampler\n\n\nFastICA\n\n\nFeatureAgglomeration\n\n\nNystroem\n\n\n\n\nAdded operator that inserts virtual features for the count of features with values of zero\n\n\nReworked parameterization of TPOT operators\n\n\n\nReduced parameter search space with information from a scikit-learn benchmark\n\n\nTPOT no longer generates arbitrary parameter values, but uses a fixed parameter set instead\n\n\n\n\nRemoved XGBoost as a dependency\n\n\n\nToo many users were having install issues with XGBoost\n\n\nReplaced with scikit-learn's GradientBoostingClassifier\n\n\n\n\nImproved descriptiveness of TPOT command line parameter documentation\n\n\nRemoved min/max/avg details during fit() when verbosity > 1\n\n\n\n\nReplaced with tqdm progress bar\n\n\nAdded tqdm as a dependency\n\n\n\n\nAdded \nfit_predict()\n convenience function\n\n\nAdded \nget_params()\n function so TPOT can operate in scikit-learn's \ncross_val_score\n & related functions\n\n\n\n\n\nVersion 0.3\n\n\n\n\nWe revised the internal optimization process of TPOT to make it more efficient, in particular in regards to the model parameters that TPOT optimizes over.\n\n\n\n\nVersion 0.2\n\n\n\n\n\n\nTPOT now has the ability to export the optimized pipelines to sklearn code.\n\n\n\n\n\n\nLogistic regression, SVM, and k-nearest neighbors classifiers were added as pipeline operators. Previously, TPOT only included decision tree and random forest classifiers.\n\n\n\n\n\n\nTPOT can now use arbitrary scoring functions for the optimization process.\n\n\n\n\n\n\nTPOT now performs multi-objective Pareto optimization to balance model complexity (i.e., # of pipeline operators) and the score of the pipeline.\n\n\n\n\n\n\nVersion 0.1\n\n\n\n\n\n\nFirst public release of TPOT.\n\n\n\n\n\n\nOptimizes pipelines with decision trees and random forest classifiers as the model, and uses a handful of feature preprocessors.", "title": "Release Notes" }, + { + "location": "/releases/#version-07", + "text": "TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only) TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the n_jobs parameter in both TPOTClassifier and TPOTRegressor. The command-line interface also supports this feature through the -njobs parameter. TPOT now support customized dictionary of operators and parameters TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the operator_dict parameter. The format of this customized dictionary can be found in online manual . The command-line interface also supports this feature through the -operator parameter but only takes a file including the dictionary instead. TPOT now allows you to specify a time limit (default time limit is 5 minutes) for evaluating a single pipeline in optimization process with the max_eval_time_mins parameter, so TPOT can skip these time-consuming pipelines. The [evolutionary algorithm] is replaced by the (mu + lambda) evolutionary algorithm. TPOT allows you to set offspring size (lambda) for pipeline optimization in TPOT with a new offspring_size parameter. The command-line interface also supports this feature through the -c parameter. Fixed issue about reproducing results with same random seed Default operators and their parameters in TPOT were refined. The TPOT point mutation operator was refined TPOT now supports sample weights to be used like TPOTRegressor.fit(x_train, y_train, sample_weights=sample_weights) TPOT now checks duplicated pipeline to accelerate optimization process. The default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn.", + "title": "Version 0.7" + }, { "location": "/releases/#version-06", "text": "TPOT now supports regression problems! We have created two separate TPOTClassifier and TPOTRegressor classes to support classification and regression problems, respectively. The command-line interface also supports this feature through the -mode parameter. TPOT now allows you to specify a time limit for the optimization process with the max_time_mins parameter, so you don't need to guess how long TPOT will take any more to recommend a pipeline to you. Added a new operator that performs feature selection using ExtraTrees feature importance scores. XGBoost has been added as an optional dependency to TPOT. If you have XGBoost installed, TPOT will automatically detect your installation and use the XGBoostClassifier and XGBoostRegressor in its pipelines. TPOT now offers a verbosity level of 3 (\"science mode\"), which outputs the entire Pareto front instead of only the current best score. This feature may be useful for users looking to make a trade-off between pipeline complexity and score.", diff --git a/docs/releases/index.html b/docs/releases/index.html index cafbe63a..5fe8ad6b 100644 --- a/docs/releases/index.html +++ b/docs/releases/index.html @@ -116,6 +116,9 @@
    +
  • Version 0.7
    • + +
  • Version 0.6
    • @@ -188,7 +191,40 @@
    -

    Version 0.6

    +

    Version 0.7

    +
      +
    • +

      TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only) TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the n_jobs parameter in both TPOTClassifier and TPOTRegressor. The command-line interface also supports this feature through the -njobs parameter.

      +
      • +
    • +

      TPOT now support customized dictionary of operators and parameters TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the operator_dict parameter. The format of this customized dictionary can be found in online manual. The command-line interface also supports this feature through the -operator parameter but only takes a file including the dictionary instead.

      +
      • +
    • +

      TPOT now allows you to specify a time limit (default time limit is 5 minutes) for evaluating a single pipeline in optimization process with the max_eval_time_mins parameter, so TPOT can skip these time-consuming pipelines.

      +
      • +
    • +

      The [evolutionary algorithm] is replaced by the (mu + lambda) evolutionary algorithm. TPOT allows you to set offspring size (lambda) for pipeline optimization in TPOT with a new offspring_size parameter. The command-line interface also supports this feature through the -c parameter.

      +
      • +
    • +

      Fixed issue about reproducing results with same random seed

      +
      • +
    • +

      Default operators and their parameters in TPOT were refined.

      +
      • +
    • +

      The TPOT point mutation operator was refined

      +
      • +
    • +

      TPOT now supports sample weights to be used like TPOTRegressor.fit(x_train, y_train, sample_weights=sample_weights)

      +
      • +
    • +

      TPOT now checks duplicated pipeline to accelerate optimization process.

      +
      • +
    • +

      The default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn.

      +
      • +
    +

    Version 0.6

    • TPOT now supports regression problems! We have created two separate TPOTClassifier and TPOTRegressor classes to support classification and regression problems, respectively. The command-line interface also supports this feature through the -mode parameter.

      diff --git a/docs/sitemap.xml b/docs/sitemap.xml index e1135dd1..f515c0f4 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -4,7 +4,7 @@ http://rhiever.github.io/tpot/ - 2017-01-17 + 2017-03-21 daily @@ -12,7 +12,7 @@ http://rhiever.github.io/tpot/installing/ - 2017-01-17 + 2017-03-21 daily @@ -20,7 +20,7 @@ http://rhiever.github.io/tpot/using/ - 2017-01-17 + 2017-03-21 daily @@ -29,25 +29,25 @@ http://rhiever.github.io/tpot/examples/MNIST_Example/ - 2017-01-17 + 2017-03-21 daily http://rhiever.github.io/tpot/examples/IRIS_Example/ - 2017-01-17 + 2017-03-21 daily http://rhiever.github.io/tpot/examples/Boston_Example/ - 2017-01-17 + 2017-03-21 daily http://rhiever.github.io/tpot/examples/Titanic_Kaggle_Example/ - 2017-01-17 + 2017-03-21 daily @@ -56,7 +56,7 @@ http://rhiever.github.io/tpot/contributing/ - 2017-01-17 + 2017-03-21 daily @@ -64,7 +64,7 @@ http://rhiever.github.io/tpot/releases/ - 2017-01-17 + 2017-03-21 daily @@ -72,7 +72,7 @@ http://rhiever.github.io/tpot/citing/ - 2017-01-17 + 2017-03-21 daily @@ -80,7 +80,7 @@ http://rhiever.github.io/tpot/support/ - 2017-01-17 + 2017-03-21 daily diff --git a/docs/using/index.html b/docs/using/index.html index 774a5692..9df6b87f 100644 --- a/docs/using/index.html +++ b/docs/using/index.html @@ -207,7 +207,7 @@

      TPOT on the command line

      -mode TPOT_MODE ['classification', 'regression'] -Whether TPOT is being used for a classification or regression problem. +Whether TPOT is being used for a supervised classification or regression problem. -o @@ -219,55 +219,61 @@

      TPOT on the command line

      -g GENERATIONS Any positive integer -Number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. +Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. -p POPULATION_SIZE Any positive integer -Number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total. +Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. + + +-os +OFFSPRING_SIZE +Any positive integer +Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE. -mr MUTATION_RATE [0.0, 1.0] -GP mutation rate. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. +GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. -xr CROSSOVER_RATE [0.0, 1.0] -GP crossover rate in the range [0.0, 1.0]. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms. +GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to "breed" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms. + + +-scoring +SCORING_FN +'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' +Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. -cv NUM_CV_FOLDS Any integer >1 -The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. +Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process. -njobs NUM_JOBS Any positive integer or -1 -The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation. - - --scoring -SCORING_FN -'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' -Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. +Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. -maxtime MAX_TIME_MINS Any positive integer -How many minutes TPOT has to optimize the pipeline. This setting will override the GENERATIONS parameter and allow TPOT to run until it runs out of time. +How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time. -maxeval MAX_EVAL_MINS Any positive integer -How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. +How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. -s @@ -276,10 +282,31 @@

      TPOT on the command line

      Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future. +-config +CONFIG_FILE +String path to a file +Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like: +
      +classifier_config_dict = {
+    'sklearn.naive_bayes.GaussianNB': {
+    },
+    'sklearn.naive_bayes.BernoulliNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    },
+    'sklearn.naive_bayes.MultinomialNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    }
+}
+
      + + + -v VERBOSITY {0, 1, 2, 3} -How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. +How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. --no-update-check @@ -323,83 +350,107 @@

      TPOT with code

      Effect -generation +generations Any positive integer -The number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. +Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. population_size Any positive integer -The number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total. +Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. + + +offspring_size +Any positive integer +Number of offspring to produce in each GP generation. By default, offspring_size = population_size. mutation_rate [0.0, 1.0] -The mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing. +Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. crossover_rate [0.0, 1.0] -The crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to "breed" every generation. We don't recommend that you tweak this parameter unless you know what you're doing. +Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to "breed" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. -num_cv_folds -[2, 10] -The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process. +scoring +'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) +Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. -n_jobs -Any positive integer or -1 -The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation. +cv +Any integer >1 +Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process. -scoring -'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) -Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. +n_jobs +Any positive integer or -1 +Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. max_time_mins Any positive integer -How many minutes TPOT has to optimize the pipeline. This setting will override the generations parameter. +How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time. max_eval_time_mins Any positive integer -How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. +How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer. random_state Any positive integer -The random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. +Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. + + +config_dict +Python dictionary +Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example: +
      +classifier_config_dict = {
+    'sklearn.naive_bayes.GaussianNB': {
+    },
+    'sklearn.naive_bayes.BernoulliNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    },
+    'sklearn.naive_bayes.MultinomialNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    }
+}
+
      + + + +warm_start +[True, False] +Flag indicating whether the TPOT instance will reuse the population from previous calls to fit(). verbosity {0, 1, 2, 3} -How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit(). +How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. - -warm_start -[True, False] -Flag indicating whether TPOT will reuse models from previous calls -to fit() for faster operation -

      Some example code with custom TPOT parameters might look like:

      from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)

      Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function:

      from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
      @@ -407,7 +458,7 @@

      TPOT with code

      You can then proceed to evaluate the final pipeline on the testing set with the score() function:

      from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 print(pipeline_optimizer.score(testing_features, testing_classes))
      @@ -415,7 +466,7 @@

      TPOT with code

      Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function:

      from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 print(pipeline_optimizer.score(testing_features, testing_classes))
 pipeline_optimizer.export('tpot_exported_pipeline.py')
diff --git a/docs_sources/installing.md b/docs_sources/installing.md
index 7f09a3dc..1da3258c 100644
--- a/docs_sources/installing.md
+++ b/docs_sources/installing.md
@@ -26,7 +26,7 @@ DEAP, update_checker, and tqdm (used for verbose TPOT runs) can be installed wit
 pip install deap update_checker tqdm
 ```
 
-**For Windows OS,**, the pywin32 module is required if the Python is NOT installed via [Anaconda Python distribution](https://www.continuum.io/downloads) and can be installed with `pip` via the command:
+**For the Windows OS**, the pywin32 module is required if the Python is NOT installed via [Anaconda Python distribution](https://www.continuum.io/downloads) and can be installed with `pip` via the command:
 
 ```Shell
 pip install pywin32
diff --git a/docs_sources/using.md b/docs_sources/using.md
index 8150728b..8d4ce204 100644
--- a/docs_sources/using.md
+++ b/docs_sources/using.md
@@ -31,7 +31,7 @@ TPOT offers several arguments that can be provided at the command line:
 -mode
 TPOT_MODE
 ['classification', 'regression']
-Whether TPOT is being used for a classification or regression problem.
+Whether TPOT is being used for a supervised classification or regression problem.
 
 
 -o
@@ -43,61 +43,61 @@ TPOT offers several arguments that can be provided at the command line:
 -g
 GENERATIONS
 Any positive integer
-Number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.
+Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 -p
 POPULATION_SIZE
 Any positive integer
-Number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate GENERATIONS x POPULATION_SIZE number of pipelines in total.
+Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
--c
+-os
 OFFSPRING_SIZE
 Any positive integer
-The number of children to produce in each generation.
+Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.
 
 
 -mr
 MUTATION_RATE
 [0.0, 1.0]
-GP mutation rate. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
+GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
 -xr
 CROSSOVER_RATE
 [0.0, 1.0]
-GP crossover rate in the range [0.0, 1.0]. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.
+GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to "breed" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.
+
+
+-scoring
+SCORING_FN
+'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
+Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
 
 
 -cv
 NUM_CV_FOLDS
 Any integer >1
-The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process.
+Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.
 
 
 -njobs
 NUM_JOBS
 Any positive integer or -1
-The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation.
-
-
--scoring
-SCORING_FN
-'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
-Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
+Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.
 
 
 -maxtime
 MAX_TIME_MINS
 Any positive integer
-How many minutes TPOT has to optimize the pipeline. This setting will override the GENERATIONS parameter and allow TPOT to run until it runs out of time.
+How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
 
 
 -maxeval
 MAX_EVAL_MINS
 Any positive integer
-How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.
+How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.
 
 
 -s
@@ -106,10 +106,10 @@ TPOT offers several arguments that can be provided at the command line:
 Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.
 
 
--operator
-OPERATOR
+-config
+CONFIG_FILE
 String path to a file
-File including a customized python dictionary to specify operators and their arguments. For example, the file's format could be like:
+Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like:
 
       classifier_config_dict = {
     'sklearn.naive_bayes.GaussianNB': {
@@ -130,7 +130,7 @@ classifier_config_dict = {
 -v
 VERBOSITY
 {0, 1, 2, 3}
-How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.
+How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.
 
 
 --no-update-check
@@ -184,64 +184,64 @@ Note that you can pass several parameters to the TPOT instantiation call:
 Effect
 
 
-generation
+generations
 Any positive integer
-The number of generations to run pipeline optimization over. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total.
+Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 population_size
 Any positive integer
-The number of individuals in the GP population. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize over. TPOT will evaluate generations x population_size number of pipelines in total.
+Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 offspring_size
 Any positive integer
-The number of children to produce in each generation.
+Number of offspring to produce in each GP generation. By default, offspring_size = population_size.
 
 
 mutation_rate
 [0.0, 1.0]
-The mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to apply random changes to every generation. We don't recommend that you tweak this parameter unless you know what you're doing.
+Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
 crossover_rate
 [0.0, 1.0]
-The crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This tells the genetic programming algorithm how many pipelines to "breed" every generation. We don't recommend that you tweak this parameter unless you know what you're doing.
+Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to "breed" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
-num_cv_folds
-[2, 10]
-The number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT pipeline optimization process.
+scoring
+'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
+Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
 
 
-n_jobs
-Any positive integer or -1
-The number of CPUs for evaluating each pipeline over cross-validation during the TPOT pipeline optimization process. Assigning this to -1 will use as many threads as possible for cross-validation.
+cv
+Any integer >1
+Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.
 
 
-scoring
-'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
-Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
+n_jobs
+Any positive integer or -1
+Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.
 
 
 max_time_mins
 Any positive integer
-How many minutes TPOT has to optimize the pipeline. This setting will override the generations parameter.
+How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
 
 
 max_eval_time_mins
 Any positive integer
-How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.
+How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.
 
 
 random_state
 Any positive integer
-The random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.
+Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.
 
 
-operator_dict
+config_dict
 Python dictionary
-A customized python dictionary to specify operators and their arguments. For example:
+Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example:
 
       classifier_config_dict = {
     'sklearn.naive_bayes.GaussianNB': {
@@ -259,21 +259,20 @@ classifier_config_dict = {
 
 
 
+warm_start
+[True, False]
+Flag indicating whether the TPOT instance will reuse the population from previous calls to fit().
+
+
 verbosity
 {0, 1, 2, 3}
-How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar to calls to fit().
+How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.
 
 
 disable_update_check
 [True, False]
 Flag indicating whether the TPOT version checker should be disabled.
 
-
-warm_start
-[True, False]
-Flag indicating whether TPOT will reuse models from previous calls
-to fit() for faster operation
-
 
 
 Some example code with custom TPOT parameters might look like:
@@ -281,7 +280,7 @@ Some example code with custom TPOT parameters might look like:
 ```Python
 from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
 ```
 
 Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the `fit` function:
@@ -289,7 +288,7 @@ Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize
 ```Python
 from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 ```
 
@@ -300,7 +299,7 @@ You can then proceed to evaluate the final pipeline on the testing set with the
 ```Python
 from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 print(pipeline_optimizer.score(testing_features, testing_classes))
 ```
@@ -310,7 +309,7 @@ Finally, you can tell TPOT to export the corresponding Python code for the optim
 ```Python
 from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, num_cv_folds=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 print(pipeline_optimizer.score(testing_features, testing_classes))
 pipeline_optimizer.export('tpot_exported_pipeline.py')
diff --git a/tpot/base.py b/tpot/base.py
index 0316c84c..90c52d4c 100644
--- a/tpot/base.py
+++ b/tpot/base.py
@@ -97,7 +97,7 @@ def __init__(self, generations=100, population_size=100, offspring_size=None,
             Mutation rate for the genetic programming algorithm in the range [0.0, 1.0].
             This parameter tells the GP algorithm how many pipelines to apply random
             changes to every generation. We recommend using the default parameter unless
-            you understand how the mutation rate affects GP algorithms
+            you understand how the mutation rate affects GP algorithms.
         crossover_rate: float (default: 0.1)
             Crossover rate for the genetic programming algorithm in the range [0.0, 1.0].
             This parameter tells the genetic programming algorithm how many pipelines to
@@ -122,10 +122,11 @@ def __init__(self, generations=100, population_size=100, offspring_size=None,
             during the TPOT optimization process.
         n_jobs: int (default: 1)
             Number of CPUs for evaluating pipelines in parallel during the TPOT
-            optimization process.
+            optimization process. Assigning this to -1 will use as many cores as available
+            on the computer.
         max_time_mins: int (default: None)
             How many minutes TPOT has to optimize the pipeline.
-            If not None, this setting will override the "generations" parameter and allow
+            If provided, this setting will override the "generations" parameter and allow
             TPOT to run until it runs out of time.
         max_eval_time_mins: int (default: 5)
             How many minutes TPOT has to optimize a single pipeline.
@@ -145,6 +146,7 @@ def __init__(self, generations=100, population_size=100, offspring_size=None,
         verbosity: int (default: 0)
             How much information TPOT communicates while it's running.
             0 = none, 1 = minimal, 2 = high, 3 = all.
+            A setting of 2 or higher will add a progress bar during the optimization procedure.
         disable_update_check: bool (default: False)
             Flag indicating whether the TPOT version checker should be disabled.
 
diff --git a/tpot/driver.py b/tpot/driver.py
index f6a475b8..3518eb21 100644
--- a/tpot/driver.py
+++ b/tpot/driver.py
@@ -93,7 +93,7 @@ def main():
         help='Whether TPOT is being used for a supervised classification or regression problem.')
 
     parser.add_argument('-o', action='store', dest='OUTPUT_FILE', default='',
-        type=str, help='File to export the final optimized pipeline.')
+        type=str, help='File to export the code for the final optimized pipeline.')
 
     parser.add_argument('-g', action='store', dest='GENERATIONS', default=100,
         type=positive_integer, help='Number of iterations to run the pipeline optimization process.\n'
@@ -140,7 +140,8 @@ def main():
 
     parser.add_argument('-njobs', action='store', dest='NUM_JOBS', default=1,
         type=int, help='Number of CPUs for evaluating pipelines in parallel '
-        ' during the TPOT optimization process.')
+        ' during the TPOT optimization process. Assigning this to -1 will use as many '
+        'cores as available on the computer.')
 
     parser.add_argument('-maxtime', action='store', dest='MAX_TIME_MINS', default=None,
         type=int, help='How many minutes TPOT has to optimize the pipeline. This '
@@ -162,8 +163,9 @@ def main():
         'that TPOT uses in the optimization process.')
 
     parser.add_argument('-v', action='store', dest='VERBOSITY', default=1,
-        choices=[0, 1, 2, 3], type=int, help='How much information TPOT  communicates '
-        'while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all.')
+        choices=[0, 1, 2, 3], type=int, help='How much information TPOT communicates '
+        'while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. '
+        'A setting of 2 or higher will add a progress bar during the optimization procedure.')
 
     parser.add_argument('--no-update-check', action='store_true',
         dest='DISABLE_UPDATE_CHECK', default=False,

From 0bb4bda520243a04590af36685e6af12f4ca87f8 Mon Sep 17 00:00:00 2001
From: Randy Olson 
Date: Tue, 21 Mar 2017 16:34:00 -0400
Subject: [PATCH 141/154] More docs cleanup for the 0.7 release

---
 docs/contributing/index.html            | 13 +++++--------
 docs/examples/Boston_Example/index.html |  2 +-
 docs/index.html                         |  2 +-
 docs/mkdocs/search_index.json           | 20 ++++++++++----------
 docs/releases/index.html                | 22 ++++++++--------------
 docs/using/index.html                   |  4 ++--
 docs_sources/contributing.md            | 12 +++++-------
 docs_sources/examples/Boston_Example.md |  2 +-
 docs_sources/releases.md                | 20 ++++++++------------
 docs_sources/using.md                   |  4 ++--
 10 files changed, 43 insertions(+), 58 deletions(-)

diff --git a/docs/contributing/index.html b/docs/contributing/index.html
index d38e5c1c..745e479d 100644
--- a/docs/contributing/index.html
+++ b/docs/contributing/index.html
@@ -191,7 +191,7 @@ 
      Project layout
      
 
      In terms of directory structure:
      
 
        
 
      • All of TPOT's code sources are in the tpot directory
        • 
-
      • The documentation sources are in the docs directory
        • 
+
      • The documentation sources are in the docs_sources directory
        • 
 
      • Images in the documentation are in the images directory
        • 
 
      • Tutorials for TPOT are in the tutorials directory
        • 
 
      • Unit tests for TPOT are in the tests.py file
        • 
@@ -209,7 +209,7 @@ 
        How to contribute
        
 
 
      • 
 
        Clone this copy to your local disk:
        
-
          $ git clone git@github.com:YourLogin/tpot.git
+
          $ git clone git@github.com:YourUsername/tpot.git
   $ cd tpot
 
        
 
        • 
@@ -285,23 +285,20 @@ 
        Before submitting your pull request
 
        Add a line to pip install the library to .travis_install.sh
        
 
 
      • 
-
        Add a line to print the version of the library to .travis_install.sh
        
+
        Add a line to print the version of the library to .travis_install.sh
        
 
        • 
 
      • 
 
        Similarly add a line to print the version of the library to .travis_test.sh
        
 
        • 
 
      
 
      Updating the documentation
      
-
      We use mkdocs to manage our documentation. This allows us to write the docs in Markdown and compile them to HTML as needed. Below are a few useful commands to know when updating the documentation. Make sure that you are running them in the base documentation directory, docs.
      
+
      We use mkdocs to manage our project documentation. This allows us to write the documentation in Markdown and compile them to HTML as needed. Below are a couple useful commands to know when updating the documentation. Make sure that you are running these commands in the base directory of the TPOT project.
      
 
        
 
      • 
 
        mkdocs serve: Hosts of a local version of the documentation that you can access at the provided URL. The local version will update automatically as you save changes to the documentation.
        
 
        • 
 
      • 
-
        mkdocs build --clean: Creates a fresh build of the documentation in HTML. Always run this before deploying the documentation to GitHub.
        
-
        • 
-
      • 
-
        mkdocs gh-deploy: Deploys the documentation to GitHub. If you're deploying on your fork of TPOT, the online documentation should be accessible at http://<YOUR GITHUB USERNAME>.github.io/tpot/. Generally, you shouldn't need to run this command because you can view your changes with mkdocs serve.
        
+
        mkdocs build --clean: Creates a fresh build of the documentation in HTML in the docs directory. Always run this before pushing the documentation to GitHub.
        
 
        • 
 
      
 
      After submitting your pull request
      
diff --git a/docs/examples/Boston_Example/index.html b/docs/examples/Boston_Example/index.html
index ff10cbd6..6e59f3a8 100644
--- a/docs/examples/Boston_Example/index.html
+++ b/docs/examples/Boston_Example/index.html
@@ -189,7 +189,7 @@
 tpot.export('tpot_boston_pipeline.py')
 
      
 
-
      Running this code should discover a pipeline that achieves about 12.77 mean squared error (MSE).
      
+
      Running this code should discover a pipeline that achieves at least 12.77 mean squared error (MSE).
      
 
      For details on how the fit(), score() and export() functions work, see the usage documentation.
      
 
      After running the above code, the corresponding Python code should be exported to the tpot_boston_pipeline.py file and look similar to the following:
      
 
      import numpy as np
diff --git a/docs/index.html b/docs/index.html
index ab353276..91ec7b8e 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -245,5 +245,5 @@
 
 
diff --git a/docs/mkdocs/search_index.json b/docs/mkdocs/search_index.json
index 45e3b3cf..3c20f11e 100644
--- a/docs/mkdocs/search_index.json
+++ b/docs/mkdocs/search_index.json
@@ -12,17 +12,17 @@
         },
         {
             "location": "/using/",
-            "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like:\n\n\nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}\n\n\n\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example:\n\n\nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n    return float(sum(y_pred == y_true)) / len(y_true)",
+            "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like:\n\n\nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}\n\n\n\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example:\n\n\nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n    return float(sum(y_pred == y_true)) / len(y_true)",
             "title": "Using TPOT"
         },
         {
             "location": "/using/#tpot-on-the-command-line",
-            "text": "To use TPOT via the command line, enter the following command with a path to the data file:  tpot /path_to/data_file.csv  TPOT offers several arguments that can be provided at the command line:    Argument  Parameter  Valid values  Effect    -is  INPUT_SEPARATOR  Any string  Character used to separate columns in the input file.    -target  TARGET_NAME  Any string  Name of the target column in the input file.    -mode  TPOT_MODE  ['classification', 'regression']  Whether TPOT is being used for a supervised classification or regression problem.    -o  OUTPUT_FILE  String path to a file  File to export the code for the final optimized pipeline.    -g  GENERATIONS  Any positive integer  Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    -p  POPULATION_SIZE  Any positive integer  Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    -os  OFFSPRING_SIZE  Any positive integer  Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.    -mr  MUTATION_RATE  [0.0, 1.0]  GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    -xr  CROSSOVER_RATE  [0.0, 1.0]  GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.    -scoring  SCORING_FN  'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'  Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on  scoring functions  for more details.    -cv  NUM_CV_FOLDS  Any integer >1  Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.    -njobs  NUM_JOBS  Any positive integer or -1  Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.    -maxtime  MAX_TIME_MINS  Any positive integer  How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.    -maxeval  MAX_EVAL_MINS  Any positive integer  How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.    -s  RANDOM_STATE  Any positive integer  Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.    -config  CONFIG_FILE  String path to a file  Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like: \nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}     -v  VERBOSITY  {0, 1, 2, 3}  How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.    --no-update-check  N/A  Flag indicating whether the TPOT version checker should be disabled.    --version  N/A  Show TPOT's version number and exit.    --help  N/A  Show TPOT's help documentation and exit.    An example command-line call to TPOT may look like:  tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2",
+            "text": "To use TPOT via the command line, enter the following command with a path to the data file:  tpot /path_to/data_file.csv  TPOT offers several arguments that can be provided at the command line:    Argument  Parameter  Valid values  Effect    -is  INPUT_SEPARATOR  Any string  Character used to separate columns in the input file.    -target  TARGET_NAME  Any string  Name of the target column in the input file.    -mode  TPOT_MODE  ['classification', 'regression']  Whether TPOT is being used for a supervised classification or regression problem.    -o  OUTPUT_FILE  String path to a file  File to export the code for the final optimized pipeline.    -g  GENERATIONS  Any positive integer  Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    -p  POPULATION_SIZE  Any positive integer  Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    -os  OFFSPRING_SIZE  Any positive integer  Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.    -mr  MUTATION_RATE  [0.0, 1.0]  GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    -xr  CROSSOVER_RATE  [0.0, 1.0]  GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.    -scoring  SCORING_FN  'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'  Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on  scoring functions  for more details.    -cv  NUM_CV_FOLDS  Any integer >1  Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.    -njobs  NUM_JOBS  Any positive integer or -1  Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.    -maxtime  MAX_TIME_MINS  Any positive integer  How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.    -maxeval  MAX_EVAL_MINS  Any positive integer  How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.    -s  RANDOM_STATE  Any positive integer  Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.    -config  CONFIG_FILE  String path to a file  Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like: \nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}     -v  VERBOSITY  {0, 1, 2, 3}  How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.    --no-update-check  N/A  Flag indicating whether the TPOT version checker should be disabled.    --version  N/A  Show TPOT's version number and exit.    --help  N/A  Show TPOT's help documentation and exit.    An example command-line call to TPOT may look like:  tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2",
             "title": "TPOT on the command line"
         },
         {
             "location": "/using/#tpot-with-code",
-            "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn.  TPOT can be imported just like any regular Python module. To import TPOT, type:  from tpot import TPOTClassifier  then create an instance of TPOT as follows:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()  It's also possible to use TPOT for regression problems with the  TPOTRegressor  class. Other than the class name, a  TPOTRegressor  is used the same way as a  TPOTClassifier .  Note that you can pass several parameters to the TPOT instantiation call:    Parameter  Valid values  Effect    generations  Any positive integer  Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    population_size  Any positive integer  Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    offspring_size  Any positive integer  Number of offspring to produce in each GP generation. By default, offspring_size = population_size.    mutation_rate  [0.0, 1.0]  Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    crossover_rate  [0.0, 1.0]  Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    scoring  'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature  scorer(y_true, y_pred)  Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on  scoring functions  for more details.    cv  Any integer >1  Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.    n_jobs  Any positive integer or -1  Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.    max_time_mins  Any positive integer  How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.    max_eval_time_mins  Any positive integer  How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.    random_state  Any positive integer  Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.    config_dict  Python dictionary  Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example: \nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}     warm_start  [True, False]  Flag indicating whether the TPOT instance will reuse the population from previous calls to fit().    verbosity  {0, 1, 2, 3}  How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.    disable_update_check  [True, False]  Flag indicating whether the TPOT version checker should be disabled.    Some example code with custom TPOT parameters might look like:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)  Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the  fit  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)  The  fit()  function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.  You can then proceed to evaluate the final pipeline on the testing set with the  score()  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))  Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the  export()  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')  Once this code finishes running,  tpot_exported_pipeline.py  will contain the Python code for the optimized pipeline.  Check our  examples  to see TPOT applied to some specific data sets.",
+            "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn.  TPOT can be imported just like any regular Python module. To import TPOT, type:  from tpot import TPOTClassifier  then create an instance of TPOT as follows:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()  It's also possible to use TPOT for regression problems with the  TPOTRegressor  class. Other than the class name, a  TPOTRegressor  is used the same way as a  TPOTClassifier .  Note that you can pass several parameters to the TPOT instantiation call:    Parameter  Valid values  Effect    generations  Any positive integer  Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    population_size  Any positive integer  Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    offspring_size  Any positive integer  Number of offspring to produce in each GP generation. By default, offspring_size = population_size.    mutation_rate  [0.0, 1.0]  Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    crossover_rate  [0.0, 1.0]  Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    scoring  'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature  scorer(y_true, y_pred)  Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on  scoring functions  for more details.    cv  Any integer >1  Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.    n_jobs  Any positive integer or -1  Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.    max_time_mins  Any positive integer  How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.    max_eval_time_mins  Any positive integer  How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.    random_state  Any positive integer  Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.    config_dict  Python dictionary  Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example: \nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}     warm_start  [True, False]  Flag indicating whether the TPOT instance will reuse the population from previous calls to fit().    verbosity  {0, 1, 2, 3}  How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.    disable_update_check  [True, False]  Flag indicating whether the TPOT version checker should be disabled.    Some example code with custom TPOT parameters might look like:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)  Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the  fit  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)  The  fit()  function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.  You can then proceed to evaluate the final pipeline on the testing set with the  score()  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))  Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the  export()  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')  Once this code finishes running,  tpot_exported_pipeline.py  will contain the Python code for the optimized pipeline.  Check our  examples  to see TPOT applied to some specific data sets.",
             "title": "TPOT with code"
         },
         {
@@ -42,7 +42,7 @@
         },
         {
             "location": "/examples/Boston_Example/",
-            "text": "The following code illustrates the usage of TPOT with the Boston house prices data set.\n\n\nfrom tpot import TPOTRegressor\nfrom sklearn.datasets import load_boston\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_boston()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ntpot = TPOTRegressor(generations=5, population_size=20, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_boston_pipeline.py')\n\n\n\n\nRunning this code should discover a pipeline that achieves about 12.77 mean squared error (MSE).\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nAfter running the above code, the corresponding Python code should be exported to the \ntpot_boston_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.pipeline import make_pipeline\n\n# NOTE: Make sure that the target is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\n\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = make_pipeline(\n    ExtraTreesRegressor(max_features=0.76, n_estimators=500)\n)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)",
+            "text": "The following code illustrates the usage of TPOT with the Boston house prices data set.\n\n\nfrom tpot import TPOTRegressor\nfrom sklearn.datasets import load_boston\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_boston()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ntpot = TPOTRegressor(generations=5, population_size=20, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_boston_pipeline.py')\n\n\n\n\nRunning this code should discover a pipeline that achieves at least 12.77 mean squared error (MSE).\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nAfter running the above code, the corresponding Python code should be exported to the \ntpot_boston_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.pipeline import make_pipeline\n\n# NOTE: Make sure that the target is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\n\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = make_pipeline(\n    ExtraTreesRegressor(max_features=0.76, n_estimators=500)\n)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)",
             "title": "Boston Example"
         },
         {
@@ -52,17 +52,17 @@
         },
         {
             "location": "/contributing/",
-            "text": "We welcome you to \ncheck the existing issues\n for bugs or enhancements to work on. If you have an idea for an extension to TPOT, please \nfile a new issue\n so we can discuss it.\n\n\nProject layout\n\n\nThe latest stable release of TPOT is on the \nmaster branch\n, whereas the latest version of TPOT in development is on the \ndevelopment branch\n. Make sure you are looking at and working on the correct branch if you're looking to contribute code.\n\n\nIn terms of directory structure:\n\n\n\n\nAll of TPOT's code sources are in the \ntpot\n directory\n\n\nThe documentation sources are in the \ndocs\n directory\n\n\nImages in the documentation are in the \nimages\n directory\n\n\nTutorials for TPOT are in the \ntutorials\n directory\n\n\nUnit tests for TPOT are in the \ntests.py\n file\n\n\n\n\nMake sure to familiarize yourself with the project layout before making any major contributions, and especially make sure to send all code changes to the \ndevelopment\n branch.\n\n\nHow to contribute\n\n\nThe preferred way to contribute to TPOT is to fork the \n\nmain repository\n on\nGitHub:\n\n\n\n\n\n\nFork the \nproject repository\n:\n   click on the 'Fork' button near the top of the page. This creates\n   a copy of the code under your account on the GitHub server.\n\n\n\n\n\n\nClone this copy to your local disk:\n\n\n  $ git clone git@github.com:YourLogin/tpot.git\n  $ cd tpot\n\n\n\n\n\n\n\nCreate a branch to hold your changes:\n\n\n  $ git checkout -b my-contribution\n\n\n\n\n\n\n\nMake sure your local environment is setup correctly for development. Installation instructions are almost identical to \nthe user instructions\n except that TPOT should \nnot\n be installed. If you have TPOT installed on your computer then make sure you are using a virtual environment that does not have TPOT installed. Furthermore, you should make sure you have installed the \nnose\n package into your development environment so that you can test changes locally.\n\n\n  $ conda install nose\n\n\n\n\n\n\n\nStart making changes on your newly created branch, remembering to never work on the \nmaster\n branch! Work on this copy on your computer using Git to do the version control.\n\n\n\n\n\n\nOnce some changes are saved locally, you can use your tweaked version of TPOT by navigating to the project's base directory and running TPOT directly from the command line:\n\n\n  $ python -m tpot.driver\n\n\n\nor by running script that imports and uses the TPOT module with code similar to \nfrom tpot import TPOTClassifier\n\n\n\n\n\n\nTo check your changes haven't broken any existing tests and to check new tests you've added pass run the following (note, you must have the \nnose\n package installed within your dev environment for this to work):\n\n\n  $ nosetests -s -v\n\n\n\n\n\n\n\nWhen you're done editing and local testing, run:\n\n\n  $ git add modified_files\n  $ git commit\n\n\n\n\n\n\n\nto record your changes in Git, then push them to GitHub with:\n\n\n      $ git push -u origin my-contribution\n\n\n\nFinally, go to the web page of your fork of the TPOT repo, and click 'Pull Request' (PR) to send your changes to the maintainers for review. Make sure that you send your PR to the \ndevelopment\n branch, as the \nmaster\n branch is reserved for the latest stable release. This will start the CI server to check all the project's unit tests run and send an email to the maintainers.\n\n\n(If any of the above seems like magic to you, then look up the \n\nGit documentation\n on the web.)\n\n\nBefore submitting your pull request\n\n\nBefore you submit a pull request for your contribution, please work through this checklist to make sure that you have done everything necessary so we can efficiently review and accept your changes.\n\n\nIf your contribution changes TPOT in any way:\n\n\n\n\n\n\nUpdate the \ndocumentation\n so all of your changes are reflected there.\n\n\n\n\n\n\nUpdate the \nREADME\n if anything there has changed.\n\n\n\n\n\n\nIf your contribution involves any code changes:\n\n\n\n\n\n\nUpdate the \nproject unit tests\n to test your code changes.\n\n\n\n\n\n\nMake sure that your code is properly commented with \ndocstrings\n and comments explaining your rationale behind non-obvious coding practices.\n\n\n\n\n\n\nIf your code affected any of the pipeline operators, make sure that the corresponding \nexport functionality\n reflects those changes.\n\n\n\n\n\n\nIf your contribution requires a new library dependency:\n\n\n\n\n\n\nDouble-check that the new dependency is easy to install via \npip\n or Anaconda and supports both Python 2 and 3. If the dependency requires a complicated installation, then we most likely won't merge your changes because we want to keep TPOT easy to install.\n\n\n\n\n\n\nAdd the required version of the library to \n.travis.yml\n\n\n\n\n\n\nAdd a line to pip install the library to \n.travis_install.sh\n\n\n\n\n\n\nAdd a line to print the version of the library to \n.travis_install.sh\n\n\n\n\n\n\nSimilarly add a line to print the version of the library to \n.travis_test.sh\n\n\n\n\n\n\nUpdating the documentation\n\n\nWe use \nmkdocs\n to manage our \ndocumentation\n. This allows us to write the docs in Markdown and compile them to HTML as needed. Below are a few useful commands to know when updating the documentation. Make sure that you are running them in the base documentation directory, \ndocs\n.\n\n\n\n\n\n\nmkdocs serve\n: Hosts of a local version of the documentation that you can access at the provided URL. The local version will update automatically as you save changes to the documentation.\n\n\n\n\n\n\nmkdocs build --clean\n: Creates a fresh build of the documentation in HTML. Always run this before deploying the documentation to GitHub.\n\n\n\n\n\n\nmkdocs gh-deploy\n: Deploys the documentation to GitHub. If you're deploying on your fork of TPOT, the online documentation should be accessible at \nhttp://.github.io/tpot/\n. Generally, you shouldn't need to run this command because you can view your changes with \nmkdocs serve\n.\n\n\n\n\n\n\nAfter submitting your pull request\n\n\nAfter submitting your pull request, \nTravis-CI\n will automatically run unit tests on your changes and make sure that your updated code builds and runs on Python 2 and 3. We also use services that automatically check code quality and test coverage.\n\n\nCheck back shortly after submitting your pull request to make sure that your code passes these checks. If any of the checks come back with a red X, then do your best to address the errors.",
+            "text": "We welcome you to \ncheck the existing issues\n for bugs or enhancements to work on. If you have an idea for an extension to TPOT, please \nfile a new issue\n so we can discuss it.\n\n\nProject layout\n\n\nThe latest stable release of TPOT is on the \nmaster branch\n, whereas the latest version of TPOT in development is on the \ndevelopment branch\n. Make sure you are looking at and working on the correct branch if you're looking to contribute code.\n\n\nIn terms of directory structure:\n\n\n\n\nAll of TPOT's code sources are in the \ntpot\n directory\n\n\nThe documentation sources are in the \ndocs_sources\n directory\n\n\nImages in the documentation are in the \nimages\n directory\n\n\nTutorials for TPOT are in the \ntutorials\n directory\n\n\nUnit tests for TPOT are in the \ntests.py\n file\n\n\n\n\nMake sure to familiarize yourself with the project layout before making any major contributions, and especially make sure to send all code changes to the \ndevelopment\n branch.\n\n\nHow to contribute\n\n\nThe preferred way to contribute to TPOT is to fork the \n\nmain repository\n on\nGitHub:\n\n\n\n\n\n\nFork the \nproject repository\n:\n   click on the 'Fork' button near the top of the page. This creates\n   a copy of the code under your account on the GitHub server.\n\n\n\n\n\n\nClone this copy to your local disk:\n\n\n  $ git clone git@github.com:YourUsername/tpot.git\n  $ cd tpot\n\n\n\n\n\n\n\nCreate a branch to hold your changes:\n\n\n  $ git checkout -b my-contribution\n\n\n\n\n\n\n\nMake sure your local environment is setup correctly for development. Installation instructions are almost identical to \nthe user instructions\n except that TPOT should \nnot\n be installed. If you have TPOT installed on your computer then make sure you are using a virtual environment that does not have TPOT installed. Furthermore, you should make sure you have installed the \nnose\n package into your development environment so that you can test changes locally.\n\n\n  $ conda install nose\n\n\n\n\n\n\n\nStart making changes on your newly created branch, remembering to never work on the \nmaster\n branch! Work on this copy on your computer using Git to do the version control.\n\n\n\n\n\n\nOnce some changes are saved locally, you can use your tweaked version of TPOT by navigating to the project's base directory and running TPOT directly from the command line:\n\n\n  $ python -m tpot.driver\n\n\n\nor by running script that imports and uses the TPOT module with code similar to \nfrom tpot import TPOTClassifier\n\n\n\n\n\n\nTo check your changes haven't broken any existing tests and to check new tests you've added pass run the following (note, you must have the \nnose\n package installed within your dev environment for this to work):\n\n\n  $ nosetests -s -v\n\n\n\n\n\n\n\nWhen you're done editing and local testing, run:\n\n\n  $ git add modified_files\n  $ git commit\n\n\n\n\n\n\n\nto record your changes in Git, then push them to GitHub with:\n\n\n      $ git push -u origin my-contribution\n\n\n\nFinally, go to the web page of your fork of the TPOT repo, and click 'Pull Request' (PR) to send your changes to the maintainers for review. Make sure that you send your PR to the \ndevelopment\n branch, as the \nmaster\n branch is reserved for the latest stable release. This will start the CI server to check all the project's unit tests run and send an email to the maintainers.\n\n\n(If any of the above seems like magic to you, then look up the \n\nGit documentation\n on the web.)\n\n\nBefore submitting your pull request\n\n\nBefore you submit a pull request for your contribution, please work through this checklist to make sure that you have done everything necessary so we can efficiently review and accept your changes.\n\n\nIf your contribution changes TPOT in any way:\n\n\n\n\n\n\nUpdate the \ndocumentation\n so all of your changes are reflected there.\n\n\n\n\n\n\nUpdate the \nREADME\n if anything there has changed.\n\n\n\n\n\n\nIf your contribution involves any code changes:\n\n\n\n\n\n\nUpdate the \nproject unit tests\n to test your code changes.\n\n\n\n\n\n\nMake sure that your code is properly commented with \ndocstrings\n and comments explaining your rationale behind non-obvious coding practices.\n\n\n\n\n\n\nIf your code affected any of the pipeline operators, make sure that the corresponding \nexport functionality\n reflects those changes.\n\n\n\n\n\n\nIf your contribution requires a new library dependency:\n\n\n\n\n\n\nDouble-check that the new dependency is easy to install via \npip\n or Anaconda and supports both Python 2 and 3. If the dependency requires a complicated installation, then we most likely won't merge your changes because we want to keep TPOT easy to install.\n\n\n\n\n\n\nAdd the required version of the library to \n.travis.yml\n\n\n\n\n\n\nAdd a line to pip install the library to \n.travis_install.sh\n\n\n\n\n\n\nAdd a line to print the version of the library to \n.travis_install.sh\n\n\n\n\n\n\nSimilarly add a line to print the version of the library to \n.travis_test.sh\n\n\n\n\n\n\nUpdating the documentation\n\n\nWe use \nmkdocs\n to manage our \nproject documentation\n. This allows us to write the documentation in Markdown and compile them to HTML as needed. Below are a couple useful commands to know when updating the documentation. Make sure that you are running these commands in the base directory of the TPOT project.\n\n\n\n\n\n\nmkdocs serve\n: Hosts of a local version of the documentation that you can access at the provided URL. The local version will update automatically as you save changes to the documentation.\n\n\n\n\n\n\nmkdocs build --clean\n: Creates a fresh build of the documentation in HTML in the \ndocs\n directory. Always run this before pushing the documentation to GitHub.\n\n\n\n\n\n\nAfter submitting your pull request\n\n\nAfter submitting your pull request, \nTravis-CI\n will automatically run unit tests on your changes and make sure that your updated code builds and runs on Python 2 and 3. We also use services that automatically check code quality and test coverage.\n\n\nCheck back shortly after submitting your pull request to make sure that your code passes these checks. If any of the checks come back with a red X, then do your best to address the errors.",
             "title": "Contributing"
         },
         {
             "location": "/contributing/#project-layout",
-            "text": "The latest stable release of TPOT is on the  master branch , whereas the latest version of TPOT in development is on the  development branch . Make sure you are looking at and working on the correct branch if you're looking to contribute code.  In terms of directory structure:   All of TPOT's code sources are in the  tpot  directory  The documentation sources are in the  docs  directory  Images in the documentation are in the  images  directory  Tutorials for TPOT are in the  tutorials  directory  Unit tests for TPOT are in the  tests.py  file   Make sure to familiarize yourself with the project layout before making any major contributions, and especially make sure to send all code changes to the  development  branch.",
+            "text": "The latest stable release of TPOT is on the  master branch , whereas the latest version of TPOT in development is on the  development branch . Make sure you are looking at and working on the correct branch if you're looking to contribute code.  In terms of directory structure:   All of TPOT's code sources are in the  tpot  directory  The documentation sources are in the  docs_sources  directory  Images in the documentation are in the  images  directory  Tutorials for TPOT are in the  tutorials  directory  Unit tests for TPOT are in the  tests.py  file   Make sure to familiarize yourself with the project layout before making any major contributions, and especially make sure to send all code changes to the  development  branch.",
             "title": "Project layout"
         },
         {
             "location": "/contributing/#how-to-contribute",
-            "text": "The preferred way to contribute to TPOT is to fork the  main repository  on\nGitHub:    Fork the  project repository :\n   click on the 'Fork' button near the top of the page. This creates\n   a copy of the code under your account on the GitHub server.    Clone this copy to your local disk:    $ git clone git@github.com:YourLogin/tpot.git\n  $ cd tpot    Create a branch to hold your changes:    $ git checkout -b my-contribution    Make sure your local environment is setup correctly for development. Installation instructions are almost identical to  the user instructions  except that TPOT should  not  be installed. If you have TPOT installed on your computer then make sure you are using a virtual environment that does not have TPOT installed. Furthermore, you should make sure you have installed the  nose  package into your development environment so that you can test changes locally.    $ conda install nose    Start making changes on your newly created branch, remembering to never work on the  master  branch! Work on this copy on your computer using Git to do the version control.    Once some changes are saved locally, you can use your tweaked version of TPOT by navigating to the project's base directory and running TPOT directly from the command line:    $ python -m tpot.driver  or by running script that imports and uses the TPOT module with code similar to  from tpot import TPOTClassifier    To check your changes haven't broken any existing tests and to check new tests you've added pass run the following (note, you must have the  nose  package installed within your dev environment for this to work):    $ nosetests -s -v    When you're done editing and local testing, run:    $ git add modified_files\n  $ git commit    to record your changes in Git, then push them to GitHub with:        $ git push -u origin my-contribution  Finally, go to the web page of your fork of the TPOT repo, and click 'Pull Request' (PR) to send your changes to the maintainers for review. Make sure that you send your PR to the  development  branch, as the  master  branch is reserved for the latest stable release. This will start the CI server to check all the project's unit tests run and send an email to the maintainers.  (If any of the above seems like magic to you, then look up the  Git documentation  on the web.)",
+            "text": "The preferred way to contribute to TPOT is to fork the  main repository  on\nGitHub:    Fork the  project repository :\n   click on the 'Fork' button near the top of the page. This creates\n   a copy of the code under your account on the GitHub server.    Clone this copy to your local disk:    $ git clone git@github.com:YourUsername/tpot.git\n  $ cd tpot    Create a branch to hold your changes:    $ git checkout -b my-contribution    Make sure your local environment is setup correctly for development. Installation instructions are almost identical to  the user instructions  except that TPOT should  not  be installed. If you have TPOT installed on your computer then make sure you are using a virtual environment that does not have TPOT installed. Furthermore, you should make sure you have installed the  nose  package into your development environment so that you can test changes locally.    $ conda install nose    Start making changes on your newly created branch, remembering to never work on the  master  branch! Work on this copy on your computer using Git to do the version control.    Once some changes are saved locally, you can use your tweaked version of TPOT by navigating to the project's base directory and running TPOT directly from the command line:    $ python -m tpot.driver  or by running script that imports and uses the TPOT module with code similar to  from tpot import TPOTClassifier    To check your changes haven't broken any existing tests and to check new tests you've added pass run the following (note, you must have the  nose  package installed within your dev environment for this to work):    $ nosetests -s -v    When you're done editing and local testing, run:    $ git add modified_files\n  $ git commit    to record your changes in Git, then push them to GitHub with:        $ git push -u origin my-contribution  Finally, go to the web page of your fork of the TPOT repo, and click 'Pull Request' (PR) to send your changes to the maintainers for review. Make sure that you send your PR to the  development  branch, as the  master  branch is reserved for the latest stable release. This will start the CI server to check all the project's unit tests run and send an email to the maintainers.  (If any of the above seems like magic to you, then look up the  Git documentation  on the web.)",
             "title": "How to contribute"
         },
         {
@@ -72,7 +72,7 @@
         },
         {
             "location": "/contributing/#updating-the-documentation",
-            "text": "We use  mkdocs  to manage our  documentation . This allows us to write the docs in Markdown and compile them to HTML as needed. Below are a few useful commands to know when updating the documentation. Make sure that you are running them in the base documentation directory,  docs .    mkdocs serve : Hosts of a local version of the documentation that you can access at the provided URL. The local version will update automatically as you save changes to the documentation.    mkdocs build --clean : Creates a fresh build of the documentation in HTML. Always run this before deploying the documentation to GitHub.    mkdocs gh-deploy : Deploys the documentation to GitHub. If you're deploying on your fork of TPOT, the online documentation should be accessible at  http://.github.io/tpot/ . Generally, you shouldn't need to run this command because you can view your changes with  mkdocs serve .",
+            "text": "We use  mkdocs  to manage our  project documentation . This allows us to write the documentation in Markdown and compile them to HTML as needed. Below are a couple useful commands to know when updating the documentation. Make sure that you are running these commands in the base directory of the TPOT project.    mkdocs serve : Hosts of a local version of the documentation that you can access at the provided URL. The local version will update automatically as you save changes to the documentation.    mkdocs build --clean : Creates a fresh build of the documentation in HTML in the  docs  directory. Always run this before pushing the documentation to GitHub.",
             "title": "Updating the documentation"
         },
         {
@@ -82,12 +82,12 @@
         },
         {
             "location": "/releases/",
-            "text": "Version 0.7\n\n\n\n\n\n\nTPOT now supports parallel computing for pipeline optimization (Linux and MacOS only)\n TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the \nn_jobs\n parameter in both TPOTClassifier and TPOTRegressor. The \ncommand-line interface\n also supports this feature through the \n-njobs\n parameter.\n\n\n\n\n\n\nTPOT now support customized dictionary of operators and parameters\n TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the \noperator_dict\n parameter. The format of this customized dictionary can be found in \nonline manual\n. The \ncommand-line interface\n also supports this feature through the \n-operator\n parameter but only takes a file including the dictionary instead.\n\n\n\n\n\n\nTPOT now allows you to \nspecify a time limit (default time limit is 5 minutes)\n for evaluating a single pipeline in optimization process with the \nmax_eval_time_mins\n parameter, so TPOT can skip these time-consuming pipelines.\n\n\n\n\n\n\nThe [evolutionary algorithm] is replaced by the (mu + lambda) evolutionary algorithm. TPOT allows you to set offspring size (lambda) for pipeline optimization in TPOT with a new \noffspring_size\n parameter. The \ncommand-line interface\n also supports this feature through the \n-c\n parameter.\n\n\n\n\n\n\nFixed issue about reproducing results with same random seed\n\n\n\n\n\n\nDefault operators and their parameters in TPOT were refined.\n\n\n\n\n\n\nThe TPOT point mutation operator was refined\n\n\n\n\n\n\nTPOT now supports sample weights to be used like \nTPOTRegressor.fit(x_train, y_train, sample_weights=sample_weights)\n\n\n\n\n\n\nTPOT now checks duplicated pipeline to accelerate optimization process.\n\n\n\n\n\n\nThe default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn.\n\n\n\n\n\n\nVersion 0.6\n\n\n\n\n\n\nTPOT now supports regression problems!\n We have created two separate \nTPOTClassifier\n and \nTPOTRegressor\n classes to support classification and regression problems, respectively. The \ncommand-line interface\n also supports this feature through the \n-mode\n parameter.\n\n\n\n\n\n\nTPOT now allows you to \nspecify a time limit\n for the optimization process with the \nmax_time_mins\n parameter, so you don't need to guess how long TPOT will take any more to recommend a pipeline to you.\n\n\n\n\n\n\nAdded a new operator that performs feature selection using \nExtraTrees\n feature importance scores.\n\n\n\n\n\n\nXGBoost\n has been added as an optional dependency to TPOT.\n If you have XGBoost installed, TPOT will automatically detect your installation and use the \nXGBoostClassifier\n and \nXGBoostRegressor\n in its pipelines.\n\n\n\n\n\n\nTPOT now offers a verbosity level of 3 (\"science mode\"), which outputs the entire Pareto front instead of only the current best score. This feature may be useful for users looking to make a trade-off between pipeline complexity and score.\n\n\n\n\n\n\nVersion 0.5\n\n\n\n\nMajor refactor: Each operator is defined in a separate class file. Hooray for easier-to-maintain code!\n\n\nTPOT now \nexports directly to scikit-learn Pipelines\n instead of hacky code.\n\n\nInternal representation of individuals now uses scikit-learn pipelines.\n\n\nParameters for each operator have been optimized so TPOT spends less time exploring useless parameters.\n\n\nWe have removed pandas as a dependency and instead use numpy matrices to store the data.\n\n\nTPOT now uses \nk-fold cross-validation\n when evaluating pipelines, with a default k = 3. This k parameter can be tuned when creating a new TPOT instance.\n\n\nImproved \nscoring function support\n: Even though TPOT uses balanced accuracy by default, you can now have TPOT use \nany of the scoring functions\n that \ncross_val_score\n supports.\n\n\nAdded the scikit-learn \nNormalizer\n preprocessor.\n\n\nMinor text fixes.\n\n\n\n\nVersion 0.4\n\n\nIn TPOT 0.4, we've made some major changes to the internals of TPOT and added some convenience functions. We've summarized the changes below.\n\n\n\n\nAdded new sklearn models and preprocessors\n\n\n\n\nAdaBoostClassifier\n\n\nBernoulliNB\n\n\nExtraTreesClassifier\n\n\nGaussianNB\n\n\nMultinomialNB\n\n\nLinearSVC\n\n\nPassiveAggressiveClassifier\n\n\nGradientBoostingClassifier\n\n\nRBFSampler\n\n\nFastICA\n\n\nFeatureAgglomeration\n\n\nNystroem\n\n\n\n\nAdded operator that inserts virtual features for the count of features with values of zero\n\n\nReworked parameterization of TPOT operators\n\n\n\nReduced parameter search space with information from a scikit-learn benchmark\n\n\nTPOT no longer generates arbitrary parameter values, but uses a fixed parameter set instead\n\n\n\n\nRemoved XGBoost as a dependency\n\n\n\nToo many users were having install issues with XGBoost\n\n\nReplaced with scikit-learn's GradientBoostingClassifier\n\n\n\n\nImproved descriptiveness of TPOT command line parameter documentation\n\n\nRemoved min/max/avg details during fit() when verbosity > 1\n\n\n\n\nReplaced with tqdm progress bar\n\n\nAdded tqdm as a dependency\n\n\n\n\nAdded \nfit_predict()\n convenience function\n\n\nAdded \nget_params()\n function so TPOT can operate in scikit-learn's \ncross_val_score\n & related functions\n\n\n\n\n\nVersion 0.3\n\n\n\n\nWe revised the internal optimization process of TPOT to make it more efficient, in particular in regards to the model parameters that TPOT optimizes over.\n\n\n\n\nVersion 0.2\n\n\n\n\n\n\nTPOT now has the ability to export the optimized pipelines to sklearn code.\n\n\n\n\n\n\nLogistic regression, SVM, and k-nearest neighbors classifiers were added as pipeline operators. Previously, TPOT only included decision tree and random forest classifiers.\n\n\n\n\n\n\nTPOT can now use arbitrary scoring functions for the optimization process.\n\n\n\n\n\n\nTPOT now performs multi-objective Pareto optimization to balance model complexity (i.e., # of pipeline operators) and the score of the pipeline.\n\n\n\n\n\n\nVersion 0.1\n\n\n\n\n\n\nFirst public release of TPOT.\n\n\n\n\n\n\nOptimizes pipelines with decision trees and random forest classifiers as the model, and uses a handful of feature preprocessors.",
+            "text": "Version 0.7\n\n\n\n\n\n\nTPOT now has multiprocessing support (Linux and macOS only).\n TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the \nn_jobs\n parameter in both TPOTClassifier and TPOTRegressor.\n\n\n\n\n\n\nTPOT now allows you to \ncustomize the operators and parameters explored during the optimization process.\n TPOT allows you to customize the list of operators and parameters in optimization process of TPOT with the \nconfig_dict\n parameter. The format of this customized dictionary can be found in the \nonline documentation\n.\n\n\n\n\n\n\nTPOT now allows you to \nspecify a time limit for evaluating a single pipeline\n  (default limit is 5 minutes) in optimization process with the \nmax_eval_time_mins\n parameter, so TPOT won't spend hours evaluating overly-complex pipelines.\n\n\n\n\n\n\nWe tweaked TPOT's underlying evolutionary optimization algorithm to work even better, including using the \nmu+lambda algorithm\n. This algorithm gives you more control of how many pipelines are generated every iteration with the \noffspring_size\n parameter.\n\n\n\n\n\n\nFixed a reproducibility issue where setting \nrandom_seed\n didn't necessarily result in the same results every time. This bug was present since version 0.6.\n\n\n\n\n\n\nRefined the default operators and parameters in TPOT, so TPOT 0.7 should work even better than 0.6.\n\n\n\n\n\n\nTPOT now supports sample weights in the fitness function if some if your samples are more important to classify correctly than others. The sample weights option works the same as in scikit-learn, e.g., \ntpot.fit(x_train, y_train, sample_weights=sample_weights)\n.\n\n\n\n\n\n\nThe default scoring metric in TPOT has been changed from balanced accuracy to accuracy, the same default metric for classification algorithms in scikit-learn. Balanced accuracy can still be used by setting \nscoring='balanced_accuracy'\n when creating a TPOT instance.\n\n\n\n\n\n\nVersion 0.6\n\n\n\n\n\n\nTPOT now supports regression problems!\n We have created two separate \nTPOTClassifier\n and \nTPOTRegressor\n classes to support classification and regression problems, respectively. The \ncommand-line interface\n also supports this feature through the \n-mode\n parameter.\n\n\n\n\n\n\nTPOT now allows you to \nspecify a time limit\n for the optimization process with the \nmax_time_mins\n parameter, so you don't need to guess how long TPOT will take any more to recommend a pipeline to you.\n\n\n\n\n\n\nAdded a new operator that performs feature selection using \nExtraTrees\n feature importance scores.\n\n\n\n\n\n\nXGBoost\n has been added as an optional dependency to TPOT.\n If you have XGBoost installed, TPOT will automatically detect your installation and use the \nXGBoostClassifier\n and \nXGBoostRegressor\n in its pipelines.\n\n\n\n\n\n\nTPOT now offers a verbosity level of 3 (\"science mode\"), which outputs the entire Pareto front instead of only the current best score. This feature may be useful for users looking to make a trade-off between pipeline complexity and score.\n\n\n\n\n\n\nVersion 0.5\n\n\n\n\nMajor refactor: Each operator is defined in a separate class file. Hooray for easier-to-maintain code!\n\n\nTPOT now \nexports directly to scikit-learn Pipelines\n instead of hacky code.\n\n\nInternal representation of individuals now uses scikit-learn pipelines.\n\n\nParameters for each operator have been optimized so TPOT spends less time exploring useless parameters.\n\n\nWe have removed pandas as a dependency and instead use numpy matrices to store the data.\n\n\nTPOT now uses \nk-fold cross-validation\n when evaluating pipelines, with a default k = 3. This k parameter can be tuned when creating a new TPOT instance.\n\n\nImproved \nscoring function support\n: Even though TPOT uses balanced accuracy by default, you can now have TPOT use \nany of the scoring functions\n that \ncross_val_score\n supports.\n\n\nAdded the scikit-learn \nNormalizer\n preprocessor.\n\n\nMinor text fixes.\n\n\n\n\nVersion 0.4\n\n\nIn TPOT 0.4, we've made some major changes to the internals of TPOT and added some convenience functions. We've summarized the changes below.\n\n\n\n\nAdded new sklearn models and preprocessors\n\n\n\n\nAdaBoostClassifier\n\n\nBernoulliNB\n\n\nExtraTreesClassifier\n\n\nGaussianNB\n\n\nMultinomialNB\n\n\nLinearSVC\n\n\nPassiveAggressiveClassifier\n\n\nGradientBoostingClassifier\n\n\nRBFSampler\n\n\nFastICA\n\n\nFeatureAgglomeration\n\n\nNystroem\n\n\n\n\nAdded operator that inserts virtual features for the count of features with values of zero\n\n\nReworked parameterization of TPOT operators\n\n\n\nReduced parameter search space with information from a scikit-learn benchmark\n\n\nTPOT no longer generates arbitrary parameter values, but uses a fixed parameter set instead\n\n\n\n\nRemoved XGBoost as a dependency\n\n\n\nToo many users were having install issues with XGBoost\n\n\nReplaced with scikit-learn's GradientBoostingClassifier\n\n\n\n\nImproved descriptiveness of TPOT command line parameter documentation\n\n\nRemoved min/max/avg details during fit() when verbosity > 1\n\n\n\n\nReplaced with tqdm progress bar\n\n\nAdded tqdm as a dependency\n\n\n\n\nAdded \nfit_predict()\n convenience function\n\n\nAdded \nget_params()\n function so TPOT can operate in scikit-learn's \ncross_val_score\n & related functions\n\n\n\n\n\nVersion 0.3\n\n\n\n\nWe revised the internal optimization process of TPOT to make it more efficient, in particular in regards to the model parameters that TPOT optimizes over.\n\n\n\n\nVersion 0.2\n\n\n\n\n\n\nTPOT now has the ability to export the optimized pipelines to sklearn code.\n\n\n\n\n\n\nLogistic regression, SVM, and k-nearest neighbors classifiers were added as pipeline operators. Previously, TPOT only included decision tree and random forest classifiers.\n\n\n\n\n\n\nTPOT can now use arbitrary scoring functions for the optimization process.\n\n\n\n\n\n\nTPOT now performs multi-objective Pareto optimization to balance model complexity (i.e., # of pipeline operators) and the score of the pipeline.\n\n\n\n\n\n\nVersion 0.1\n\n\n\n\n\n\nFirst public release of TPOT.\n\n\n\n\n\n\nOptimizes pipelines with decision trees and random forest classifiers as the model, and uses a handful of feature preprocessors.",
             "title": "Release Notes"
         },
         {
             "location": "/releases/#version-07",
-            "text": "TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only)  TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the  n_jobs  parameter in both TPOTClassifier and TPOTRegressor. The  command-line interface  also supports this feature through the  -njobs  parameter.    TPOT now support customized dictionary of operators and parameters  TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the  operator_dict  parameter. The format of this customized dictionary can be found in  online manual . The  command-line interface  also supports this feature through the  -operator  parameter but only takes a file including the dictionary instead.    TPOT now allows you to  specify a time limit (default time limit is 5 minutes)  for evaluating a single pipeline in optimization process with the  max_eval_time_mins  parameter, so TPOT can skip these time-consuming pipelines.    The [evolutionary algorithm] is replaced by the (mu + lambda) evolutionary algorithm. TPOT allows you to set offspring size (lambda) for pipeline optimization in TPOT with a new  offspring_size  parameter. The  command-line interface  also supports this feature through the  -c  parameter.    Fixed issue about reproducing results with same random seed    Default operators and their parameters in TPOT were refined.    The TPOT point mutation operator was refined    TPOT now supports sample weights to be used like  TPOTRegressor.fit(x_train, y_train, sample_weights=sample_weights)    TPOT now checks duplicated pipeline to accelerate optimization process.    The default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn.",
+            "text": "TPOT now has multiprocessing support (Linux and macOS only).  TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the  n_jobs  parameter in both TPOTClassifier and TPOTRegressor.    TPOT now allows you to  customize the operators and parameters explored during the optimization process.  TPOT allows you to customize the list of operators and parameters in optimization process of TPOT with the  config_dict  parameter. The format of this customized dictionary can be found in the  online documentation .    TPOT now allows you to  specify a time limit for evaluating a single pipeline   (default limit is 5 minutes) in optimization process with the  max_eval_time_mins  parameter, so TPOT won't spend hours evaluating overly-complex pipelines.    We tweaked TPOT's underlying evolutionary optimization algorithm to work even better, including using the  mu+lambda algorithm . This algorithm gives you more control of how many pipelines are generated every iteration with the  offspring_size  parameter.    Fixed a reproducibility issue where setting  random_seed  didn't necessarily result in the same results every time. This bug was present since version 0.6.    Refined the default operators and parameters in TPOT, so TPOT 0.7 should work even better than 0.6.    TPOT now supports sample weights in the fitness function if some if your samples are more important to classify correctly than others. The sample weights option works the same as in scikit-learn, e.g.,  tpot.fit(x_train, y_train, sample_weights=sample_weights) .    The default scoring metric in TPOT has been changed from balanced accuracy to accuracy, the same default metric for classification algorithms in scikit-learn. Balanced accuracy can still be used by setting  scoring='balanced_accuracy'  when creating a TPOT instance.",
             "title": "Version 0.7"
         },
         {
diff --git a/docs/releases/index.html b/docs/releases/index.html
index 5fe8ad6b..b0d329f4 100644
--- a/docs/releases/index.html
+++ b/docs/releases/index.html
@@ -194,34 +194,28 @@
                 
      Version 0.7
      
 
        
 
      • 
-
        TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only) TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the n_jobs parameter in both TPOTClassifier and TPOTRegressor. The command-line interface also supports this feature through the -njobs parameter.
        
+
        TPOT now has multiprocessing support (Linux and macOS only). TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the n_jobs parameter in both TPOTClassifier and TPOTRegressor.
        
 
        • 
 
      • 
-
        TPOT now support customized dictionary of operators and parameters TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the operator_dict parameter. The format of this customized dictionary can be found in online manual. The command-line interface also supports this feature through the -operator parameter but only takes a file including the dictionary instead.
        
+
        TPOT now allows you to customize the operators and parameters explored during the optimization process. TPOT allows you to customize the list of operators and parameters in optimization process of TPOT with the config_dict parameter. The format of this customized dictionary can be found in the online documentation.
        
 
        • 
 
      • 
-
        TPOT now allows you to specify a time limit (default time limit is 5 minutes) for evaluating a single pipeline in optimization process with the max_eval_time_mins parameter, so TPOT can skip these time-consuming pipelines.
        
+
        TPOT now allows you to specify a time limit for evaluating a single pipeline  (default limit is 5 minutes) in optimization process with the max_eval_time_mins parameter, so TPOT won't spend hours evaluating overly-complex pipelines.
        
 
        • 
 
      • 
-
        The [evolutionary algorithm] is replaced by the (mu + lambda) evolutionary algorithm. TPOT allows you to set offspring size (lambda) for pipeline optimization in TPOT with a new offspring_size parameter. The command-line interface also supports this feature through the -c parameter.
        
+
        We tweaked TPOT's underlying evolutionary optimization algorithm to work even better, including using the mu+lambda algorithm. This algorithm gives you more control of how many pipelines are generated every iteration with the offspring_size parameter.
        
 
        • 
 
      • 
-
        Fixed issue about reproducing results with same random seed
        
+
        Fixed a reproducibility issue where setting random_seed didn't necessarily result in the same results every time. This bug was present since version 0.6.
        
 
        • 
 
      • 
-
        Default operators and their parameters in TPOT were refined.
        
+
        Refined the default operators and parameters in TPOT, so TPOT 0.7 should work even better than 0.6.
        
 
        • 
 
      • 
-
        The TPOT point mutation operator was refined
        
+
        TPOT now supports sample weights in the fitness function if some if your samples are more important to classify correctly than others. The sample weights option works the same as in scikit-learn, e.g., tpot.fit(x_train, y_train, sample_weights=sample_weights).
        
 
        • 
 
      • 
-
        TPOT now supports sample weights to be used like TPOTRegressor.fit(x_train, y_train, sample_weights=sample_weights)
        
-
        • 
-
      • 
-
        TPOT now checks duplicated pipeline to accelerate optimization process.
        
-
        • 
-
      • 
-
        The default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn.
        
+
        The default scoring metric in TPOT has been changed from balanced accuracy to accuracy, the same default metric for classification algorithms in scikit-learn. Balanced accuracy can still be used by setting scoring='balanced_accuracy' when creating a TPOT instance.
        
 
        • 
 
      
 
      Version 0.6
      
diff --git a/docs/using/index.html b/docs/using/index.html
index 9df6b87f..bc2f684e 100644
--- a/docs/using/index.html
+++ b/docs/using/index.html
@@ -248,7 +248,7 @@ 
      TPOT on the command line
      
 
 -scoring
 SCORING_FN
-'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
+'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
 Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
 
 
@@ -376,7 +376,7 @@ 
      TPOT with code
      
 
 
 scoring
-'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
+'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
 Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
 
 
diff --git a/docs_sources/contributing.md b/docs_sources/contributing.md
index 604d2333..d708b52b 100644
--- a/docs_sources/contributing.md
+++ b/docs_sources/contributing.md
@@ -7,7 +7,7 @@ The latest stable release of TPOT is on the [master branch](https://github.com/r
 In terms of directory structure:
 
 * All of TPOT's code sources are in the `tpot` directory
-* The documentation sources are in the `docs` directory
+* The documentation sources are in the `docs_sources` directory
 * Images in the documentation are in the `images` directory
 * Tutorials for TPOT are in the `tutorials` directory
 * Unit tests for TPOT are in the `tests.py` file
@@ -26,7 +26,7 @@ GitHub:
 
 2. Clone this copy to your local disk:
 
-          $ git clone git@github.com:YourLogin/tpot.git
+          $ git clone git@github.com:YourUsername/tpot.git
           $ cd tpot
 
 3. Create a branch to hold your changes:
@@ -89,19 +89,17 @@ If your contribution requires a new library dependency:
 
 * Add a line to pip install the library to [.travis_install.sh](https://github.com/rhiever/tpot/blob/master/ci/.travis_install.sh#L46)
 
-* Add a line to print the version of the library to [.travis_install.sh](https://github.com/rhiever/tpot/blob/master/ci/.travis_install.sh#L61)
+* Add a line to print the version of the library to [.travis_install.sh](https://github.com/rhiever/tpot/blob/master/ci/.travis_install.sh#L63)
 
 * Similarly add a line to print the version of the library to [.travis_test.sh](https://github.com/rhiever/tpot/blob/master/ci/.travis_test.sh#L13)
 
 ## Updating the documentation
 
-We use [mkdocs](http://www.mkdocs.org/) to manage our [documentation](http://rhiever.github.io/tpot/). This allows us to write the docs in Markdown and compile them to HTML as needed. Below are a few useful commands to know when updating the documentation. Make sure that you are running them in the base documentation directory, `docs`.
+We use [mkdocs](http://www.mkdocs.org/) to manage our [project documentation](http://rhiever.github.io/tpot/). This allows us to write the documentation in Markdown and compile them to HTML as needed. Below are a couple useful commands to know when updating the documentation. Make sure that you are running these commands in the base directory of the TPOT project.
 
 * `mkdocs serve`: Hosts of a local version of the documentation that you can access at the provided URL. The local version will update automatically as you save changes to the documentation.
 
-* `mkdocs build --clean`: Creates a fresh build of the documentation in HTML. Always run this before deploying the documentation to GitHub.
-
-* `mkdocs gh-deploy`: Deploys the documentation to GitHub. If you're deploying on your fork of TPOT, the online documentation should be accessible at `http://.github.io/tpot/`. Generally, you shouldn't need to run this command because you can view your changes with `mkdocs serve`.
+* `mkdocs build --clean`: Creates a fresh build of the documentation in HTML in the `docs` directory. Always run this before pushing the documentation to GitHub.
 
 ## After submitting your pull request
 
diff --git a/docs_sources/examples/Boston_Example.md b/docs_sources/examples/Boston_Example.md
index bbd30e64..dfbcade0 100644
--- a/docs_sources/examples/Boston_Example.md
+++ b/docs_sources/examples/Boston_Example.md
@@ -15,7 +15,7 @@ print(tpot.score(X_test, y_test))
 tpot.export('tpot_boston_pipeline.py')
 ```
 
-Running this code should discover a pipeline that achieves about 12.77 mean squared error (MSE).
+Running this code should discover a pipeline that achieves at least 12.77 mean squared error (MSE).
 
 For details on how the `fit()`, `score()` and `export()` functions work, see the [usage documentation](/using/).
 
diff --git a/docs_sources/releases.md b/docs_sources/releases.md
index 0cfea94e..7b862114 100644
--- a/docs_sources/releases.md
+++ b/docs_sources/releases.md
@@ -1,24 +1,20 @@
 # Version 0.7
 
-* **TPOT now supports parallel computing for pipeline optimization (Linux and MacOS only)** TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the `n_jobs` parameter in both TPOTClassifier and TPOTRegressor. The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-njobs` parameter.
+* **TPOT now has multiprocessing support (Linux and macOS only).** TPOT allows you to use multiple processes for accelerating pipeline optimization in TPOT with the `n_jobs` parameter in both TPOTClassifier and TPOTRegressor.
 
-* **TPOT now support customized dictionary of operators and parameters** TPOT allows you to customize the list of preferred operators and parameters in optimization process of TPOT with the `operator_dict` parameter. The format of this customized dictionary can be found in [online manual](/using/#tpot-with-code). The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-operator` parameter but only takes a file including the dictionary instead.
+* TPOT now allows you to **customize the operators and parameters explored during the optimization process.** TPOT allows you to customize the list of operators and parameters in optimization process of TPOT with the `config_dict` parameter. The format of this customized dictionary can be found in the [online documentation](/using/#tpot-with-code).
 
-* TPOT now allows you to **specify a time limit (default time limit is 5 minutes)** for evaluating a single pipeline in optimization process with the `max_eval_time_mins` parameter, so TPOT can skip these time-consuming pipelines.
+* TPOT now allows you to **specify a time limit for evaluating a single pipeline**  (default limit is 5 minutes) in optimization process with the `max_eval_time_mins` parameter, so TPOT won't spend hours evaluating overly-complex pipelines.
 
-* The [evolutionary algorithm] is replaced by the (mu + lambda) evolutionary algorithm. TPOT allows you to set offspring size (lambda) for pipeline optimization in TPOT with a new `offspring_size` parameter. The [command-line interface](/using/#tpot-on-the-command-line) also supports this feature through the `-c` parameter.
+* We tweaked TPOT's underlying evolutionary optimization algorithm to work even better, including using the [mu+lambda algorithm](http://deap.readthedocs.io/en/master/api/algo.html#deap.algorithms.eaMuPlusLambda). This algorithm gives you more control of how many pipelines are generated every iteration with the `offspring_size` parameter.
 
-* Fixed issue about reproducing results with same random seed
+* Fixed a reproducibility issue where setting `random_seed` didn't necessarily result in the same results every time. This bug was present since version 0.6.
 
-* Default operators and their parameters in TPOT were refined.
+* Refined the default operators and parameters in TPOT, so TPOT 0.7 should work even better than 0.6.
 
-* The TPOT point mutation operator was refined
+* TPOT now supports sample weights in the fitness function if some if your samples are more important to classify correctly than others. The sample weights option works the same as in scikit-learn, e.g., `tpot.fit(x_train, y_train, sample_weights=sample_weights)`.
 
-* TPOT now supports sample weights to be used like `TPOTRegressor.fit(x_train, y_train, sample_weights=sample_weights)`
-
-* TPOT now checks duplicated pipeline to accelerate optimization process.
-
-* The default scoring metric in TPOT change from balanced accuracy to accuracy, the same default metric in scikit-learn.
+* The default scoring metric in TPOT has been changed from balanced accuracy to accuracy, the same default metric for classification algorithms in scikit-learn. Balanced accuracy can still be used by setting `scoring='balanced_accuracy'` when creating a TPOT instance.
 
 
 # Version 0.6
diff --git a/docs_sources/using.md b/docs_sources/using.md
index 8d4ce204..a6d6d8e1 100644
--- a/docs_sources/using.md
+++ b/docs_sources/using.md
@@ -72,7 +72,7 @@ TPOT offers several arguments that can be provided at the command line:
 
 -scoring
 SCORING_FN
-'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
+'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
 Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
 
 
@@ -210,7 +210,7 @@ Note that you can pass several parameters to the TPOT instantiation call:
 
 
 scoring
-'accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
+'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
 Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
 
 

From da23c9d201314e19fcd85a39536501be31022835 Mon Sep 17 00:00:00 2001
From: Randy Olson 
Date: Tue, 21 Mar 2017 17:00:12 -0400
Subject: [PATCH 142/154] Minor code quality cleanup

---
 tests.py                  |  8 ++++----
 tpot/base.py              | 18 +++++++++---------
 tpot/config_classifier.py |  2 --
 tpot/config_regressor.py  |  2 --
 tpot/gp_types.py          |  9 +--------
 5 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/tests.py b/tests.py
index 52fde637..2a46b4fa 100644
--- a/tests.py
+++ b/tests.py
@@ -8,7 +8,7 @@
 from tpot.base import TPOTBase
 from tpot.driver import positive_integer, float_range
 from tpot.export_utils import export_pipeline, generate_import_code, _indent, generate_pipeline_code, get_by_name
-from tpot.gp_types import Output_DF
+from tpot.gp_types import Output_Array
 from tpot.gp_deap import mutNodeReplacement
 from tpot.decorators import _timeout, TimedOutExc
 
@@ -523,7 +523,7 @@ def test_mutNodeReplacement():
     'KNeighborsClassifier__n_neighbors=10, '
     'KNeighborsClassifier__p=1,KNeighborsClassifier__weights=uniform')
     pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset)
-    pipeline[0].ret = Output_DF
+    pipeline[0].ret = Output_Array
     old_ret_type_list = [node.ret for node in pipeline]
     old_prims_list = [node for node in pipeline if node.arity != 0]
     mut_ind = mutNodeReplacement(pipeline, pset = tpot_obj._pset)
@@ -534,7 +534,7 @@ def test_mutNodeReplacement():
     else: # Primitive mutated
         diff_prims = list(set(new_prims_list).symmetric_difference(old_prims_list))
         assert diff_prims[0].ret == diff_prims[1].ret
-    assert mut_ind[0][0].ret == Output_DF
+    assert mut_ind[0][0].ret == Output_Array
 
 
 def test_export_pipeline():
@@ -676,7 +676,7 @@ def test_gen():
     pipeline = tpot_obj._gen_grow_safe(tpot_obj._pset, 1, 3)
 
     assert len(pipeline) > 1
-    assert pipeline[0].ret == Output_DF
+    assert pipeline[0].ret == Output_Array
 
 
 def test_positive_integer():
diff --git a/tpot/base.py b/tpot/base.py
index 90c52d4c..ee30480d 100644
--- a/tpot/base.py
+++ b/tpot/base.py
@@ -30,7 +30,7 @@
 
 import numpy as np
 import deap
-from deap import algorithms, base, creator, tools, gp
+from deap import base, creator, tools, gp
 from tqdm import tqdm
 
 from sklearn.base import BaseEstimator
@@ -50,7 +50,7 @@
 from .built_in_operators import CombineDFs
 
 from .metrics import SCORERS
-from .gp_types import Bool, Output_DF
+from .gp_types import Output_Array
 from .gp_deap import eaMuPlusLambda, mutNodeReplacement
 
 # hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS
@@ -252,7 +252,7 @@ def _setup_pset(self):
             random.seed(self.random_state)
             np.random.seed(self.random_state)
 
-        self._pset = gp.PrimitiveSetTyped('MAIN', [np.ndarray], Output_DF)
+        self._pset = gp.PrimitiveSetTyped('MAIN', [np.ndarray], Output_Array)
 
         # Rename pipeline input to "input_df"
         self._pset.renameArguments(ARG0='input_matrix')
@@ -263,9 +263,9 @@ def _setup_pset(self):
 
             if op.root:
                 # We need to add rooted primitives twice so that they can
-                # return both an Output_DF (and thus be the root of the tree),
+                # return both an Output_Array (and thus be the root of the tree),
                 # and return a np.ndarray so they can exist elsewhere in the tree.
-                p_types = (op.parameter_types()[0], Output_DF)
+                p_types = (op.parameter_types()[0], Output_Array)
                 self._pset.addPrimitive(op, *p_types)
 
             self._pset.addPrimitive(op, *op.parameter_types())
@@ -590,7 +590,7 @@ def _set_param_recursive(self, pipeline_steps, parameter, value):
         None
 
         """
-        for (pname, obj) in pipeline_steps:
+        for (_, obj) in pipeline_steps:
             recursive_attrs = ['steps', 'transformer_list', 'estimators']
             for attr in recursive_attrs:
                 if hasattr(obj, attr):
@@ -672,7 +672,7 @@ def _evaluate_individuals(self, individuals, features, classes, sample_weight =
                              type(node) is deap.gp.Primitive and node.name == 'CombineDFs'):
                             continue
                         operator_count += 1
-                except:
+                except Exception:
                     fitnesses_dict[indidx] = (5000., -float('inf'))
                     if not self._pbar.disable:
                         self._pbar.update(1)
@@ -697,7 +697,7 @@ def _wrapped_cross_val_score(sklearn_pipeline, features=features, classes=classe
                 resulting_score = np.mean(cv_scores)
             except TimedOutExc:
                 resulting_score = 'Timeout'
-            except:
+            except Exception:
                 resulting_score = -float('inf')
             return resulting_score
 
@@ -806,7 +806,7 @@ def _gen_grow_safe(self, pset, min_, max_, type_=None):
         def condition(height, depth, type_):
             """Expression generation stops when the depth is equal to height or
             when it is randomly determined that a a node should be a terminal"""
-            return type_ not in [np.ndarray, Output_DF] or depth == height
+            return type_ not in [np.ndarray, Output_Array] or depth == height
 
         return self._generate(pset, min_, max_, condition, type_)
 
diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py
index 0e77216d..dbb37bf9 100644
--- a/tpot/config_classifier.py
+++ b/tpot/config_classifier.py
@@ -16,9 +16,7 @@
 details. You should have received a copy of the GNU General Public License along
 with the TPOT library. If not, see http://www.gnu.org/licenses/.
 
-"""
 
-"""
 dictionary format (json-like format):
 key:
     operator name
diff --git a/tpot/config_regressor.py b/tpot/config_regressor.py
index 8701152c..aec94489 100644
--- a/tpot/config_regressor.py
+++ b/tpot/config_regressor.py
@@ -14,9 +14,7 @@
 details. You should have received a copy of the GNU General Public License along
 with the TPOT library. If not, see http://www.gnu.org/licenses/.
 
-"""
 
-"""
 dictionary format (json-like format):
 key:
     operator name
diff --git a/tpot/gp_types.py b/tpot/gp_types.py
index db1a95dc..a305f545 100644
--- a/tpot/gp_types.py
+++ b/tpot/gp_types.py
@@ -18,14 +18,7 @@
 
 """
 
-
-class Bool(object):
-
-    """Boolean class used for deap due to deap's poor handling of booleans"""
-
-    pass
-
-class Output_DF(object):
+class Output_Array(object):
 
     """Output data type of pipelines"""
 

From fd10393927879904e654bc5178dce524c0d6ad57 Mon Sep 17 00:00:00 2001
From: Weixuan Fu 
Date: Tue, 21 Mar 2017 17:01:49 -0400
Subject: [PATCH 143/154] slove issue 379 and 380

---
 tpot/decorators.py     | 2 +-
 tpot/operator_utils.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tpot/decorators.py b/tpot/decorators.py
index 5670b63c..07e2dcab 100644
--- a/tpot/decorators.py
+++ b/tpot/decorators.py
@@ -116,8 +116,8 @@ def limitedTime(*args, **kwargs):
                 if tmp_it.isAlive():
                     raise TimedOutExc('Time Out!')
                 sys.tracebacklimit = 1000
-                return tmp_it.result
                 tmp_it.stop()
+                return tmp_it.result
         return limitedTime
     return wrap_func
 
diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py
index e2a482a3..f76a4afd 100644
--- a/tpot/operator_utils.py
+++ b/tpot/operator_utils.py
@@ -230,7 +230,7 @@ def export(cls, *args):
                     op_arguments.append("{}={}".format(aname_split[-1], arg_value))
                 else: # parameter of internal operator as a parameter in the operator, usually in Selector
                     if not list(dep_op_list.values()).count(aname_split[1]):
-                        raise TypeError('Warning: the {} is not in right format!'.format(self.sklearn_class.__name__))
+                        raise TypeError('Warning: the operator {} is not in right format in the operator dictionary'.format(aname_split[0]))
                     else:
                         if aname_split[1] not in dep_op_arguments:
                             dep_op_arguments[aname_split[1]] = []

From 72a083ec2662a2ad2b466354ada39a1181ad47bf Mon Sep 17 00:00:00 2001
From: Randy Olson 
Date: Wed, 22 Mar 2017 11:55:54 -0400
Subject: [PATCH 144/154] Fixed #378

---
 mkdocs.yml                 | 2 +-
 tpot/__init__.py           | 2 +-
 tpot/_version.py           | 2 +-
 tpot/base.py               | 2 +-
 tpot/built_in_operators.py | 2 +-
 tpot/config_classifier.py  | 2 +-
 tpot/config_regressor.py   | 4 +++-
 tpot/decorators.py         | 2 +-
 tpot/driver.py             | 2 +-
 tpot/export_utils.py       | 2 +-
 tpot/gp_deap.py            | 6 ++++--
 tpot/gp_types.py           | 2 +-
 tpot/metrics.py            | 2 +-
 tpot/operator_utils.py     | 2 +-
 tpot/tpot.py               | 2 +-
 15 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/mkdocs.yml b/mkdocs.yml
index 184019bb..5006c1ff 100755
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -13,7 +13,7 @@ markdown_extensions:
   - tables
   - fenced_code
 
-copyright: Copyright © 2016-Present Randal S. Olson
+copyright: Copyright © 2015-Present Randal S. Olson
 
 pages:
 - Home: index.md
diff --git a/tpot/__init__.py b/tpot/__init__.py
index 86bcc7df..4c2b536d 100644
--- a/tpot/__init__.py
+++ b/tpot/__init__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/_version.py b/tpot/_version.py
index 1f9dc9ed..9e61c526 100644
--- a/tpot/_version.py
+++ b/tpot/_version.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/base.py b/tpot/base.py
index ee30480d..518549cf 100644
--- a/tpot/base.py
+++ b/tpot/base.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/built_in_operators.py b/tpot/built_in_operators.py
index 0641aeb9..7cce351c 100644
--- a/tpot/built_in_operators.py
+++ b/tpot/built_in_operators.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/config_classifier.py b/tpot/config_classifier.py
index dbb37bf9..2734389d 100644
--- a/tpot/config_classifier.py
+++ b/tpot/config_classifier.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/config_regressor.py b/tpot/config_regressor.py
index aec94489..341963b0 100644
--- a/tpot/config_regressor.py
+++ b/tpot/config_regressor.py
@@ -1,5 +1,7 @@
+# -*- coding: utf-8 -*-
+
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/decorators.py b/tpot/decorators.py
index 07e2dcab..964519e2 100644
--- a/tpot/decorators.py
+++ b/tpot/decorators.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/driver.py b/tpot/driver.py
index 3518eb21..6da1d58d 100644
--- a/tpot/driver.py
+++ b/tpot/driver.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/export_utils.py b/tpot/export_utils.py
index 02708317..119cde67 100644
--- a/tpot/export_utils.py
+++ b/tpot/export_utils.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py
index 396cb3d6..884a08b7 100644
--- a/tpot/gp_deap.py
+++ b/tpot/gp_deap.py
@@ -1,7 +1,9 @@
+# -*- coding: utf-8 -*-
+
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
-his file is modified based on codes for alogrithms.eaSimple module in DEAP.
+This file is modified based on codes for alogrithms.eaSimple module in DEAP.
 
 This file is part of the TPOT library.
 
diff --git a/tpot/gp_types.py b/tpot/gp_types.py
index a305f545..b887024a 100644
--- a/tpot/gp_types.py
+++ b/tpot/gp_types.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/metrics.py b/tpot/metrics.py
index 303f91e9..bc16b5a4 100644
--- a/tpot/metrics.py
+++ b/tpot/metrics.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py
index f76a4afd..b4d8301e 100644
--- a/tpot/operator_utils.py
+++ b/tpot/operator_utils.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 
diff --git a/tpot/tpot.py b/tpot/tpot.py
index 3fe3f43e..ea3a3d59 100644
--- a/tpot/tpot.py
+++ b/tpot/tpot.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 """
-Copyright 2016 Randal S. Olson
+Copyright 2015-Present Randal S. Olson
 
 This file is part of the TPOT library.
 

From c434da1dca62c54329330cbcfecbe9b45d6de959 Mon Sep 17 00:00:00 2001
From: Randy Olson 
Date: Wed, 22 Mar 2017 11:56:12 -0400
Subject: [PATCH 145/154] Fixed #378 for docs

---
 docs/citing/index.html                        |  2 +-
 docs/contributing/index.html                  |  2 +-
 docs/examples/Boston_Example/index.html       |  2 +-
 docs/examples/IRIS_Example/index.html         |  2 +-
 docs/examples/MNIST_Example/index.html        |  2 +-
 .../Titanic_Kaggle_Example/index.html         |  2 +-
 docs/index.html                               |  4 ++--
 docs/installing/index.html                    |  2 +-
 docs/releases/index.html                      |  2 +-
 docs/search.html                              |  2 +-
 docs/sitemap.xml                              | 22 +++++++++----------
 docs/support/index.html                       |  2 +-
 docs/using/index.html                         |  2 +-
 13 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/docs/citing/index.html b/docs/citing/index.html
index 2bf6509e..18ab9308 100644
--- a/docs/citing/index.html
+++ b/docs/citing/index.html
@@ -230,7 +230,7 @@
   
      
     
     
-      
      Copyright © 2016-Present Randal S. Olson
      
+      
      Copyright © 2015-Present Randal S. Olson
      
     
   
      
 
diff --git a/docs/contributing/index.html b/docs/contributing/index.html
index 745e479d..7915234d 100644
--- a/docs/contributing/index.html
+++ b/docs/contributing/index.html
@@ -324,7 +324,7 @@ 
      After submitting your pull request
     
     
-      
      Copyright © 2016-Present Randal S. Olson
      
+      
      Copyright © 2015-Present Randal S. Olson
    diff --git a/docs/examples/Boston_Example/index.html b/docs/examples/Boston_Example/index.html index 6e59f3a8..d779e130 100644 --- a/docs/examples/Boston_Example/index.html +++ b/docs/examples/Boston_Example/index.html @@ -233,7 +233,7 @@
    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    diff --git a/docs/examples/IRIS_Example/index.html b/docs/examples/IRIS_Example/index.html index 2013e590..54f7f7c0 100644 --- a/docs/examples/IRIS_Example/index.html +++ b/docs/examples/IRIS_Example/index.html @@ -235,7 +235,7 @@
    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    diff --git a/docs/examples/MNIST_Example/index.html b/docs/examples/MNIST_Example/index.html index ed55ee69..1c7da702 100644 --- a/docs/examples/MNIST_Example/index.html +++ b/docs/examples/MNIST_Example/index.html @@ -230,7 +230,7 @@
    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    diff --git a/docs/examples/Titanic_Kaggle_Example/index.html b/docs/examples/Titanic_Kaggle_Example/index.html index 5a2c2c88..da5bb935 100644 --- a/docs/examples/Titanic_Kaggle_Example/index.html +++ b/docs/examples/Titanic_Kaggle_Example/index.html @@ -195,7 +195,7 @@
    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    diff --git a/docs/index.html b/docs/index.html index 91ec7b8e..360fb7a8 100644 --- a/docs/index.html +++ b/docs/index.html @@ -213,7 +213,7 @@
    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    @@ -245,5 +245,5 @@ diff --git a/docs/installing/index.html b/docs/installing/index.html index febd3e78..9c21b071 100644 --- a/docs/installing/index.html +++ b/docs/installing/index.html @@ -234,7 +234,7 @@
    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    diff --git a/docs/releases/index.html b/docs/releases/index.html index b0d329f4..442b6cae 100644 --- a/docs/releases/index.html +++ b/docs/releases/index.html @@ -337,7 +337,7 @@

    Version 0.1

    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    diff --git a/docs/search.html b/docs/search.html index c72c7332..624d949e 100644 --- a/docs/search.html +++ b/docs/search.html @@ -180,7 +180,7 @@

    Search Results

    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    diff --git a/docs/sitemap.xml b/docs/sitemap.xml index f515c0f4..3c43bc1f 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -4,7 +4,7 @@ http://rhiever.github.io/tpot/ - 2017-03-21 + 2017-03-22 daily @@ -12,7 +12,7 @@ http://rhiever.github.io/tpot/installing/ - 2017-03-21 + 2017-03-22 daily @@ -20,7 +20,7 @@ http://rhiever.github.io/tpot/using/ - 2017-03-21 + 2017-03-22 daily @@ -29,25 +29,25 @@ http://rhiever.github.io/tpot/examples/MNIST_Example/ - 2017-03-21 + 2017-03-22 daily http://rhiever.github.io/tpot/examples/IRIS_Example/ - 2017-03-21 + 2017-03-22 daily http://rhiever.github.io/tpot/examples/Boston_Example/ - 2017-03-21 + 2017-03-22 daily http://rhiever.github.io/tpot/examples/Titanic_Kaggle_Example/ - 2017-03-21 + 2017-03-22 daily @@ -56,7 +56,7 @@ http://rhiever.github.io/tpot/contributing/ - 2017-03-21 + 2017-03-22 daily @@ -64,7 +64,7 @@ http://rhiever.github.io/tpot/releases/ - 2017-03-21 + 2017-03-22 daily @@ -72,7 +72,7 @@ http://rhiever.github.io/tpot/citing/ - 2017-03-21 + 2017-03-22 daily @@ -80,7 +80,7 @@ http://rhiever.github.io/tpot/support/ - 2017-03-21 + 2017-03-22 daily diff --git a/docs/support/index.html b/docs/support/index.html index 5d97923e..b5bbc0b9 100644 --- a/docs/support/index.html +++ b/docs/support/index.html @@ -190,7 +190,7 @@
    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    diff --git a/docs/using/index.html b/docs/using/index.html index bc2f684e..e52edcf0 100644 --- a/docs/using/index.html +++ b/docs/using/index.html @@ -508,7 +508,7 @@

    Scoring functions

    -

    Copyright © 2016-Present Randal S. Olson

    +

    Copyright © 2015-Present Randal S. Olson

    From cebd82044b7c78878182f36ab7235fa207c997fc Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Wed, 22 Mar 2017 12:13:57 -0400 Subject: [PATCH 146/154] add warning messge --- tpot/base.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tpot/base.py b/tpot/base.py index ee30480d..dd470233 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -28,6 +28,7 @@ from datetime import datetime from pathos.multiprocessing import ProcessPool + import numpy as np import deap from deap import base, creator, tools, gp @@ -39,6 +40,7 @@ from sklearn.preprocessing import FunctionTransformer from sklearn.ensemble import VotingClassifier from sklearn.metrics.scorer import make_scorer +from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier from update_checker import update_check @@ -110,7 +112,8 @@ def __init__(self, generations=100, population_size=100, offspring_size=None, TPOT assumes that this scoring function should be maximized, i.e., higher is better. - Offers the same options as sklearn.model_selection.cross_val_score: + Offers the same options as sklearn.model_selection.cross_val_score as well as + a built-in score "balanced_accuracy": ['accuracy', 'adjusted_rand_score', 'average_precision', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', @@ -233,6 +236,9 @@ def __init__(self, generations=100, population_size=100, offspring_size=None, SCORERS[scoring_name] = make_scorer(scoring, greater_is_better=greater_is_better) self.scoring_function = scoring_name else: + if scoring not in SCORERS: + raise TypeError('The scoring function {} is not available. ' + 'Please choose scoring function on TPOT manual'.format(scoring)) self.scoring_function = scoring self.cv = cv @@ -335,6 +341,18 @@ def fit(self, features, classes, sample_weight=None): """ features = features.astype(np.float64) + # check input data format + if self.classification: + clf = DecisionTreeClassifier(max_depth=5) + else: + clf = DecisionTreeRegressor(max_depth=5) + + try: + clf = clf.fit(features, classes) + except: + raise TypeError('Warning: TypeError in input dataset. Please check your data format! \n' + 'Tips: features need to a 2-D array but classes should be a 1-D array.') + # Set the seed for the GP run if self.random_state is not None: random.seed(self.random_state) # deap use random From 11ffba5255150954e687b11859d9c307287133fb Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Wed, 22 Mar 2017 14:06:31 -0400 Subject: [PATCH 147/154] Minor code cleanup for many of the exceptions we raise during runtime --- docs_sources/using.md | 56 ++++++++++++++++++++----------------------- tpot/base.py | 34 +++++++++++++------------- 2 files changed, 44 insertions(+), 46 deletions(-) diff --git a/docs_sources/using.md b/docs_sources/using.md index a6d6d8e1..3ba7b158 100644 --- a/docs_sources/using.md +++ b/docs_sources/using.md @@ -109,22 +109,7 @@ TPOT offers several arguments that can be provided at the command line: -config CONFIG_FILE String path to a file -Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like: -
    -classifier_config_dict = {
-    'sklearn.naive_bayes.GaussianNB': {
-    },
-    'sklearn.naive_bayes.BernoulliNB': {
-        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
-        'fit_prior': [True, False]
-    },
-    'sklearn.naive_bayes.MultinomialNB': {
-        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
-        'fit_prior': [True, False]
-    }
-}
-
    - +Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples. -v @@ -241,20 +226,7 @@ Note that you can pass several parameters to the TPOT instantiation call: config_dict Python dictionary -Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example: -
    -classifier_config_dict = {
-    'sklearn.naive_bayes.GaussianNB': {
-    },
-    'sklearn.naive_bayes.BernoulliNB': {
-        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
-        'fit_prior': [True, False]
-    },
-    'sklearn.naive_bayes.MultinomialNB': {
-        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
-        'fit_prior': [True, False]
-    }
-}
+Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples.
    @@ -332,3 +304,27 @@ TPOT makes use of `sklearn.model_selection.cross_val_score`, and as such offers def accuracy(y_true, y_pred): return float(sum(y_pred == y_true)) / len(y_true) ``` + + +## Customizing TPOT's operators and parameters + +TPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, sometimes it's useful to limit the algorithms and parameters that TPOT explores + +For example, the configuration file's format could be like: + +
    +classifier_config_dict = {
+    'sklearn.naive_bayes.GaussianNB': {
+    },
+    'sklearn.naive_bayes.BernoulliNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    },
+    'sklearn.naive_bayes.MultinomialNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    }
+}
+
    + + diff --git a/tpot/base.py b/tpot/base.py index b922140c..76957363 100644 --- a/tpot/base.py +++ b/tpot/base.py @@ -115,8 +115,8 @@ def __init__(self, generations=100, population_size=100, offspring_size=None, Offers the same options as sklearn.model_selection.cross_val_score as well as a built-in score "balanced_accuracy": - ['accuracy', 'adjusted_rand_score', 'average_precision', 'f1', - 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', + ['accuracy', 'adjusted_rand_score', 'average_precision', 'balanced accuracy', + 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'] @@ -206,7 +206,7 @@ def __init__(self, generations=100, population_size=100, offspring_size=None, self.crossover_rate = crossover_rate if self.mutation_rate + self.crossover_rate > 1: - raise TypeError('The sum of the crossover and mutation probabilities must be <= 1.0.') + raise ValueError('The sum of the crossover and mutation probabilities must be <= 1.0.') self.verbosity = verbosity self.operators_context = { @@ -237,14 +237,15 @@ def __init__(self, generations=100, population_size=100, offspring_size=None, self.scoring_function = scoring_name else: if scoring not in SCORERS: - raise TypeError('The scoring function {} is not available. ' - 'Please choose scoring function on TPOT manual'.format(scoring)) + raise ValueError('The scoring function {} is not available. ' + 'Please choose a valid scoring function from the TPOT ' + 'documentation.'.format(scoring)) self.scoring_function = scoring self.cv = cv # If the OS is windows, reset cpu number to 1 since the OS did not have multiprocessing module if sys.platform.startswith('win') and n_jobs > 1: - print('Warning: Parallelization is not currently supported in TPOT for Windows.', + print('Warning: Parallelization is not currently supported in TPOT for Windows. ', 'Setting n_jobs to 1 during the TPOT optimization process.') self.n_jobs = 1 else: @@ -341,7 +342,7 @@ def fit(self, features, classes, sample_weight=None): """ features = features.astype(np.float64) - # check input data format + # Check that the input data is formatted correctly for scikit-learn if self.classification: clf = DecisionTreeClassifier(max_depth=5) else: @@ -350,12 +351,14 @@ def fit(self, features, classes, sample_weight=None): try: clf = clf.fit(features, classes) except: - raise TypeError('Warning: TypeError in input dataset. Please check your data format! \n' - 'Tips: features need to a 2-D array but classes should be a 1-D array.') + raise ValueError('Error: Input data is not in a valid format. ' + 'Please confirm that the input data is scikit-learn compatible. ' + 'For example, the features must be a 2-D array and target labels ' + 'must be a 1-D array.') # Set the seed for the GP run if self.random_state is not None: - random.seed(self.random_state) # deap use random + random.seed(self.random_state) # deap uses random np.random.seed(self.random_state) self._start_datetime = datetime.now() @@ -479,7 +482,7 @@ def predict(self, features): """ if not self._fitted_pipeline: - raise ValueError('A pipeline has not yet been optimized. Please call fit() first.') + raise RuntimeError('A pipeline has not yet been optimized. Please call fit() first.') return self._fitted_pipeline.predict(features.astype(np.float64)) def fit_predict(self, features, classes): @@ -519,8 +522,7 @@ def score(self, testing_features, testing_classes): """ if self._fitted_pipeline is None: - raise ValueError('A pipeline has not yet been optimized. ' - 'Please call fit() first.') + raise RuntimeError('A pipeline has not yet been optimized. Please call fit() first.') # If the scoring function is a string, we must adjust to use the sklearn scoring interface return abs(SCORERS[self.scoring_function](self._fitted_pipeline, @@ -541,10 +543,10 @@ def predict_proba(self, features): """ if not self._fitted_pipeline: - raise ValueError('A pipeline has not yet been optimized. Please call fit() first.') + raise RuntimeError('A pipeline has not yet been optimized. Please call fit() first.') else: if not(hasattr(self._fitted_pipeline, 'predict_proba')): - raise ValueError('The fitted pipeline does not have probability prediction functionality') + raise RuntimeError('The fitted pipeline does not have the predict_proba() function.') return self._fitted_pipeline.predict_proba(features.astype(np.float64)) def set_params(self, **params): @@ -572,7 +574,7 @@ def export(self, output_file_name): """ if self._optimized_pipeline is None: - raise ValueError('A pipeline has not yet been optimized. Please call fit() first.') + raise RuntimeError('A pipeline has not yet been optimized. Please call fit() first.') with open(output_file_name, 'w') as output_file: output_file.write(export_pipeline(self._optimized_pipeline, self.operators, self._pset)) From c160817e8b93b0d8205d03c6d67da989ba79dfa3 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Wed, 22 Mar 2017 14:30:23 -0400 Subject: [PATCH 148/154] Fixed #377 --- docs/index.html | 2 +- docs/mkdocs/search_index.json | 11 +++-- docs/using/index.html | 89 +++++++++++++++++++++++------------ docs_sources/using.md | 49 +++++++++++++++++-- 4 files changed, 113 insertions(+), 38 deletions(-) diff --git a/docs/index.html b/docs/index.html index 360fb7a8..f04dfc07 100644 --- a/docs/index.html +++ b/docs/index.html @@ -245,5 +245,5 @@ diff --git a/docs/mkdocs/search_index.json b/docs/mkdocs/search_index.json index 3c20f11e..c0d833dc 100644 --- a/docs/mkdocs/search_index.json +++ b/docs/mkdocs/search_index.json @@ -12,17 +12,17 @@ }, { "location": "/using/", - "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like:\n\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\n\n\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example:\n\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)", + "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process. See the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)\n\n\n\n\n\n\nCustomizing TPOT's operators and parameters\n\n\nTPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters.\n\n\nThe custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., \nsklearn.naive_bayes.MultinomialNB\n) and the second level key is the corresponding parameter name for that operator (e.g., \nfit_prior\n). The second level key should point to a list of parameter values for that parameter, e.g., \n'fit_prior': [True, False]\n.\n\n\nFor a simple example, the configuration could be:\n\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\n\n\n\nin which case TPOT would only explore pipelines containing \nGaussianNB\n, \nBernoulliNB\n, \nMultinomialNB\n, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the \nTPOTClassifier\n/\nTPOTRegressor\n \nconfig_dict\n parameter, described above. For example:\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n train_size=0.75, test_size=0.25)\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n config_dict=classifier_config_dict)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\nCommand-line users must create a separate \n.py\n file with the custom configuration and provide the path to the file to the \ntpot\n call. For example, if the simple example configuration above is saved in \ntpot_classifier_config.py\n, that configuration could be used on the command line with the command:\n\n\ntpot data/mnist.csv -is , -target class -config tpot_classifier_config.py -g 5 -p 20 -v 2 -o tpot_exported_pipeline.py\n\n\n\n\nFor more detailed examples of how to customize TPOT's operator configuration, see the default configurations for \nclassification\n and \nregression\n in TPOT's source code.\n\n\nNote that you must have all of the corresponding packages for the operators installed on your computer, otherwise TPOT will not be able to use them. For example, if XGBoost is not installed on your computer, then TPOT will simply not import nor use XGBoost in the pipelines it explores.", "title": "Using TPOT" }, { "location": "/using/#tpot-on-the-command-line", - "text": "To use TPOT via the command line, enter the following command with a path to the data file: tpot /path_to/data_file.csv TPOT offers several arguments that can be provided at the command line: Argument Parameter Valid values Effect -is INPUT_SEPARATOR Any string Character used to separate columns in the input file. -target TARGET_NAME Any string Name of the target column in the input file. -mode TPOT_MODE ['classification', 'regression'] Whether TPOT is being used for a supervised classification or regression problem. -o OUTPUT_FILE String path to a file File to export the code for the final optimized pipeline. -g GENERATIONS Any positive integer Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. -p POPULATION_SIZE Any positive integer Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. -os OFFSPRING_SIZE Any positive integer Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE. -mr MUTATION_RATE [0.0, 1.0] GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. -xr CROSSOVER_RATE [0.0, 1.0] GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms. -scoring SCORING_FN 'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. -cv NUM_CV_FOLDS Any integer >1 Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process. -njobs NUM_JOBS Any positive integer or -1 Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. -maxtime MAX_TIME_MINS Any positive integer How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time. -maxeval MAX_EVAL_MINS Any positive integer How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. -s RANDOM_STATE Any positive integer Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future. -config CONFIG_FILE String path to a file Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like: \nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n} -v VERBOSITY {0, 1, 2, 3} How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. --no-update-check N/A Flag indicating whether the TPOT version checker should be disabled. --version N/A Show TPOT's version number and exit. --help N/A Show TPOT's help documentation and exit. An example command-line call to TPOT may look like: tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2", + "text": "To use TPOT via the command line, enter the following command with a path to the data file: tpot /path_to/data_file.csv TPOT offers several arguments that can be provided at the command line: Argument Parameter Valid values Effect -is INPUT_SEPARATOR Any string Character used to separate columns in the input file. -target TARGET_NAME Any string Name of the target column in the input file. -mode TPOT_MODE ['classification', 'regression'] Whether TPOT is being used for a supervised classification or regression problem. -o OUTPUT_FILE String path to a file File to export the code for the final optimized pipeline. -g GENERATIONS Any positive integer Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. -p POPULATION_SIZE Any positive integer Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. -os OFFSPRING_SIZE Any positive integer Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE. -mr MUTATION_RATE [0.0, 1.0] GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. -xr CROSSOVER_RATE [0.0, 1.0] GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms. -scoring SCORING_FN 'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. -cv NUM_CV_FOLDS Any integer >1 Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process. -njobs NUM_JOBS Any positive integer or -1 Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. -maxtime MAX_TIME_MINS Any positive integer How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time. -maxeval MAX_EVAL_MINS Any positive integer How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer. -s RANDOM_STATE Any positive integer Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future. -config CONFIG_FILE String path to a file Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples. -v VERBOSITY {0, 1, 2, 3} How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. --no-update-check N/A Flag indicating whether the TPOT version checker should be disabled. --version N/A Show TPOT's version number and exit. --help N/A Show TPOT's help documentation and exit. An example command-line call to TPOT may look like: tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2", "title": "TPOT on the command line" }, { "location": "/using/#tpot-with-code", - "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn. TPOT can be imported just like any regular Python module. To import TPOT, type: from tpot import TPOTClassifier then create an instance of TPOT as follows: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier() It's also possible to use TPOT for regression problems with the TPOTRegressor class. Other than the class name, a TPOTRegressor is used the same way as a TPOTClassifier . Note that you can pass several parameters to the TPOT instantiation call: Parameter Valid values Effect generations Any positive integer Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. population_size Any positive integer Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. offspring_size Any positive integer Number of offspring to produce in each GP generation. By default, offspring_size = population_size. mutation_rate [0.0, 1.0] Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. crossover_rate [0.0, 1.0] Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. cv Any integer >1 Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process. n_jobs Any positive integer or -1 Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. max_time_mins Any positive integer How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time. max_eval_time_mins Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer. random_state Any positive integer Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. config_dict Python dictionary Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example: \nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n} warm_start [True, False] Flag indicating whether the TPOT instance will reuse the population from previous calls to fit(). verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. Some example code with custom TPOT parameters might look like: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2) Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes) The fit() function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score. You can then proceed to evaluate the final pipeline on the testing set with the score() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes)) Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py') Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline. Check our examples to see TPOT applied to some specific data sets.", + "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn. TPOT can be imported just like any regular Python module. To import TPOT, type: from tpot import TPOTClassifier then create an instance of TPOT as follows: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier() It's also possible to use TPOT for regression problems with the TPOTRegressor class. Other than the class name, a TPOTRegressor is used the same way as a TPOTClassifier . Note that you can pass several parameters to the TPOT instantiation call: Parameter Valid values Effect generations Any positive integer Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. population_size Any positive integer Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. offspring_size Any positive integer Number of offspring to produce in each GP generation. By default, offspring_size = population_size. mutation_rate [0.0, 1.0] Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. crossover_rate [0.0, 1.0] Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. cv Any integer >1 Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process. n_jobs Any positive integer or -1 Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. max_time_mins Any positive integer How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time. max_eval_time_mins Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer. random_state Any positive integer Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. config_dict Python dictionary Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples. warm_start [True, False] Flag indicating whether the TPOT instance will reuse the population from previous calls to fit(). verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. Some example code with custom TPOT parameters might look like: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2) Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes) The fit() function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score. You can then proceed to evaluate the final pipeline on the testing set with the score() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes)) Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py') Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline. Check our examples to see TPOT applied to some specific data sets.", "title": "TPOT with code" }, { @@ -30,6 +30,11 @@ "text": "TPOT makes use of sklearn.model_selection.cross_val_score , and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT: You can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line. You can pass in a function with the signature scorer(y_true, y_pred) , where y_true are the true target values and y_pred are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation. def accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)", "title": "Scoring functions" }, + { + "location": "/using/#customizing-tpots-operators-and-parameters", + "text": "TPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters. The custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., sklearn.naive_bayes.MultinomialNB ) and the second level key is the corresponding parameter name for that operator (e.g., fit_prior ). The second level key should point to a list of parameter values for that parameter, e.g., 'fit_prior': [True, False] . For a simple example, the configuration could be: classifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n} in which case TPOT would only explore pipelines containing GaussianNB , BernoulliNB , MultinomialNB , and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the TPOTClassifier / TPOTRegressor config_dict parameter, described above. For example: from tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n train_size=0.75, test_size=0.25)\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n config_dict=classifier_config_dict)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py') Command-line users must create a separate .py file with the custom configuration and provide the path to the file to the tpot call. For example, if the simple example configuration above is saved in tpot_classifier_config.py , that configuration could be used on the command line with the command: tpot data/mnist.csv -is , -target class -config tpot_classifier_config.py -g 5 -p 20 -v 2 -o tpot_exported_pipeline.py For more detailed examples of how to customize TPOT's operator configuration, see the default configurations for classification and regression in TPOT's source code. Note that you must have all of the corresponding packages for the operators installed on your computer, otherwise TPOT will not be able to use them. For example, if XGBoost is not installed on your computer, then TPOT will simply not import nor use XGBoost in the pipelines it explores.", + "title": "Customizing TPOT's operators and parameters" + }, { "location": "/examples/MNIST_Example/", "text": "Below is a minimal working example with the practice MNIST data set.\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n train_size=0.75, test_size=0.25)\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nRunning this code should discover a pipeline that achieves about 98% testing accuracy, and the corresponding Python code should be exported to the \ntpot_mnist_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.pipeline import make_pipeline\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR')\nfeatures = tpot_data.view((np.float64, len(tpot_data.dtype.names)))\nfeatures = np.delete(features, tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes = train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = make_pipeline(\n KNeighborsClassifier(n_neighbors=3, weights=\"uniform\")\n)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)", diff --git a/docs/using/index.html b/docs/using/index.html index e52edcf0..cff5c284 100644 --- a/docs/using/index.html +++ b/docs/using/index.html @@ -73,6 +73,8 @@
  • Scoring functions
    • +
  • Customizing TPOT's operators and parameters
    • +
@@ -285,22 +287,7 @@

TPOT on the command line

-config CONFIG_FILE String path to a file -Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. For example, the configuration file's format could be like: -
-classifier_config_dict = {
-    'sklearn.naive_bayes.GaussianNB': {
-    },
-    'sklearn.naive_bayes.BernoulliNB': {
-        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
-        'fit_prior': [True, False]
-    },
-    'sklearn.naive_bayes.MultinomialNB': {
-        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
-        'fit_prior': [True, False]
-    }
-}
-
- +Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples. -v @@ -407,20 +394,7 @@

TPOT with code

config_dict Python dictionary -Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. For example: -
-classifier_config_dict = {
-    'sklearn.naive_bayes.GaussianNB': {
-    },
-    'sklearn.naive_bayes.BernoulliNB': {
-        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
-        'fit_prior': [True, False]
-    },
-    'sklearn.naive_bayes.MultinomialNB': {
-        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
-        'fit_prior': [True, False]
-    }
-}
+Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples.
@@ -488,6 +462,61 @@

Scoring functions

def accuracy(y_true, y_pred):
     return float(sum(y_pred == y_true)) / len(y_true)
+ +

+

Customizing TPOT's operators and parameters

+

TPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters.

+

The custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., sklearn.naive_bayes.MultinomialNB) and the second level key is the corresponding parameter name for that operator (e.g., fit_prior). The second level key should point to a list of parameter values for that parameter, e.g., 'fit_prior': [True, False].

+

For a simple example, the configuration could be:

+
classifier_config_dict = {
+    'sklearn.naive_bayes.GaussianNB': {
+    },
+    'sklearn.naive_bayes.BernoulliNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    },
+    'sklearn.naive_bayes.MultinomialNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    }
+}
+
+ +

in which case TPOT would only explore pipelines containing GaussianNB, BernoulliNB, MultinomialNB, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the TPOTClassifier/TPOTRegressor config_dict parameter, described above. For example:

+
from tpot import TPOTClassifier
+from sklearn.datasets import load_digits
+from sklearn.model_selection import train_test_split
+
+digits = load_digits()
+X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,
+                                                    train_size=0.75, test_size=0.25)
+
+classifier_config_dict = {
+    'sklearn.naive_bayes.GaussianNB': {
+    },
+    'sklearn.naive_bayes.BernoulliNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    },
+    'sklearn.naive_bayes.MultinomialNB': {
+        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],
+        'fit_prior': [True, False]
+    }
+}
+
+tpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,
+                      config_dict=classifier_config_dict)
+tpot.fit(X_train, y_train)
+print(tpot.score(X_test, y_test))
+tpot.export('tpot_mnist_pipeline.py')
+
+ +

Command-line users must create a separate .py file with the custom configuration and provide the path to the file to the tpot call. For example, if the simple example configuration above is saved in tpot_classifier_config.py, that configuration could be used on the command line with the command:

+
tpot data/mnist.csv -is , -target class -config tpot_classifier_config.py -g 5 -p 20 -v 2 -o tpot_exported_pipeline.py
+
+ +

For more detailed examples of how to customize TPOT's operator configuration, see the default configurations for classification and regression in TPOT's source code.

+

Note that you must have all of the corresponding packages for the operators installed on your computer, otherwise TPOT will not be able to use them. For example, if XGBoost is not installed on your computer, then TPOT will simply not import nor use XGBoost in the pipelines it explores.

diff --git a/docs_sources/using.md b/docs_sources/using.md index 3ba7b158..2271dd21 100644 --- a/docs_sources/using.md +++ b/docs_sources/using.md @@ -308,11 +308,13 @@ def accuracy(y_true, y_pred): ## Customizing TPOT's operators and parameters -TPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, sometimes it's useful to limit the algorithms and parameters that TPOT explores +TPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters. -For example, the configuration file's format could be like: +The custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., `sklearn.naive_bayes.MultinomialNB`) and the second level key is the corresponding parameter name for that operator (e.g., `fit_prior`). The second level key should point to a list of parameter values for that parameter, e.g., `'fit_prior': [True, False]`. -
+For a simple example, the configuration could be:
+
+```
 classifier_config_dict = {
     'sklearn.naive_bayes.GaussianNB': {
     },
@@ -325,6 +327,45 @@ classifier_config_dict = {
         'fit_prior': [True, False]
     }
 }
-
+``` + +in which case TPOT would only explore pipelines containing `GaussianNB`, `BernoulliNB`, `MultinomialNB`, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the `TPOTClassifier`/`TPOTRegressor` `config_dict` parameter, described above. For example: + +``` +from tpot import TPOTClassifier +from sklearn.datasets import load_digits +from sklearn.model_selection import train_test_split + +digits = load_digits() +X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target, + train_size=0.75, test_size=0.25) + +classifier_config_dict = { + 'sklearn.naive_bayes.GaussianNB': { + }, + 'sklearn.naive_bayes.BernoulliNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + }, + 'sklearn.naive_bayes.MultinomialNB': { + 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.], + 'fit_prior': [True, False] + } +} + +tpot = TPOTClassifier(generations=5, population_size=20, verbosity=2, + config_dict=classifier_config_dict) +tpot.fit(X_train, y_train) +print(tpot.score(X_test, y_test)) +tpot.export('tpot_mnist_pipeline.py') +``` + +Command-line users must create a separate `.py` file with the custom configuration and provide the path to the file to the `tpot` call. For example, if the simple example configuration above is saved in `tpot_classifier_config.py`, that configuration could be used on the command line with the command: + +``` +tpot data/mnist.csv -is , -target class -config tpot_classifier_config.py -g 5 -p 20 -v 2 -o tpot_exported_pipeline.py +``` +For more detailed examples of how to customize TPOT's operator configuration, see the default configurations for [classification](https://github.com/rhiever/tpot/blob/master/tpot/config_classifier.py) and [regression](https://github.com/rhiever/tpot/blob/master/tpot/config_regressor.py) in TPOT's source code. +Note that you must have all of the corresponding packages for the operators installed on your computer, otherwise TPOT will not be able to use them. For example, if XGBoost is not installed on your computer, then TPOT will simply not import nor use XGBoost in the pipelines it explores. From 0920fc911fa43fc337851ede19c1152897330162 Mon Sep 17 00:00:00 2001 From: Randy Olson Date: Wed, 22 Mar 2017 14:39:37 -0400 Subject: [PATCH 149/154] Expand the docs for the custom scoring functions --- docs/index.html | 2 +- docs/mkdocs/search_index.json | 6 +++--- docs/using/index.html | 38 ++++++++++++++++++++++++++--------- docs_sources/using.md | 36 ++++++++++++++++++++++++--------- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/docs/index.html b/docs/index.html index f04dfc07..360e3fc1 100644 --- a/docs/index.html +++ b/docs/index.html @@ -245,5 +245,5 @@ diff --git a/docs/mkdocs/search_index.json b/docs/mkdocs/search_index.json index c0d833dc..e4eb83ed 100644 --- a/docs/mkdocs/search_index.json +++ b/docs/mkdocs/search_index.json @@ -12,7 +12,7 @@ }, { "location": "/using/", - "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process. See the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.\n\n\n\n\n\n\nYou can pass in a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)\n\n\n\n\n\n\nCustomizing TPOT's operators and parameters\n\n\nTPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters.\n\n\nThe custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., \nsklearn.naive_bayes.MultinomialNB\n) and the second level key is the corresponding parameter name for that operator (e.g., \nfit_prior\n). The second level key should point to a list of parameter values for that parameter, e.g., \n'fit_prior': [True, False]\n.\n\n\nFor a simple example, the configuration could be:\n\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\n\n\n\nin which case TPOT would only explore pipelines containing \nGaussianNB\n, \nBernoulliNB\n, \nMultinomialNB\n, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the \nTPOTClassifier\n/\nTPOTRegressor\n \nconfig_dict\n parameter, described above. For example:\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n train_size=0.75, test_size=0.25)\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n config_dict=classifier_config_dict)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\nCommand-line users must create a separate \n.py\n file with the custom configuration and provide the path to the file to the \ntpot\n call. For example, if the simple example configuration above is saved in \ntpot_classifier_config.py\n, that configuration could be used on the command line with the command:\n\n\ntpot data/mnist.csv -is , -target class -config tpot_classifier_config.py -g 5 -p 20 -v 2 -o tpot_exported_pipeline.py\n\n\n\n\nFor more detailed examples of how to customize TPOT's operator configuration, see the default configurations for \nclassification\n and \nregression\n in TPOT's source code.\n\n\nNote that you must have all of the corresponding packages for the operators installed on your computer, otherwise TPOT will not be able to use them. For example, if XGBoost is not installed on your computer, then TPOT will simply not import nor use XGBoost in the pipelines it explores.", + "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process. See the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n for evaluating pipelines, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string to the \nscoring\n parameter from the list above. Any other strings will cause TPOT to throw an exception.\n\n\n\n\n\n\nYou can pass a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n train_size=0.75, test_size=0.25)\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n scoring=accuracy)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\n\n\nCustomizing TPOT's operators and parameters\n\n\nTPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters.\n\n\nThe custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., \nsklearn.naive_bayes.MultinomialNB\n) and the second level key is the corresponding parameter name for that operator (e.g., \nfit_prior\n). The second level key should point to a list of parameter values for that parameter, e.g., \n'fit_prior': [True, False]\n.\n\n\nFor a simple example, the configuration could be:\n\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\n\n\n\nin which case TPOT would only explore pipelines containing \nGaussianNB\n, \nBernoulliNB\n, \nMultinomialNB\n, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the \nTPOTClassifier\n/\nTPOTRegressor\n \nconfig_dict\n parameter, described above. For example:\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n train_size=0.75, test_size=0.25)\n\nclassifier_config_dict = {\n 'sklearn.naive_bayes.GaussianNB': {\n },\n 'sklearn.naive_bayes.BernoulliNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n },\n 'sklearn.naive_bayes.MultinomialNB': {\n 'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n 'fit_prior': [True, False]\n }\n}\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n config_dict=classifier_config_dict)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\nCommand-line users must create a separate \n.py\n file with the custom configuration and provide the path to the file to the \ntpot\n call. For example, if the simple example configuration above is saved in \ntpot_classifier_config.py\n, that configuration could be used on the command line with the command:\n\n\ntpot data/mnist.csv -is , -target class -config tpot_classifier_config.py -g 5 -p 20 -v 2 -o tpot_exported_pipeline.py\n\n\n\n\nFor more detailed examples of how to customize TPOT's operator configuration, see the default configurations for \nclassification\n and \nregression\n in TPOT's source code.\n\n\nNote that you must have all of the corresponding packages for the operators installed on your computer, otherwise TPOT will not be able to use them. For example, if XGBoost is not installed on your computer, then TPOT will simply not import nor use XGBoost in the pipelines it explores.", "title": "Using TPOT" }, { @@ -22,12 +22,12 @@ }, { "location": "/using/#tpot-with-code", - "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn. TPOT can be imported just like any regular Python module. To import TPOT, type: from tpot import TPOTClassifier then create an instance of TPOT as follows: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier() It's also possible to use TPOT for regression problems with the TPOTRegressor class. Other than the class name, a TPOTRegressor is used the same way as a TPOTClassifier . Note that you can pass several parameters to the TPOT instantiation call: Parameter Valid values Effect generations Any positive integer Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. population_size Any positive integer Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. offspring_size Any positive integer Number of offspring to produce in each GP generation. By default, offspring_size = population_size. mutation_rate [0.0, 1.0] Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. crossover_rate [0.0, 1.0] Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. cv Any integer >1 Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process. n_jobs Any positive integer or -1 Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. max_time_mins Any positive integer How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time. max_eval_time_mins Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer. random_state Any positive integer Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. config_dict Python dictionary Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples. warm_start [True, False] Flag indicating whether the TPOT instance will reuse the population from previous calls to fit(). verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. Some example code with custom TPOT parameters might look like: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2) Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes) The fit() function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score. You can then proceed to evaluate the final pipeline on the testing set with the score() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes)) Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py') Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline. Check our examples to see TPOT applied to some specific data sets.", + "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn. TPOT can be imported just like any regular Python module. To import TPOT, type: from tpot import TPOTClassifier then create an instance of TPOT as follows: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier() It's also possible to use TPOT for regression problems with the TPOTRegressor class. Other than the class name, a TPOTRegressor is used the same way as a TPOTClassifier . Note that you can pass several parameters to the TPOT instantiation call: Parameter Valid values Effect generations Any positive integer Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. population_size Any positive integer Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total. offspring_size Any positive integer Number of offspring to produce in each GP generation. By default, offspring_size = population_size. mutation_rate [0.0, 1.0] Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. crossover_rate [0.0, 1.0] Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms. scoring 'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred) Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details. cv Any integer >1 Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process. n_jobs Any positive integer or -1 Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer. max_time_mins Any positive integer How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time. max_eval_time_mins Any positive integer How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer. random_state Any positive integer Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed. config_dict Python dictionary Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples. warm_start [True, False] Flag indicating whether the TPOT instance will reuse the population from previous calls to fit(). verbosity {0, 1, 2, 3} How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure. disable_update_check [True, False] Flag indicating whether the TPOT version checker should be disabled. Some example code with custom TPOT parameters might look like: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n random_state=42, verbosity=2) Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes) The fit() function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score. You can then proceed to evaluate the final pipeline on the testing set with the score() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes)) Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function: from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py') Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline. Check our examples to see TPOT applied to some specific data sets.", "title": "TPOT with code" }, { "location": "/using/#scoring-functions", - "text": "TPOT makes use of sklearn.model_selection.cross_val_score , and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT: You can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line. You can pass in a function with the signature scorer(y_true, y_pred) , where y_true are the true target values and y_pred are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation. def accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)", + "text": "TPOT makes use of sklearn.model_selection.cross_val_score for evaluating pipelines, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT: You can pass in a string to the scoring parameter from the list above. Any other strings will cause TPOT to throw an exception. You can pass a function with the signature scorer(y_true, y_pred) , where y_true are the true target values and y_pred are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation. from tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n train_size=0.75, test_size=0.25)\n\ndef accuracy(y_true, y_pred):\n return float(sum(y_pred == y_true)) / len(y_true)\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n scoring=accuracy)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')", "title": "Scoring functions" }, { diff --git a/docs/using/index.html b/docs/using/index.html index cff5c284..8e92d85b 100644 --- a/docs/using/index.html +++ b/docs/using/index.html @@ -418,13 +418,15 @@

TPOT with code

Some example code with custom TPOT parameters might look like:

from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
+                                    random_state=42, verbosity=2)

Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the fit function:

from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
+                                    random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
@@ -432,7 +434,8 @@

TPOT with code

You can then proceed to evaluate the final pipeline on the testing set with the score() function:

from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
+                                    random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 print(pipeline_optimizer.score(testing_features, testing_classes))
@@ -440,7 +443,8 @@

TPOT with code

Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the export() function:

from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
+                                    random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 print(pipeline_optimizer.score(testing_features, testing_classes))
 pipeline_optimizer.export('tpot_exported_pipeline.py')
@@ -450,17 +454,31 @@ 
TPOT with code

 
Check our examples to see TPOT applied to some specific data sets.

 

 
Scoring functions

-
TPOT makes use of sklearn.model_selection.cross_val_score, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:

+
TPOT makes use of sklearn.model_selection.cross_val_score for evaluating pipelines, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:

 
    
 
  1. 
-
    You can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.
    
+
    You can pass in a string to the scoring parameter from the list above. Any other strings will cause TPOT to throw an exception.
    
 
    2. 
 
  2. 
-
    You can pass in a function with the signature scorer(y_true, y_pred), where y_true are the true target values and y_pred are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.
    
+
    You can pass a function with the signature scorer(y_true, y_pred), where y_true are the true target values and y_pred are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.
    
 
    3. 
 

-
def accuracy(y_true, y_pred):
+
from tpot import TPOTClassifier
+from sklearn.datasets import load_digits
+from sklearn.model_selection import train_test_split
+
+digits = load_digits()
+X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,
+                                                    train_size=0.75, test_size=0.25)
+
+def accuracy(y_true, y_pred):
     return float(sum(y_pred == y_true)) / len(y_true)
+
+tpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,
+                      scoring=accuracy)
+tpot.fit(X_train, y_train)
+print(tpot.score(X_test, y_test))
+tpot.export('tpot_mnist_pipeline.py')
 

 
 

@@ -468,7 +486,7 @@ 
Customizing TPOT's operators
 
TPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters.

 
The custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., sklearn.naive_bayes.MultinomialNB) and the second level key is the corresponding parameter name for that operator (e.g., fit_prior). The second level key should point to a list of parameter values for that parameter, e.g., 'fit_prior': [True, False].

 
For a simple example, the configuration could be:

-
classifier_config_dict = {
+
classifier_config_dict = {
     'sklearn.naive_bayes.GaussianNB': {
     },
     'sklearn.naive_bayes.BernoulliNB': {
@@ -483,7 +501,7 @@ 
Customizing TPOT's operators
 

 
 
in which case TPOT would only explore pipelines containing GaussianNB, BernoulliNB, MultinomialNB, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the TPOTClassifier/TPOTRegressor config_dict parameter, described above. For example:

-
from tpot import TPOTClassifier
+
from tpot import TPOTClassifier
 from sklearn.datasets import load_digits
 from sklearn.model_selection import train_test_split
 
diff --git a/docs_sources/using.md b/docs_sources/using.md
index 2271dd21..f9a88d03 100644
--- a/docs_sources/using.md
+++ b/docs_sources/using.md
@@ -252,7 +252,8 @@ Some example code with custom TPOT parameters might look like:
 ```Python
 from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
+                                    random_state=42, verbosity=2)
 ```
 
 Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the `fit` function:
@@ -260,7 +261,8 @@ Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize
 ```Python
 from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
+                                    random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 ```
 
@@ -271,7 +273,8 @@ You can then proceed to evaluate the final pipeline on the testing set with the
 ```Python
 from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
+                                    random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 print(pipeline_optimizer.score(testing_features, testing_classes))
 ```
@@ -281,7 +284,8 @@ Finally, you can tell TPOT to export the corresponding Python code for the optim
 ```Python
 from tpot import TPOTClassifier
 
-pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5, random_state=42, verbosity=2)
+pipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,
+                                    random_state=42, verbosity=2)
 pipeline_optimizer.fit(training_features, training_classes)
 print(pipeline_optimizer.score(testing_features, testing_classes))
 pipeline_optimizer.export('tpot_exported_pipeline.py')
@@ -294,15 +298,29 @@ Check our [examples](examples/MNIST_Example/) to see TPOT applied to some specif
 
 ## Scoring functions
 
-TPOT makes use of `sklearn.model_selection.cross_val_score`, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:
+TPOT makes use of `sklearn.model_selection.cross_val_score` for evaluating pipelines, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:
 
-1. You can pass in a string from the list described in the table above. Any other strings will cause internal issues that may break your code down the line.
+1. You can pass in a string to the `scoring` parameter from the list above. Any other strings will cause TPOT to throw an exception.
 
-2. You can pass in a function with the signature `scorer(y_true, y_pred)`, where `y_true` are the true target values and `y_pred` are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.
+2. You can pass a function with the signature `scorer(y_true, y_pred)`, where `y_true` are the true target values and `y_pred` are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.
 
 ```Python
+from tpot import TPOTClassifier
+from sklearn.datasets import load_digits
+from sklearn.model_selection import train_test_split
+
+digits = load_digits()
+X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,
+                                                    train_size=0.75, test_size=0.25)
+
 def accuracy(y_true, y_pred):
     return float(sum(y_pred == y_true)) / len(y_true)
+
+tpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,
+                      scoring=accuracy)
+tpot.fit(X_train, y_train)
+print(tpot.score(X_test, y_test))
+tpot.export('tpot_mnist_pipeline.py')
 ```
 
 
@@ -314,7 +332,7 @@ The custom TPOT configuration must be in nested dictionary format, where the fir
 
 For a simple example, the configuration could be:
 
-```
+```Python
 classifier_config_dict = {
     'sklearn.naive_bayes.GaussianNB': {
     },
@@ -331,7 +349,7 @@ classifier_config_dict = {
 
 in which case TPOT would only explore pipelines containing `GaussianNB`, `BernoulliNB`, `MultinomialNB`, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the `TPOTClassifier`/`TPOTRegressor` `config_dict` parameter, described above. For example:
 
-```
+```Python
 from tpot import TPOTClassifier
 from sklearn.datasets import load_digits
 from sklearn.model_selection import train_test_split

From f48b31ce8cb2b13cf8c98b9c97ce013589484fc9 Mon Sep 17 00:00:00 2001
From: Weixuan Fu 
Date: Wed, 22 Mar 2017 14:43:21 -0400
Subject: [PATCH 150/154] fix and add unit tests

---
 tests.py | 38 +++++++++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/tests.py b/tests.py
index 2a46b4fa..c59b8a80 100644
--- a/tests.py
+++ b/tests.py
@@ -94,13 +94,37 @@ def test_timeout_func():
     ret_timeout = int(test_timeout_func())
     assert ret_timeout == 1
 
-
 def test_init_default_scoring():
     """Assert that TPOT intitializes with the correct default scoring function"""
 
     tpot_obj = TPOTRegressor()
     assert tpot_obj.scoring_function == 'neg_mean_squared_error'
 
+    tpot_obj = TPOTClassifier()
+    assert tpot_obj.scoring_function == 'accuracy'
+
+def test_invaild_score_warning():
+    """Assert that the TPOT fit function raises a ValueError when the scoring metrics is not available in SCORERS"""
+    try:
+        tpot_obj = TPOTClassifier(scoring='balanced_accuray') # typo for balanced_accuracy
+        assert False
+    except ValueError:
+        pass
+    try:
+        tpot_obj = TPOTClassifier(scoring='balanced_accuracy') # correct one
+        assert True
+    except:
+        assert False
+
+def test_invaild_dataset_warning():
+    """Assert that the TPOT fit function raises a ValueError when dataset is not in right format"""
+    tpot_obj = TPOTClassifier(random_state=42, population_size=1, offspring_size=2, generations=1, verbosity=0)
+    bad_training_classes = training_classes.reshape((1, len(training_classes)))# common mistake in classes
+    try:
+        tpot_obj.fit(training_features ,bad_training_classes) # typo for balanced_accuracy
+        assert False
+    except ValueError:
+        pass
 
 def test_init_max_time_mins():
     """Assert that the TPOT init stores max run time and sets generations to 1000000"""
@@ -185,14 +209,14 @@ def test_random_ind_2():
     assert expected_code == export_pipeline(pipeline, tpot_obj.operators, tpot_obj._pset)
 
 def test_score():
-    """Assert that the TPOT score function raises a ValueError when no optimized pipeline exists"""
+    """Assert that the TPOT score function raises a RuntimeError when no optimized pipeline exists"""
 
     tpot_obj = TPOTClassifier()
 
     try:
         tpot_obj.score(testing_features, testing_classes)
         assert False  # Should be unreachable
-    except ValueError:
+    except RuntimeError:
         pass
 
 
@@ -292,14 +316,14 @@ def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
 
 
 def test_predict():
-    """Assert that the TPOT predict function raises a ValueError when no optimized pipeline exists"""
+    """Assert that the TPOT predict function raises a RuntimeError when no optimized pipeline exists"""
 
     tpot_obj = TPOTClassifier()
 
     try:
         tpot_obj.predict(testing_features)
         assert False  # Should be unreachable
-    except ValueError:
+    except RuntimeError:
         pass
 
 
@@ -445,13 +469,13 @@ def test_operators():
 
 
 def test_export():
-    """Assert that TPOT's export function throws a ValueError when no optimized pipeline exists"""
+    """Assert that TPOT's export function throws a RuntimeError when no optimized pipeline exists"""
     tpot_obj = TPOTClassifier()
 
     try:
         tpot_obj.export("test_export.py")
         assert False  # Should be unreachable
-    except ValueError:
+    except RuntimeError:
         pass
 
 

From 00c1a2044b51ad6d56692e6ab530bc183d35b05b Mon Sep 17 00:00:00 2001
From: Randy Olson 
Date: Wed, 22 Mar 2017 14:54:35 -0400
Subject: [PATCH 151/154] Clean up docs to make the easier to read

---
 docs/index.html               |   2 +-
 docs/mkdocs/search_index.json |   6 +-
 docs/using/index.html         | 117 ++++++++++++++++++++++++----------
 docs_sources/using.md         | 117 ++++++++++++++++++++++++----------
 4 files changed, 174 insertions(+), 68 deletions(-)

diff --git a/docs/index.html b/docs/index.html
index 360e3fc1..8fe7fb22 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -245,5 +245,5 @@
 
 
diff --git a/docs/mkdocs/search_index.json b/docs/mkdocs/search_index.json
index e4eb83ed..50d9f32b 100644
--- a/docs/mkdocs/search_index.json
+++ b/docs/mkdocs/search_index.json
@@ -12,17 +12,17 @@
         },
         {
             "location": "/using/",
-            "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process. See the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nN/A\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nN/A\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nN/A\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation. By default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n for evaluating pipelines, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string to the \nscoring\n parameter from the list above. Any other strings will cause TPOT to throw an exception.\n\n\n\n\n\n\nYou can pass a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ndef accuracy(y_true, y_pred):\n    return float(sum(y_pred == y_true)) / len(y_true)\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n                      scoring=accuracy)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\n\n\nCustomizing TPOT's operators and parameters\n\n\nTPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters.\n\n\nThe custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., \nsklearn.naive_bayes.MultinomialNB\n) and the second level key is the corresponding parameter name for that operator (e.g., \nfit_prior\n). The second level key should point to a list of parameter values for that parameter, e.g., \n'fit_prior': [True, False]\n.\n\n\nFor a simple example, the configuration could be:\n\n\nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}\n\n\n\n\nin which case TPOT would only explore pipelines containing \nGaussianNB\n, \nBernoulliNB\n, \nMultinomialNB\n, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the \nTPOTClassifier\n/\nTPOTRegressor\n \nconfig_dict\n parameter, described above. For example:\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n                      config_dict=classifier_config_dict)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\nCommand-line users must create a separate \n.py\n file with the custom configuration and provide the path to the file to the \ntpot\n call. For example, if the simple example configuration above is saved in \ntpot_classifier_config.py\n, that configuration could be used on the command line with the command:\n\n\ntpot data/mnist.csv -is , -target class -config tpot_classifier_config.py -g 5 -p 20 -v 2 -o tpot_exported_pipeline.py\n\n\n\n\nFor more detailed examples of how to customize TPOT's operator configuration, see the default configurations for \nclassification\n and \nregression\n in TPOT's source code.\n\n\nNote that you must have all of the corresponding packages for the operators installed on your computer, otherwise TPOT will not be able to use them. For example, if XGBoost is not installed on your computer, then TPOT will simply not import nor use XGBoost in the pipelines it explores.",
+            "text": "TPOT on the command line\n\n\nTo use TPOT via the command line, enter the following command with a path to the data file:\n\n\ntpot /path_to/data_file.csv\n\n\n\n\nTPOT offers several arguments that can be provided at the command line:\n\n\n\n\n\n\nArgument\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\n-is\n\n\nINPUT_SEPARATOR\n\n\nAny string\n\n\nCharacter used to separate columns in the input file.\n\n\n\n\n\n\n-target\n\n\nTARGET_NAME\n\n\nAny string\n\n\nName of the target column in the input file.\n\n\n\n\n\n\n-mode\n\n\nTPOT_MODE\n\n\n['classification', 'regression']\n\n\nWhether TPOT is being used for a supervised classification or regression problem.\n\n\n\n\n\n\n-o\n\n\nOUTPUT_FILE\n\n\nString path to a file\n\n\nFile to export the code for the final optimized pipeline.\n\n\n\n\n\n\n-g\n\n\nGENERATIONS\n\n\nAny positive integer\n\n\nNumber of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline.\n\n\nTPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-p\n\n\nPOPULATION_SIZE\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline.\n\n\nTPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\n-os\n\n\nOFFSPRING_SIZE\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation.\n\n\nBy default, OFFSPRING_SIZE = POPULATION_SIZE.\n\n\n\n\n\n\n-mr\n\n\nMUTATION_RATE\n\n\n[0.0, 1.0]\n\n\nGP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.\n\n\nWe recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\n-xr\n\n\nCROSSOVER_RATE\n\n\n[0.0, 1.0]\n\n\nGP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation.\n\n\nWe recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.\n\n\n\n\n\n\n-scoring\n\n\nSCORING_FN\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy',\n'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, accuracy is used for classification and mean squared error (MSE) is used for regression.\n\n\nTPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized.\n\n\nSee the section on \nscoring functions\n for more details.\n\n\n\n\n\n\n-cv\n\n\nNUM_CV_FOLDS\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\n-njobs\n\n\nNUM_JOBS\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process.\n\n\nAssigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\n-maxtime\n\n\nMAX_TIME_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline.\n\n\nIf provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\n-maxeval\n\n\nMAX_EVAL_MINS\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to evaluate a single pipeline.\n\n\nSetting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.\n\n\n\n\n\n\n-s\n\n\nRANDOM_STATE\n\n\nAny positive integer\n\n\nRandom number generator seed for reproducibility.\n\n\nSet this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.\n\n\n\n\n\n\n-config\n\n\nCONFIG_FILE\n\n\nString path to a file\n\n\nConfiguration file for customizing the operators and parameters that TPOT uses in the optimization process.\n\n\nSee the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n-v\n\n\nVERBOSITY\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it is running.\n\n\n0 = none, 1 = minimal, 2 = high, 3 = all.\n\n\nA setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\n--no-update-check\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n--version\n\n\nShow TPOT's version number and exit.\n\n\n\n\n\n\n--help\n\n\nShow TPOT's help documentation and exit.\n\n\n\n\n\n\n\nAn example command-line call to TPOT may look like:\n\n\ntpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2\n\n\n\n\nTPOT with code\n\n\nWe've taken care to design the TPOT interface to be as similar as possible to scikit-learn.\n\n\nTPOT can be imported just like any regular Python module. To import TPOT, type:\n\n\nfrom tpot import TPOTClassifier\n\n\n\n\nthen create an instance of TPOT as follows:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()\n\n\n\n\nIt's also possible to use TPOT for regression problems with the \nTPOTRegressor\n class. Other than the class name, a \nTPOTRegressor\n is used the same way as a \nTPOTClassifier\n.\n\n\nNote that you can pass several parameters to the TPOT instantiation call:\n\n\n\n\n\n\nParameter\n\n\nValid values\n\n\nEffect\n\n\n\n\n\n\ngenerations\n\n\nAny positive integer\n\n\nNumber of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline.\n\n\nTPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\npopulation_size\n\n\nAny positive integer\n\n\nNumber of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline.\n\n\nTPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.\n\n\n\n\n\n\noffspring_size\n\n\nAny positive integer\n\n\nNumber of offspring to produce in each GP generation.\n\n\nBy default, offspring_size = population_size.\n\n\n\n\n\n\nmutation_rate\n\n\n[0.0, 1.0]\n\n\nMutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation.\n\n\nWe recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\ncrossover_rate\n\n\n[0.0, 1.0]\n\n\nCrossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation.\n\n\nWe recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.\n\n\n\n\n\n\nscoring\n\n\n'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy',\n'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature \nscorer(y_true, y_pred)\n\n\nFunction used to evaluate the quality of a given pipeline for the problem. By default, accuracy is used for classification and mean squared error (MSE) is used for regression.\n\n\nTPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized.\n\n\nSee the section on \nscoring functions\n for more details.\n\n\n\n\n\n\ncv\n\n\nAny integer >1\n\n\nNumber of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.\n\n\n\n\n\n\nn_jobs\n\n\nAny positive integer or -1\n\n\nNumber of CPUs for evaluating pipelines in parallel during the TPOT optimization process.\n\n\nAssigning this to -1 will use as many cores as available on the computer.\n\n\n\n\n\n\nmax_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize the pipeline.\n\n\nIf provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.\n\n\n\n\n\n\nmax_eval_time_mins\n\n\nAny positive integer\n\n\nHow many minutes TPOT has to optimize a single pipeline.\n\n\nSetting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.\n\n\n\n\n\n\nrandom_state\n\n\nAny positive integer\n\n\nRandom number generator seed for TPOT.\n\n\nUse this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.\n\n\n\n\n\n\nconfig_dict\n\n\nPython dictionary\n\n\nConfiguration dictionary for customizing the operators and parameters that TPOT uses in the optimization process.\n\n\nSee the \ncustom configuration\n section for more information and examples.\n\n\n\n\n\n\n\n\n\nwarm_start\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT instance will reuse the population from previous calls to fit().\n\n\n\n\n\n\nverbosity\n\n\n{0, 1, 2, 3}\n\n\nHow much information TPOT communicates while it's running.\n\n\n0 = none, 1 = minimal, 2 = high, 3 = all.\n\n\nA setting of 2 or higher will add a progress bar during the optimization procedure.\n\n\n\n\n\n\ndisable_update_check\n\n\n[True, False]\n\n\nFlag indicating whether the TPOT version checker should be disabled.\n\n\n\n\n\n\n\nSome example code with custom TPOT parameters might look like:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\n\n\n\n\nNow TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the \nfit\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\n\n\n\n\nThe \nfit()\n function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.\n\n\nYou can then proceed to evaluate the final pipeline on the testing set with the \nscore()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\n\n\n\n\nFinally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the \nexport()\n function:\n\n\nfrom tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')\n\n\n\n\nOnce this code finishes running, \ntpot_exported_pipeline.py\n will contain the Python code for the optimized pipeline.\n\n\nCheck our \nexamples\n to see TPOT applied to some specific data sets.\n\n\n\n\nScoring functions\n\n\nTPOT makes use of \nsklearn.model_selection.cross_val_score\n for evaluating pipelines, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:\n\n\n\n\n\n\nYou can pass in a string to the \nscoring\n parameter from the list above. Any other strings will cause TPOT to throw an exception.\n\n\n\n\n\n\nYou can pass a function with the signature \nscorer(y_true, y_pred)\n, where \ny_true\n are the true target values and \ny_pred\n are the predicted target values from an estimator. To do this, you should implement your own function. See the example below for further explanation.\n\n\n\n\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ndef accuracy(y_true, y_pred):\n    return float(sum(y_pred == y_true)) / len(y_true)\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n                      scoring=accuracy)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\n\n\nCustomizing TPOT's operators and parameters\n\n\nTPOT comes with a handful of default operators and parameter configurations that we believe work well for optimizing machine learning pipelines. However, in some cases it is useful to limit the algorithms and parameters that TPOT explores. For that reason, we allow users to provide TPOT with a custom configuration for its operators and parameters.\n\n\nThe custom TPOT configuration must be in nested dictionary format, where the first level key is the path and name of the operator (e.g., \nsklearn.naive_bayes.MultinomialNB\n) and the second level key is the corresponding parameter name for that operator (e.g., \nfit_prior\n). The second level key should point to a list of parameter values for that parameter, e.g., \n'fit_prior': [True, False]\n.\n\n\nFor a simple example, the configuration could be:\n\n\nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}\n\n\n\n\nin which case TPOT would only explore pipelines containing \nGaussianNB\n, \nBernoulliNB\n, \nMultinomialNB\n, and tune those algorithm's parameters in the ranges provided. This dictionary can be passed directly within the code to the \nTPOTClassifier\n/\nTPOTRegressor\n \nconfig_dict\n parameter, described above. For example:\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\nclassifier_config_dict = {\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    }\n}\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2,\n                      config_dict=classifier_config_dict)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\nCommand-line users must create a separate \n.py\n file with the custom configuration and provide the path to the file to the \ntpot\n call. For example, if the simple example configuration above is saved in \ntpot_classifier_config.py\n, that configuration could be used on the command line with the command:\n\n\ntpot data/mnist.csv -is , -target class -config tpot_classifier_config.py -g 5 -p 20 -v 2 -o tpot_exported_pipeline.py\n\n\n\n\nFor more detailed examples of how to customize TPOT's operator configuration, see the default configurations for \nclassification\n and \nregression\n in TPOT's source code.\n\n\nNote that you must have all of the corresponding packages for the operators installed on your computer, otherwise TPOT will not be able to use them. For example, if XGBoost is not installed on your computer, then TPOT will simply not import nor use XGBoost in the pipelines it explores.",
             "title": "Using TPOT"
         },
         {
             "location": "/using/#tpot-on-the-command-line",
-            "text": "To use TPOT via the command line, enter the following command with a path to the data file:  tpot /path_to/data_file.csv  TPOT offers several arguments that can be provided at the command line:    Argument  Parameter  Valid values  Effect    -is  INPUT_SEPARATOR  Any string  Character used to separate columns in the input file.    -target  TARGET_NAME  Any string  Name of the target column in the input file.    -mode  TPOT_MODE  ['classification', 'regression']  Whether TPOT is being used for a supervised classification or regression problem.    -o  OUTPUT_FILE  String path to a file  File to export the code for the final optimized pipeline.    -g  GENERATIONS  Any positive integer  Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    -p  POPULATION_SIZE  Any positive integer  Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    -os  OFFSPRING_SIZE  Any positive integer  Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.    -mr  MUTATION_RATE  [0.0, 1.0]  GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    -xr  CROSSOVER_RATE  [0.0, 1.0]  GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.    -scoring  SCORING_FN  'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'  Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on  scoring functions  for more details.    -cv  NUM_CV_FOLDS  Any integer >1  Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.    -njobs  NUM_JOBS  Any positive integer or -1  Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.    -maxtime  MAX_TIME_MINS  Any positive integer  How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.    -maxeval  MAX_EVAL_MINS  Any positive integer  How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.    -s  RANDOM_STATE  Any positive integer  Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.    -config  CONFIG_FILE  String path to a file  Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. See the  custom configuration  section for more information and examples.    -v  VERBOSITY  {0, 1, 2, 3}  How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.    --no-update-check  N/A  Flag indicating whether the TPOT version checker should be disabled.    --version  N/A  Show TPOT's version number and exit.    --help  N/A  Show TPOT's help documentation and exit.    An example command-line call to TPOT may look like:  tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2",
+            "text": "To use TPOT via the command line, enter the following command with a path to the data file:  tpot /path_to/data_file.csv  TPOT offers several arguments that can be provided at the command line:    Argument  Parameter  Valid values  Effect    -is  INPUT_SEPARATOR  Any string  Character used to separate columns in the input file.    -target  TARGET_NAME  Any string  Name of the target column in the input file.    -mode  TPOT_MODE  ['classification', 'regression']  Whether TPOT is being used for a supervised classification or regression problem.    -o  OUTPUT_FILE  String path to a file  File to export the code for the final optimized pipeline.    -g  GENERATIONS  Any positive integer  Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. \nTPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    -p  POPULATION_SIZE  Any positive integer  Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. \nTPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    -os  OFFSPRING_SIZE  Any positive integer  Number of offspring to produce in each GP generation. \nBy default, OFFSPRING_SIZE = POPULATION_SIZE.    -mr  MUTATION_RATE  [0.0, 1.0]  GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation. \nWe recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    -xr  CROSSOVER_RATE  [0.0, 1.0]  GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to \"breed\" every generation. \nWe recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.    -scoring  SCORING_FN  'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'  Function used to evaluate the quality of a given pipeline for the problem. By default, accuracy is used for classification and mean squared error (MSE) is used for regression. \nTPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. \nSee the section on  scoring functions  for more details.    -cv  NUM_CV_FOLDS  Any integer >1  Number of folds to evaluate each pipeline over in 'k-fold cross-validation during the TPOT optimization process.    -njobs  NUM_JOBS  Any positive integer or -1  Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. \nAssigning this to -1 will use as many cores as available on the computer.    -maxtime  MAX_TIME_MINS  Any positive integer  How many minutes TPOT has to optimize the pipeline. \nIf provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.    -maxeval  MAX_EVAL_MINS  Any positive integer  How many minutes TPOT has to evaluate a single pipeline. \nSetting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.    -s  RANDOM_STATE  Any positive integer  Random number generator seed for reproducibility. \nSet this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.    -config  CONFIG_FILE  String path to a file  Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. \nSee the  custom configuration  section for more information and examples.    -v  VERBOSITY  {0, 1, 2, 3}  How much information TPOT communicates while it is running. \n0 = none, 1 = minimal, 2 = high, 3 = all. \nA setting of 2 or higher will add a progress bar during the optimization procedure.    --no-update-check  Flag indicating whether the TPOT version checker should be disabled.    --version  Show TPOT's version number and exit.    --help  Show TPOT's help documentation and exit.    An example command-line call to TPOT may look like:  tpot data/mnist.csv -is , -target class -o tpot_exported_pipeline.py -g 5 -p 20 -cv 5 -s 42 -v 2",
             "title": "TPOT on the command line"
         },
         {
             "location": "/using/#tpot-with-code",
-            "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn.  TPOT can be imported just like any regular Python module. To import TPOT, type:  from tpot import TPOTClassifier  then create an instance of TPOT as follows:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()  It's also possible to use TPOT for regression problems with the  TPOTRegressor  class. Other than the class name, a  TPOTRegressor  is used the same way as a  TPOTClassifier .  Note that you can pass several parameters to the TPOT instantiation call:    Parameter  Valid values  Effect    generations  Any positive integer  Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    population_size  Any positive integer  Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    offspring_size  Any positive integer  Number of offspring to produce in each GP generation. By default, offspring_size = population_size.    mutation_rate  [0.0, 1.0]  Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    crossover_rate  [0.0, 1.0]  Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    scoring  'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature  scorer(y_true, y_pred)  Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on  scoring functions  for more details.    cv  Any integer >1  Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.    n_jobs  Any positive integer or -1  Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.    max_time_mins  Any positive integer  How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.    max_eval_time_mins  Any positive integer  How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.    random_state  Any positive integer  Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.    config_dict  Python dictionary  Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the  custom configuration  section for more information and examples.     warm_start  [True, False]  Flag indicating whether the TPOT instance will reuse the population from previous calls to fit().    verbosity  {0, 1, 2, 3}  How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.    disable_update_check  [True, False]  Flag indicating whether the TPOT version checker should be disabled.    Some example code with custom TPOT parameters might look like:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)  Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the  fit  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)  The  fit()  function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.  You can then proceed to evaluate the final pipeline on the testing set with the  score()  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))  Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the  export()  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')  Once this code finishes running,  tpot_exported_pipeline.py  will contain the Python code for the optimized pipeline.  Check our  examples  to see TPOT applied to some specific data sets.",
+            "text": "We've taken care to design the TPOT interface to be as similar as possible to scikit-learn.  TPOT can be imported just like any regular Python module. To import TPOT, type:  from tpot import TPOTClassifier  then create an instance of TPOT as follows:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier()  It's also possible to use TPOT for regression problems with the  TPOTRegressor  class. Other than the class name, a  TPOTRegressor  is used the same way as a  TPOTClassifier .  Note that you can pass several parameters to the TPOT instantiation call:    Parameter  Valid values  Effect    generations  Any positive integer  Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. \nTPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    population_size  Any positive integer  Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. \nTPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.    offspring_size  Any positive integer  Number of offspring to produce in each GP generation. \nBy default, offspring_size = population_size.    mutation_rate  [0.0, 1.0]  Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. \nWe recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    crossover_rate  [0.0, 1.0]  Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to \"breed\" every generation. \nWe recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.    scoring  'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature  scorer(y_true, y_pred)  Function used to evaluate the quality of a given pipeline for the problem. By default, accuracy is used for classification and mean squared error (MSE) is used for regression. \nTPOT assumes that any function with \"error\" or \"loss\" in the name is meant to be minimized, whereas any other functions will be maximized. \nSee the section on  scoring functions  for more details.    cv  Any integer >1  Number of folds to evaluate each pipeline over in k-fold cross-validation during the TPOT optimization process.    n_jobs  Any positive integer or -1  Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. \nAssigning this to -1 will use as many cores as available on the computer.    max_time_mins  Any positive integer  How many minutes TPOT has to optimize the pipeline. \nIf provided, this setting will override the \"generations\" parameter and allow TPOT to run until it runs out of time.    max_eval_time_mins  Any positive integer  How many minutes TPOT has to optimize a single pipeline. \nSetting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.    random_state  Any positive integer  Random number generator seed for TPOT. \nUse this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.    config_dict  Python dictionary  Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. \nSee the  custom configuration  section for more information and examples.     warm_start  [True, False]  Flag indicating whether the TPOT instance will reuse the population from previous calls to fit().    verbosity  {0, 1, 2, 3}  How much information TPOT communicates while it's running. \n0 = none, 1 = minimal, 2 = high, 3 = all. \nA setting of 2 or higher will add a progress bar during the optimization procedure.    disable_update_check  [True, False]  Flag indicating whether the TPOT version checker should be disabled.    Some example code with custom TPOT parameters might look like:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)  Now TPOT is ready to optimize a pipeline for you. You can tell TPOT to optimize a pipeline based on a data set with the  fit  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)  The  fit()  function takes in a training data set and uses k-fold cross-validation when evaluating pipelines. It then initializes the genetic programming algoritm to find the best pipeline based on average k-fold score.  You can then proceed to evaluate the final pipeline on the testing set with the  score()  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))  Finally, you can tell TPOT to export the corresponding Python code for the optimized pipeline to a text file with the  export()  function:  from tpot import TPOTClassifier\n\npipeline_optimizer = TPOTClassifier(generations=5, population_size=20, cv=5,\n                                    random_state=42, verbosity=2)\npipeline_optimizer.fit(training_features, training_classes)\nprint(pipeline_optimizer.score(testing_features, testing_classes))\npipeline_optimizer.export('tpot_exported_pipeline.py')  Once this code finishes running,  tpot_exported_pipeline.py  will contain the Python code for the optimized pipeline.  Check our  examples  to see TPOT applied to some specific data sets.",
             "title": "TPOT with code"
         },
         {
diff --git a/docs/using/index.html b/docs/using/index.html
index 8e92d85b..bab47183 100644
--- a/docs/using/index.html
+++ b/docs/using/index.html
@@ -221,37 +221,51 @@ 
TPOT on the command line

 -g
 GENERATIONS
 Any positive integer
-Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
+Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline.
+


+TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 -p
 POPULATION_SIZE
 Any positive integer
-Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
+Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline.
+


+TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 -os
 OFFSPRING_SIZE
 Any positive integer
-Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.
+Number of offspring to produce in each GP generation.
+


+By default, OFFSPRING_SIZE = POPULATION_SIZE.
 
 
 -mr
 MUTATION_RATE
 [0.0, 1.0]
-GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
+GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.
+


+We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
 -xr
 CROSSOVER_RATE
 [0.0, 1.0]
-GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to "breed" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.
+GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to "breed" every generation.
+


+We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.
 
 
 -scoring
 SCORING_FN
-'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
-Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
+'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy',
'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
+Function used to evaluate the quality of a given pipeline for the problem. By default, accuracy is used for classification and mean squared error (MSE) is used for regression.
+


+TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized.
+


+See the section on scoring functions for more details.
 
 
 -cv
@@ -263,51 +277,62 @@ 
TPOT on the command line

 -njobs
 NUM_JOBS
 Any positive integer or -1
-Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.
+Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process.
+


+Assigning this to -1 will use as many cores as available on the computer.
 
 
 -maxtime
 MAX_TIME_MINS
 Any positive integer
-How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
+How many minutes TPOT has to optimize the pipeline.
+


+If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
 
 
 -maxeval
 MAX_EVAL_MINS
 Any positive integer
-How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.
+How many minutes TPOT has to evaluate a single pipeline.
+


+Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.
 
 
 -s
 RANDOM_STATE
 Any positive integer
-Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.
+Random number generator seed for reproducibility.
+


+Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.
 
 
 -config
 CONFIG_FILE
 String path to a file
-Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples.
+Configuration file for customizing the operators and parameters that TPOT uses in the optimization process.
+


+See the custom configuration section for more information and examples.
 
 
 -v
 VERBOSITY
 {0, 1, 2, 3}
-How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.
+How much information TPOT communicates while it is running.
+


+0 = none, 1 = minimal, 2 = high, 3 = all.
+


+A setting of 2 or higher will add a progress bar during the optimization procedure.
 
 
---no-update-check
-N/A
+--no-update-check
 Flag indicating whether the TPOT version checker should be disabled.
 
 
---version
-N/A
+--version
 Show TPOT's version number and exit.
 
 
---help
-N/A
+--help
 Show TPOT's help documentation and exit.
 
 
@@ -339,32 +364,46 @@ 
TPOT with code

 
 generations
 Any positive integer
-Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
+Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline.
+


+TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 population_size
 Any positive integer
-Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
+Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline.
+


+TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 offspring_size
 Any positive integer
-Number of offspring to produce in each GP generation. By default, offspring_size = population_size.
+Number of offspring to produce in each GP generation.
+


+By default, offspring_size = population_size.
 
 
 mutation_rate
 [0.0, 1.0]
-Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
+Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation.
+


+We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
 crossover_rate
 [0.0, 1.0]
-Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to "breed" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
+Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to "breed" every generation.
+


+We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
 scoring
-'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
-Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
+'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy',
'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
+Function used to evaluate the quality of a given pipeline for the problem. By default, accuracy is used for classification and mean squared error (MSE) is used for regression.
+


+TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized.
+


+See the section on scoring functions for more details.
 
 
 cv
@@ -374,27 +413,37 @@ 
TPOT with code

 
 n_jobs
 Any positive integer or -1
-Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.
+Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process.
+


+Assigning this to -1 will use as many cores as available on the computer.
 
 
 max_time_mins
 Any positive integer
-How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
+How many minutes TPOT has to optimize the pipeline.
+


+If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
 
 
 max_eval_time_mins
 Any positive integer
-How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.
+How many minutes TPOT has to optimize a single pipeline.
+


+Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.
 
 
 random_state
 Any positive integer
-Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.
+Random number generator seed for TPOT.
+


+Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.
 
 
 config_dict
 Python dictionary
-Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples.
+Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process.
+


+See the custom configuration section for more information and examples.
 

 
 
@@ -406,7 +455,11 @@ 
TPOT with code

 
 verbosity
 {0, 1, 2, 3}
-How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.
+How much information TPOT communicates while it's running.
+


+0 = none, 1 = minimal, 2 = high, 3 = all.
+


+A setting of 2 or higher will add a progress bar during the optimization procedure.
 
 
 disable_update_check
diff --git a/docs_sources/using.md b/docs_sources/using.md
index f9a88d03..3d284d7c 100644
--- a/docs_sources/using.md
+++ b/docs_sources/using.md
@@ -43,37 +43,51 @@ TPOT offers several arguments that can be provided at the command line:
 -g
 GENERATIONS
 Any positive integer
-Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
+Number of iterations to run the pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline.
+


+TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 -p
 POPULATION_SIZE
 Any positive integer
-Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
+Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline.
+


+TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 -os
 OFFSPRING_SIZE
 Any positive integer
-Number of offspring to produce in each GP generation. By default, OFFSPRING_SIZE = POPULATION_SIZE.
+Number of offspring to produce in each GP generation.
+


+By default, OFFSPRING_SIZE = POPULATION_SIZE.
 
 
 -mr
 MUTATION_RATE
 [0.0, 1.0]
-GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.  We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
+GP mutation rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to apply random changes to every generation.
+


+We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
 -xr
 CROSSOVER_RATE
 [0.0, 1.0]
-GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to "breed" every generation. We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.
+GP crossover rate in the range [0.0, 1.0]. This tells the GP algorithm how many pipelines to "breed" every generation.
+


+We recommend using the default parameter unless you understand how the crossover rate affects GP algorithms.
 
 
 -scoring
 SCORING_FN
-'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
-Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
+'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy',
'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc'
+Function used to evaluate the quality of a given pipeline for the problem. By default, accuracy is used for classification and mean squared error (MSE) is used for regression.
+


+TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized.
+


+See the section on scoring functions for more details.
 
 
 -cv
@@ -85,51 +99,62 @@ TPOT offers several arguments that can be provided at the command line:
 -njobs
 NUM_JOBS
 Any positive integer or -1
-Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.
+Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process.
+


+Assigning this to -1 will use as many cores as available on the computer.
 
 
 -maxtime
 MAX_TIME_MINS
 Any positive integer
-How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
+How many minutes TPOT has to optimize the pipeline.
+


+If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
 
 
 -maxeval
 MAX_EVAL_MINS
 Any positive integer
-How many minutes TPOT has to evaluate a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.
+How many minutes TPOT has to evaluate a single pipeline.
+


+Setting this parameter to higher values will allow TPOT to explore more complex pipelines but will also allow TPOT to run longer.
 
 
 -s
 RANDOM_STATE
 Any positive integer
-Random number generator seed for reproducibility. Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.
+Random number generator seed for reproducibility.
+


+Set this seed if you want your TPOT run to be reproducible with the same seed and data set in the future.
 
 
 -config
 CONFIG_FILE
 String path to a file
-Configuration file for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples.
+Configuration file for customizing the operators and parameters that TPOT uses in the optimization process.
+


+See the custom configuration section for more information and examples.
 
 
 -v
 VERBOSITY
 {0, 1, 2, 3}
-How much information TPOT communicates while it is running: 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.
+How much information TPOT communicates while it is running.
+


+0 = none, 1 = minimal, 2 = high, 3 = all.
+


+A setting of 2 or higher will add a progress bar during the optimization procedure.
 
 
---no-update-check
-N/A
+--no-update-check
 Flag indicating whether the TPOT version checker should be disabled.
 
 
---version
-N/A
+--version
 Show TPOT's version number and exit.
 
 
---help
-N/A
+--help
 Show TPOT's help documentation and exit.
 
 
@@ -171,32 +196,46 @@ Note that you can pass several parameters to the TPOT instantiation call:
 
 generations
 Any positive integer
-Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
+Number of iterations to the run pipeline optimization process. Generally, TPOT will work better when you give it more generations (and therefore time) to optimize the pipeline.
+


+TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 population_size
 Any positive integer
-Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline. TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
+Number of individuals to retain in the GP population every generation. Generally, TPOT will work better when you give it more individuals (and therefore time) to optimize the pipeline.
+


+TPOT will evaluate POPULATION_SIZE + GENERATIONS x OFFSPRING_SIZE pipelines in total.
 
 
 offspring_size
 Any positive integer
-Number of offspring to produce in each GP generation. By default, offspring_size = population_size.
+Number of offspring to produce in each GP generation.
+


+By default, offspring_size = population_size.
 
 
 mutation_rate
 [0.0, 1.0]
-Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
+Mutation rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the GP algorithm how many pipelines to apply random changes to every generation.
+


+We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
 crossover_rate
 [0.0, 1.0]
-Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to "breed" every generation. We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
+Crossover rate for the genetic programming algorithm in the range [0.0, 1.0]. This parameter tells the genetic programming algorithm how many pipelines to "breed" every generation.
+


+We recommend using the default parameter unless you understand how the mutation rate affects GP algorithms.
 
 
 scoring
-'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
-Function used to evaluate the quality of a given pipeline for the problem. By default, balanced accuracy is used for classification and mean squared error is used for regression. TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized. See the section on scoring functions for more details.
+'accuracy', 'adjusted_rand_score', 'average_precision', 'balanced_accuracy',
'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted', 'log_loss', 'mean_absolute_error', 'mean_squared_error', 'median_absolute_error', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'r2', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'roc_auc' or a callable function with signature scorer(y_true, y_pred)
+Function used to evaluate the quality of a given pipeline for the problem. By default, accuracy is used for classification and mean squared error (MSE) is used for regression.
+


+TPOT assumes that any function with "error" or "loss" in the name is meant to be minimized, whereas any other functions will be maximized.
+


+See the section on scoring functions for more details.
 
 
 cv
@@ -206,27 +245,37 @@ Note that you can pass several parameters to the TPOT instantiation call:
 
 n_jobs
 Any positive integer or -1
-Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process. Assigning this to -1 will use as many cores as available on the computer.
+Number of CPUs for evaluating pipelines in parallel during the TPOT optimization process.
+


+Assigning this to -1 will use as many cores as available on the computer.
 
 
 max_time_mins
 Any positive integer
-How many minutes TPOT has to optimize the pipeline. If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
+How many minutes TPOT has to optimize the pipeline.
+


+If provided, this setting will override the "generations" parameter and allow TPOT to run until it runs out of time.
 
 
 max_eval_time_mins
 Any positive integer
-How many minutes TPOT has to optimize a single pipeline. Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.
+How many minutes TPOT has to optimize a single pipeline.
+


+Setting this parameter to higher values will allow TPOT to explore more complex pipelines, but will also allow TPOT to run longer.
 
 
 random_state
 Any positive integer
-Random number generator seed for TPOT. Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.
+Random number generator seed for TPOT.
+


+Use this to make sure that TPOT will give you the same results each time you run it against the same data set with that seed.
 
 
 config_dict
 Python dictionary
-Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process. See the custom configuration section for more information and examples.
+Configuration dictionary for customizing the operators and parameters that TPOT uses in the optimization process.
+


+See the custom configuration section for more information and examples.
 

 
 
@@ -238,7 +287,11 @@ Note that you can pass several parameters to the TPOT instantiation call:
 
 verbosity
 {0, 1, 2, 3}
-How much information TPOT communicates while it's running. 0 = none, 1 = minimal, 2 = high, 3 = all. A setting of 2 or higher will add a progress bar during the optimization procedure.
+How much information TPOT communicates while it's running.
+


+0 = none, 1 = minimal, 2 = high, 3 = all.
+


+A setting of 2 or higher will add a progress bar during the optimization procedure.
 
 
 disable_update_check

From f929b40500db9dd8178d8827cbb61ebe40118284 Mon Sep 17 00:00:00 2001
From: Randy Olson 
Date: Wed, 22 Mar 2017 15:14:46 -0400
Subject: [PATCH 152/154] Update CI to test on Python 3.5 and 3.6

---
 .travis.yml           | 4 ++--
 MANIFEST.in           | 2 +-
 ci/.travis_install.sh | 1 +
 ci/.travis_test.sh    | 1 +
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 67beda6c..a167d0a0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,9 +5,9 @@ env:
   matrix:
     # let's start simple:
     - PYTHON_VERSION="2.7" LATEST="true"
-    - PYTHON_VERSION="3.4" LATEST="true"
-    - PYTHON_VERSION="3.5" COVERAGE="true" LATEST="true" "DEAP_VERSION=1.0.1" "XGBOOST_VERSION=0.4a30"
     - PYTHON_VERSION="3.5" LATEST="true"
+    - PYTHON_VERSION="3.6" COVERAGE="true" LATEST="true" "DEAP_VERSION=1.0.1" "XGBOOST_VERSION=0.4a30"
+    - PYTHON_VERSION="3.6" LATEST="true"
 install: source ./ci/.travis_install.sh
 script: bash ./ci/.travis_test.sh
 after_success:
diff --git a/MANIFEST.in b/MANIFEST.in
index e6e1e6a3..3ee3ca24 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,3 @@
-include README.md LICENSE tests.py
+include README.md LICENSE tests.py tests.csv
 recursive-include images *
 recursive-include tpot *.py
diff --git a/ci/.travis_install.sh b/ci/.travis_install.sh
index 23a2b0ac..c2830dd4 100755
--- a/ci/.travis_install.sh
+++ b/ci/.travis_install.sh
@@ -68,4 +68,5 @@ python -c "import deap; print('deap %s' % deap.__version__)"
 python -c "import xgboost; print('xgboost %s ' % xgboost.__version__)"
 python -c "import update_checker; print('update_checker %s' % update_checker.__version__)"
 python -c "import tqdm; print('tqdm %s' % tqdm.__version__)"
+python -c "import pathos; print('pathos %s' % pathos.__version__)"
 python setup.py build_ext --inplace
diff --git a/ci/.travis_test.sh b/ci/.travis_test.sh
index 77b3733b..162d8092 100755
--- a/ci/.travis_test.sh
+++ b/ci/.travis_test.sh
@@ -17,6 +17,7 @@ python -c "import deap; print('deap %s' % deap.__version__)"
 python -c "import xgboost; print('xgboost %s ' % xgboost.__version__)"
 python -c "import update_checker; print('update_checker %s ' % update_checker.__version__)"
 python -c "import tqdm; print('tqdm %s' % tqdm.__version__)"
+python -c "import pathos; print('pathos %s' % pathos.__version__)"
 
 if [[ "$COVERAGE" == "true" ]]; then
     nosetests -s -v --with-coverage

From aedf9ea2a618cc08d25d2f42d99b4f4031d5a823 Mon Sep 17 00:00:00 2001
From: Randy Olson 
Date: Wed, 22 Mar 2017 16:02:32 -0400
Subject: [PATCH 153/154] Reorganize examples in the docs

Now all of the examples are on one page.
---
 README.md                                     |   2 +-
 docs/citing/index.html                        | 117 ++-----
 docs/contributing/index.html                  | 141 +++-----
 docs/css/theme_extra.css                      |  17 +-
 docs/examples/Boston_Example/index.html       | 266 ---------------
 docs/examples/IRIS_Example/index.html         | 268 ---------------
 docs/examples/MNIST_Example/index.html        | 263 ---------------
 .../Titanic_Kaggle_Example/index.html         | 228 -------------
 docs/examples/index.html                      | 310 ++++++++++++++++++
 docs/index.html                               | 121 +++----
 docs/installing/index.html                    | 117 ++-----
 docs/js/theme.js                              |  29 +-
 docs/mkdocs/search_index.json                 |  29 +-
 docs/releases/index.html                      | 151 +++------
 docs/search.html                              | 112 ++-----
 docs/sitemap.xml                              |  22 +-
 docs/support/index.html                       | 117 ++-----
 docs/using/index.html                         | 135 +++-----
 docs_sources/examples.md                      | 141 ++++++++
 docs_sources/examples/Boston_Example.md       |  45 ---
 docs_sources/examples/IRIS_Example.md         |  47 ---
 docs_sources/examples/MNIST_Example.md        |  41 ---
 .../examples/Titanic_Kaggle_Example.md        |   1 -
 docs_sources/using.md                         |   2 +-
 mkdocs.yml                                    |   6 +-
 25 files changed, 844 insertions(+), 1884 deletions(-)
 delete mode 100644 docs/examples/Boston_Example/index.html
 delete mode 100644 docs/examples/IRIS_Example/index.html
 delete mode 100644 docs/examples/MNIST_Example/index.html
 delete mode 100644 docs/examples/Titanic_Kaggle_Example/index.html
 create mode 100644 docs/examples/index.html
 create mode 100644 docs_sources/examples.md
 delete mode 100644 docs_sources/examples/Boston_Example.md
 delete mode 100644 docs_sources/examples/IRIS_Example.md
 delete mode 100644 docs_sources/examples/MNIST_Example.md
 delete mode 100644 docs_sources/examples/Titanic_Kaggle_Example.md

diff --git a/README.md b/README.md
index 6d07b45e..edb6c0cb 100644
--- a/README.md
+++ b/README.md
@@ -138,7 +138,7 @@ exported_pipeline.fit(training_features, training_classes)
 results = exported_pipeline.predict(testing_features)
 ```
 
-Check the documentation for [more examples and tutorials](http://rhiever.github.io/tpot/examples/MNIST_Example/).
+Check the documentation for [more examples and tutorials](http://rhiever.github.io/tpot/examples/).
 
 ## Contributing to TPOT
 
diff --git a/docs/citing/index.html b/docs/citing/index.html
index 18ab9308..cd420bc3 100644
--- a/docs/citing/index.html
+++ b/docs/citing/index.html
@@ -45,95 +45,50 @@
 
       
@@ -247,7 +202,7 @@
   

     
       
-           GitHub
+           GitHub
       
       
         « Previous
diff --git a/docs/contributing/index.html b/docs/contributing/index.html
index 7915234d..1ef02632 100644
--- a/docs/contributing/index.html
+++ b/docs/contributing/index.html
@@ -45,110 +45,65 @@
 
       
@@ -314,7 +269,7 @@ 
After submitting your pull requestNext 
       
       
-         Previous
+         Previous
       
     

   
@@ -341,10 +296,10 @@ 
After submitting your pull request
     
       
-           GitHub
+           GitHub
       
       
-        « Previous
+        « Previous
       
       
         Next »
diff --git a/docs/css/theme_extra.css b/docs/css/theme_extra.css
index b8b06d7f..e53d320a 100644
--- a/docs/css/theme_extra.css
+++ b/docs/css/theme_extra.css
@@ -22,10 +22,25 @@
  * area doesn't scroll.
  *
  * https://github.com/mkdocs/mkdocs/pull/202
+ *
+ * Builds upon pull 202 https://github.com/mkdocs/mkdocs/pull/202
+ * to make toc scrollbar end before navigations buttons to not be overlapping.
  */
 .wy-nav-side {
-    height: 100%;
+    height: calc(100% - 45px);
     overflow-y: auto;
+    min-height: 0;
+}
+
+.rst-versions{
+    border-top: 0;
+    height: 45px;
+}
+
+@media screen and (max-width: 768px) {
+    .wy-nav-side {
+        height: 100%;
+    }
 }
 
 /*
diff --git a/docs/examples/Boston_Example/index.html b/docs/examples/Boston_Example/index.html
deleted file mode 100644
index d779e130..00000000
--- a/docs/examples/Boston_Example/index.html
+++ /dev/null
@@ -1,266 +0,0 @@
-
-
-  
-
-  
-  
-  
-  
-  
-  
-  Boston Example - TPOT
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-   
-  
-
-
-
-
-  

-
-    
-    
-
-    

-
-      
-      
-
-      
-      

-        

-          

-  
-  

-

-          

-            

-              
-                
The following code illustrates the usage of TPOT with the Boston house prices data set.

-
from tpot import TPOTRegressor
-from sklearn.datasets import load_boston
-from sklearn.model_selection import train_test_split
-
-digits = load_boston()
-X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,
-                                                    train_size=0.75, test_size=0.25)
-
-tpot = TPOTRegressor(generations=5, population_size=20, verbosity=2)
-tpot.fit(X_train, y_train)
-print(tpot.score(X_test, y_test))
-tpot.export('tpot_boston_pipeline.py')
-

-
-
Running this code should discover a pipeline that achieves at least 12.77 mean squared error (MSE).

-
For details on how the fit(), score() and export() functions work, see the usage documentation.

-
After running the above code, the corresponding Python code should be exported to the tpot_boston_pipeline.py file and look similar to the following:

-
import numpy as np
-
-from sklearn.model_selection import train_test_split
-from sklearn.ensemble import ExtraTreesRegressor
-from sklearn.pipeline import make_pipeline
-
-# NOTE: Make sure that the target is labeled 'class' in the data file
-tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)
-features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),
-                     tpot_data.dtype.names.index('class'), axis=1)
-
-training_features, testing_features, training_classes, testing_classes = \
-    train_test_split(features, tpot_data['class'], random_state=42)
-
-exported_pipeline = make_pipeline(
-    ExtraTreesRegressor(max_features=0.76, n_estimators=500)
-)
-
-exported_pipeline.fit(training_features, training_classes)
-results = exported_pipeline.predict(testing_features)
-

-              
-            

-          

-          
-	  
-        

-      

-
-    

-    
-  

-
-  

-    
-      
-           GitHub
-      
-      
-        « Previous
-      
-      
-        Next »
-      
-    
-

-    
-
-
-
diff --git a/docs/examples/IRIS_Example/index.html b/docs/examples/IRIS_Example/index.html
deleted file mode 100644
index 54f7f7c0..00000000
--- a/docs/examples/IRIS_Example/index.html
+++ /dev/null
@@ -1,268 +0,0 @@
-
-
-  
-
-  
-  
-  
-  
-  
-  
-  IRIS Example - TPOT
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-   
-  
-
-
-
-
-  

-
-    
-    
-
-    

-
-      
-      
-
-      
-      

-        

-          

-  
-  

-

-          

-            

-              
-                
The following code illustrates the usage of TPOT with the IRIS data set.

-
from tpot import TPOTClassifier
-from sklearn.datasets import load_iris
-from sklearn.model_selection import train_test_split
-import numpy as np
-
-iris = load_iris()
-X_train, X_test, y_train, y_test = train_test_split(iris.data.astype(np.float64),
-    iris.target.astype(np.float64), train_size=0.75, test_size=0.25)
-
-tpot = TPOTClassifier(generations=5, population_size=20, verbosity=2)
-tpot.fit(X_train, y_train)
-print(tpot.score(X_test, y_test))
-tpot.export('tpot_iris_pipeline.py')
-

-
-
Running this code should discover a pipeline that achieves ~96% testing accuracy.

-
For details on how the fit(), score() and export() functions work, see the usage documentation.

-
After running the above code, the corresponding Python code should be exported to the tpot_iris_pipeline.py file and look similar to the following:

-
import numpy as np
-
-from sklearn.model_selection import train_test_split
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.pipeline import make_pipeline, make_union
-from sklearn.preprocessing import FunctionTransformer, PolynomialFeatures
-
-# NOTE: Make sure that the class is labeled 'class' in the data file
-tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)
-features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1)
-training_features, testing_features, training_classes, testing_classes = \
-    train_test_split(features, tpot_data['class'], random_state=42)
-
-exported_pipeline = make_pipeline(
-    PolynomialFeatures(degree=2, include_bias=False, interaction_only=False),
-    LogisticRegression(C=0.9, dual=False, penalty="l2")
-)
-
-exported_pipeline.fit(training_features, training_classes)
-results = exported_pipeline.predict(testing_features)
-

-              
-            

-          

-          
-	  
-        

-      

-
-    

-    
-  

-
-  

-    
-      
-           GitHub
-      
-      
-        « Previous
-      
-      
-        Next »
-      
-    
-

-    
-
-
-
diff --git a/docs/examples/MNIST_Example/index.html b/docs/examples/MNIST_Example/index.html
deleted file mode 100644
index 1c7da702..00000000
--- a/docs/examples/MNIST_Example/index.html
+++ /dev/null
@@ -1,263 +0,0 @@
-
-
-  
-
-  
-  
-  
-  
-  
-  
-  MNIST Example - TPOT
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-   
-  
-
-
-
-
-  

-
-    
-    
-
-    

-
-      
-      
-
-      
-      

-        

-          

-  
-  

-

-          

-            

-              
-                
Below is a minimal working example with the practice MNIST data set.

-
from tpot import TPOTClassifier
-from sklearn.datasets import load_digits
-from sklearn.model_selection import train_test_split
-
-digits = load_digits()
-X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,
-                                                    train_size=0.75, test_size=0.25)
-
-tpot = TPOTClassifier(generations=5, population_size=20, verbosity=2)
-tpot.fit(X_train, y_train)
-print(tpot.score(X_test, y_test))
-tpot.export('tpot_mnist_pipeline.py')
-

-
-
For details on how the fit(), score() and export() functions work, see the usage documentation.

-
Running this code should discover a pipeline that achieves about 98% testing accuracy, and the corresponding Python code should be exported to the tpot_mnist_pipeline.py file and look similar to the following:

-
import numpy as np
-
-from sklearn.model_selection import train_test_split
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.pipeline import make_pipeline
-
-# NOTE: Make sure that the class is labeled 'class' in the data file
-tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR')
-features = tpot_data.view((np.float64, len(tpot_data.dtype.names)))
-features = np.delete(features, tpot_data.dtype.names.index('class'), axis=1)
-training_features, testing_features, training_classes, testing_classes =     train_test_split(features, tpot_data['class'], random_state=42)
-
-exported_pipeline = make_pipeline(
-    KNeighborsClassifier(n_neighbors=3, weights="uniform")
-)
-
-exported_pipeline.fit(training_features, training_classes)
-results = exported_pipeline.predict(testing_features)
-

-              
-            

-          

-          
-	  
-        

-      

-
-    

-    
-  

-
-  

-    
-      
-           GitHub
-      
-      
-        « Previous
-      
-      
-        Next »
-      
-    
-

-    
-
-
-
diff --git a/docs/examples/Titanic_Kaggle_Example/index.html b/docs/examples/Titanic_Kaggle_Example/index.html
deleted file mode 100644
index da5bb935..00000000
--- a/docs/examples/Titanic_Kaggle_Example/index.html
+++ /dev/null
@@ -1,228 +0,0 @@
-
-
-  
-
-  
-  
-  
-  
-  
-  
-  Titanic Kaggle Example - TPOT
-  
-
-  
-  
-  
-  
-  
-  
-  
-  
-   
-  
-
-
-
-
-  

-
-    
-    
-
-    

-
-      
-      
-
-      
-      

-        

-          

-  
-  

-

-          

-            

-              
-                
To see the TPOT applied the Titanic Kaggle dataset, see the Jupyter notebook here.

-              
-            

-          

-          
-	  
-        

-      

-
-    

-    
-  

-
-  

-    
-      
-           GitHub
-      
-      
-        « Previous
-      
-      
-        Next »
-      
-    
-

-    
-
-
-
diff --git a/docs/examples/index.html b/docs/examples/index.html
new file mode 100644
index 00000000..17ab8308
--- /dev/null
+++ b/docs/examples/index.html
@@ -0,0 +1,310 @@
+
+
+  
+
+  
+  
+  
+  
+  
+  
+  Examples - TPOT
+  
+
+  
+  
+  
+  
+  
+  
+  
+  
+   
+  
+
+
+
+
+  

+
+    
+    
+
+    

+
+      
+      
+
+      
+      

+        

+          

+  
+  

+

+          

+            

+              
+                
Iris flower classification

+
The following code illustrates the usage of TPOT with the Iris data set, which is a simple supervised classification problem.

+
from tpot import TPOTClassifier
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+import numpy as np
+
+iris = load_iris()
+X_train, X_test, y_train, y_test = train_test_split(iris.data.astype(np.float64),
+    iris.target.astype(np.float64), train_size=0.75, test_size=0.25)
+
+tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2)
+tpot.fit(X_train, y_train)
+print(tpot.score(X_test, y_test))
+tpot.export('tpot_iris_pipeline.py')
+

+
+
Running this code should discover a pipeline that achieves about 97% testing accuracy.

+
For details on how the fit(), score() and export() functions work, see the usage documentation.

+
After running the above code, the corresponding Python code should be exported to the tpot_iris_pipeline.py file and look similar to the following:

+
import numpy as np
+
+from sklearn.model_selection import train_test_split
+from sklearn.naive_bayes import GaussianNB
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import Normalizer
+
+# NOTE: Make sure that the class is labeled 'class' in the data file
+tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)
+features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),
+                     tpot_data.dtype.names.index('class'), axis=1)
+training_features, testing_features, training_classes, testing_classes = \
+    train_test_split(features, tpot_data['class'], random_state=42)
+
+exported_pipeline = make_pipeline(
+    Normalizer(),
+    GaussianNB()
+)
+
+exported_pipeline.fit(training_features, training_classes)
+results = exported_pipeline.predict(testing_features)
+

+
+
MNIST digit recognition

+
Below is a minimal working example with the practice MNIST data set, which is an image classification problem.

+
from tpot import TPOTClassifier
+from sklearn.datasets import load_digits
+from sklearn.model_selection import train_test_split
+
+digits = load_digits()
+X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,
+                                                    train_size=0.75, test_size=0.25)
+
+tpot = TPOTClassifier(generations=5, population_size=50, verbosity=2)
+tpot.fit(X_train, y_train)
+print(tpot.score(X_test, y_test))
+tpot.export('tpot_mnist_pipeline.py')
+

+
+
For details on how the fit(), score() and export() functions work, see the usage documentation.

+
Running this code should discover a pipeline that achieves about 98% testing accuracy, and the corresponding Python code should be exported to the tpot_mnist_pipeline.py file and look similar to the following:

+
import numpy as np
+
+from sklearn.model_selection import train_test_split
+from sklearn.neighbors import KNeighborsClassifier
+
+# NOTE: Make sure that the class is labeled 'class' in the data file
+tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)
+features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),
+                     tpot_data.dtype.names.index('class'), axis=1)
+training_features, testing_features, training_classes, testing_classes = \
+    train_test_split(features, tpot_data['class'], random_state=42)
+
+exported_pipeline = KNeighborsClassifier(n_neighbors=6, weights="distance")
+
+exported_pipeline.fit(training_features, training_classes)
+results = exported_pipeline.predict(testing_features)
+

+
+
Boston housing prices modeling

+
The following code illustrates the usage of TPOT with the Boston housing prices data set, which is a regression problem.

+
from tpot import TPOTRegressor
+from sklearn.datasets import load_boston
+from sklearn.model_selection import train_test_split
+
+digits = load_boston()
+X_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,
+                                                    train_size=0.75, test_size=0.25)
+
+tpot = TPOTRegressor(generations=5, population_size=50, verbosity=2)
+tpot.fit(X_train, y_train)
+print(tpot.score(X_test, y_test))
+tpot.export('tpot_boston_pipeline.py')
+

+
+
Running this code should discover a pipeline that achieves at least 10 mean squared error (MSE) on the test set.

+
For details on how the fit(), score() and export() functions work, see the usage documentation.

+
After running the above code, the corresponding Python code should be exported to the tpot_boston_pipeline.py file and look similar to the following:

+
import numpy as np
+
+from sklearn.ensemble import GradientBoostingRegressor
+from sklearn.model_selection import train_test_split
+
+# NOTE: Make sure that the class is labeled 'class' in the data file
+tpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)
+features = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),
+                     tpot_data.dtype.names.index('class'), axis=1)
+training_features, testing_features, training_classes, testing_classes = \
+    train_test_split(features, tpot_data['class'], random_state=42)
+
+exported_pipeline = GradientBoostingRegressor(alpha=0.85, learning_rate=0.1, loss="ls",
+                                              max_features=0.9, min_samples_leaf=5,
+                                              min_samples_split=6)
+
+exported_pipeline.fit(training_features, training_classes)
+results = exported_pipeline.predict(testing_features)
+

+
+
Titanic survival analysis

+
To see the TPOT applied the Titanic Kaggle dataset, see the Jupyter notebook here. This example shows how to take a messy dataset and preprocess it such that it can be used in scikit-learn and TPOT.

+              
+            

+          

+          
+	  
+        

+      

+
+    

+    
+  

+
+  

+    
+      
+           GitHub
+      
+      
+        « Previous
+      
+      
+        Next »
+      
+    
+

+    
+
+
+
diff --git a/docs/index.html b/docs/index.html
index 8fe7fb22..cb1f68c0 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -45,95 +45,50 @@
 
       
@@ -230,7 +185,7 @@
   

     
       
-           GitHub
+           GitHub
       
       
       
@@ -244,6 +199,6 @@
 
 
 
diff --git a/docs/installing/index.html b/docs/installing/index.html
index 9c21b071..bfb34f12 100644
--- a/docs/installing/index.html
+++ b/docs/installing/index.html
@@ -45,95 +45,50 @@
 
       
@@ -251,7 +206,7 @@
   

     
       
-           GitHub
+           GitHub
       
       
         « Previous
diff --git a/docs/js/theme.js b/docs/js/theme.js
index 6396162c..f1f0a588 100644
--- a/docs/js/theme.js
+++ b/docs/js/theme.js
@@ -1,5 +1,4 @@
 $( document ).ready(function() {
-
     // Shift nav in mobile when clicking the menu.
     $(document).on('click', "[data-toggle='wy-nav-top']", function() {
       $("[data-toggle='wy-nav-shift']").toggleClass("shift");
@@ -53,3 +52,31 @@ window.SphinxRtdTheme = (function (jquery) {
         StickyNav : stickyNav
     };
 }($));
+
+// The code below is a copy of @seanmadsen code posted Jan 10, 2017 on issue 803.
+// https://github.com/mkdocs/mkdocs/issues/803
+// This just incorporates the auto scroll into the theme itself without
+// the need for additional custom.js file.
+//
+$(function() {
+  $.fn.isFullyWithinViewport = function(){
+      var viewport = {};
+      viewport.top = $(window).scrollTop();
+      viewport.bottom = viewport.top + $(window).height();
+      var bounds = {};
+      bounds.top = this.offset().top;
+      bounds.bottom = bounds.top + this.outerHeight();
+      return ( ! (
+        (bounds.top <= viewport.top) ||
+        (bounds.bottom >= viewport.bottom)
+      ) );
+  };
+  if( !$('li.toctree-l1.current').isFullyWithinViewport() ) {
+    $('.wy-nav-side')
+      .scrollTop(
+        $('li.toctree-l1.current').offset().top -
+        $('.wy-nav-side').offset().top -
+        60
+      );
+  }
+});
diff --git a/docs/mkdocs/search_index.json b/docs/mkdocs/search_index.json
index 50d9f32b..0f9d93cc 100644
--- a/docs/mkdocs/search_index.json
+++ b/docs/mkdocs/search_index.json
@@ -36,24 +36,29 @@
             "title": "Customizing TPOT's operators and parameters"
         },
         {
-            "location": "/examples/MNIST_Example/",
-            "text": "Below is a minimal working example with the practice MNIST data set.\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nRunning this code should discover a pipeline that achieves about 98% testing accuracy, and the corresponding Python code should be exported to the \ntpot_mnist_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.pipeline import make_pipeline\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR')\nfeatures = tpot_data.view((np.float64, len(tpot_data.dtype.names)))\nfeatures = np.delete(features, tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes =     train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = make_pipeline(\n    KNeighborsClassifier(n_neighbors=3, weights=\"uniform\")\n)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)",
-            "title": "MNIST Example"
+            "location": "/examples/",
+            "text": "Iris flower classification\n\n\nThe following code illustrates the usage of TPOT with the Iris data set, which is a simple supervised classification problem.\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_iris\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\n\niris = load_iris()\nX_train, X_test, y_train, y_test = train_test_split(iris.data.astype(np.float64),\n    iris.target.astype(np.float64), train_size=0.75, test_size=0.25)\n\ntpot = TPOTClassifier(generations=5, population_size=50, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_iris_pipeline.py')\n\n\n\n\nRunning this code should discover a pipeline that achieves about 97% testing accuracy.\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nAfter running the above code, the corresponding Python code should be exported to the \ntpot_iris_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import Normalizer\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = make_pipeline(\n    Normalizer(),\n    GaussianNB()\n)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)\n\n\n\n\nMNIST digit recognition\n\n\nBelow is a minimal working example with the practice MNIST data set, which is an image classification problem.\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ntpot = TPOTClassifier(generations=5, population_size=50, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')\n\n\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nRunning this code should discover a pipeline that achieves about 98% testing accuracy, and the corresponding Python code should be exported to the \ntpot_mnist_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.neighbors import KNeighborsClassifier\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = KNeighborsClassifier(n_neighbors=6, weights=\"distance\")\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)\n\n\n\n\nBoston housing prices modeling\n\n\nThe following code illustrates the usage of TPOT with the Boston housing prices data set, which is a regression problem.\n\n\nfrom tpot import TPOTRegressor\nfrom sklearn.datasets import load_boston\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_boston()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ntpot = TPOTRegressor(generations=5, population_size=50, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_boston_pipeline.py')\n\n\n\n\nRunning this code should discover a pipeline that achieves at least 10 mean squared error (MSE) on the test set.\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nAfter running the above code, the corresponding Python code should be exported to the \ntpot_boston_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.model_selection import train_test_split\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = GradientBoostingRegressor(alpha=0.85, learning_rate=0.1, loss=\"ls\",\n                                              max_features=0.9, min_samples_leaf=5,\n                                              min_samples_split=6)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)\n\n\n\n\nTitanic survival analysis\n\n\nTo see the TPOT applied the Titanic Kaggle dataset, see the Jupyter notebook \nhere\n. This example shows how to take a messy dataset and preprocess it such that it can be used in scikit-learn and TPOT.",
+            "title": "Examples"
         },
         {
-            "location": "/examples/IRIS_Example/",
-            "text": "The following code illustrates the usage of TPOT with the IRIS data set.\n\n\nfrom tpot import TPOTClassifier\nfrom sklearn.datasets import load_iris\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\n\niris = load_iris()\nX_train, X_test, y_train, y_test = train_test_split(iris.data.astype(np.float64),\n    iris.target.astype(np.float64), train_size=0.75, test_size=0.25)\n\ntpot = TPOTClassifier(generations=5, population_size=20, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_iris_pipeline.py')\n\n\n\n\nRunning this code should discover a pipeline that achieves ~96% testing accuracy.\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nAfter running the above code, the corresponding Python code should be exported to the \ntpot_iris_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import make_pipeline, make_union\nfrom sklearn.preprocessing import FunctionTransformer, PolynomialFeatures\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1), tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = make_pipeline(\n    PolynomialFeatures(degree=2, include_bias=False, interaction_only=False),\n    LogisticRegression(C=0.9, dual=False, penalty=\"l2\")\n)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)",
-            "title": "IRIS Example"
+            "location": "/examples/#iris-flower-classification",
+            "text": "The following code illustrates the usage of TPOT with the Iris data set, which is a simple supervised classification problem.  from tpot import TPOTClassifier\nfrom sklearn.datasets import load_iris\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\n\niris = load_iris()\nX_train, X_test, y_train, y_test = train_test_split(iris.data.astype(np.float64),\n    iris.target.astype(np.float64), train_size=0.75, test_size=0.25)\n\ntpot = TPOTClassifier(generations=5, population_size=50, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_iris_pipeline.py')  Running this code should discover a pipeline that achieves about 97% testing accuracy.  For details on how the  fit() ,  score()  and  export()  functions work, see the  usage documentation .  After running the above code, the corresponding Python code should be exported to the  tpot_iris_pipeline.py  file and look similar to the following:  import numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import Normalizer\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = make_pipeline(\n    Normalizer(),\n    GaussianNB()\n)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)",
+            "title": "Iris flower classification"
         },
         {
-            "location": "/examples/Boston_Example/",
-            "text": "The following code illustrates the usage of TPOT with the Boston house prices data set.\n\n\nfrom tpot import TPOTRegressor\nfrom sklearn.datasets import load_boston\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_boston()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ntpot = TPOTRegressor(generations=5, population_size=20, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_boston_pipeline.py')\n\n\n\n\nRunning this code should discover a pipeline that achieves at least 12.77 mean squared error (MSE).\n\n\nFor details on how the \nfit()\n, \nscore()\n and \nexport()\n functions work, see the \nusage documentation\n.\n\n\nAfter running the above code, the corresponding Python code should be exported to the \ntpot_boston_pipeline.py\n file and look similar to the following:\n\n\nimport numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import ExtraTreesRegressor\nfrom sklearn.pipeline import make_pipeline\n\n# NOTE: Make sure that the target is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\n\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = make_pipeline(\n    ExtraTreesRegressor(max_features=0.76, n_estimators=500)\n)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)",
-            "title": "Boston Example"
+            "location": "/examples/#mnist-digit-recognition",
+            "text": "Below is a minimal working example with the practice MNIST data set, which is an image classification problem.  from tpot import TPOTClassifier\nfrom sklearn.datasets import load_digits\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_digits()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ntpot = TPOTClassifier(generations=5, population_size=50, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_mnist_pipeline.py')  For details on how the  fit() ,  score()  and  export()  functions work, see the  usage documentation .  Running this code should discover a pipeline that achieves about 98% testing accuracy, and the corresponding Python code should be exported to the  tpot_mnist_pipeline.py  file and look similar to the following:  import numpy as np\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.neighbors import KNeighborsClassifier\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = KNeighborsClassifier(n_neighbors=6, weights=\"distance\")\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)",
+            "title": "MNIST digit recognition"
         },
         {
-            "location": "/examples/Titanic_Kaggle_Example/",
-            "text": "To see the TPOT applied the Titanic Kaggle dataset, see the Jupyter notebook \nhere\n.",
-            "title": "Titanic Kaggle Example"
+            "location": "/examples/#boston-housing-prices-modeling",
+            "text": "The following code illustrates the usage of TPOT with the Boston housing prices data set, which is a regression problem.  from tpot import TPOTRegressor\nfrom sklearn.datasets import load_boston\nfrom sklearn.model_selection import train_test_split\n\ndigits = load_boston()\nX_train, X_test, y_train, y_test = train_test_split(digits.data, digits.target,\n                                                    train_size=0.75, test_size=0.25)\n\ntpot = TPOTRegressor(generations=5, population_size=50, verbosity=2)\ntpot.fit(X_train, y_train)\nprint(tpot.score(X_test, y_test))\ntpot.export('tpot_boston_pipeline.py')  Running this code should discover a pipeline that achieves at least 10 mean squared error (MSE) on the test set.  For details on how the  fit() ,  score()  and  export()  functions work, see the  usage documentation .  After running the above code, the corresponding Python code should be exported to the  tpot_boston_pipeline.py  file and look similar to the following:  import numpy as np\n\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.model_selection import train_test_split\n\n# NOTE: Make sure that the class is labeled 'class' in the data file\ntpot_data = np.recfromcsv('PATH/TO/DATA/FILE', delimiter='COLUMN_SEPARATOR', dtype=np.float64)\nfeatures = np.delete(tpot_data.view(np.float64).reshape(tpot_data.size, -1),\n                     tpot_data.dtype.names.index('class'), axis=1)\ntraining_features, testing_features, training_classes, testing_classes = \\\n    train_test_split(features, tpot_data['class'], random_state=42)\n\nexported_pipeline = GradientBoostingRegressor(alpha=0.85, learning_rate=0.1, loss=\"ls\",\n                                              max_features=0.9, min_samples_leaf=5,\n                                              min_samples_split=6)\n\nexported_pipeline.fit(training_features, training_classes)\nresults = exported_pipeline.predict(testing_features)",
+            "title": "Boston housing prices modeling"
+        },
+        {
+            "location": "/examples/#titanic-survival-analysis",
+            "text": "To see the TPOT applied the Titanic Kaggle dataset, see the Jupyter notebook  here . This example shows how to take a messy dataset and preprocess it such that it can be used in scikit-learn and TPOT.",
+            "title": "Titanic survival analysis"
         },
         {
             "location": "/contributing/",
diff --git a/docs/releases/index.html b/docs/releases/index.html
index 442b6cae..bd19a426 100644
--- a/docs/releases/index.html
+++ b/docs/releases/index.html
@@ -45,116 +45,71 @@
 
       
@@ -354,7 +309,7 @@ 
Version 0.1

   

     
       
-           GitHub
+           GitHub
       
       
         « Previous
diff --git a/docs/search.html b/docs/search.html
index 624d949e..ac514832 100644
--- a/docs/search.html
+++ b/docs/search.html
@@ -41,91 +41,47 @@
 
       
@@ -197,7 +153,7 @@ 
Search Results

   

     
       
-           GitHub
+           GitHub
       
       
       
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index 3c43bc1f..86906610 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -26,31 +26,11 @@
     
 
     
-        
     
-     http://rhiever.github.io/tpot/examples/MNIST_Example/
+     http://rhiever.github.io/tpot/examples/
      2017-03-22
      daily
     
-        
-    
-     http://rhiever.github.io/tpot/examples/IRIS_Example/
-     2017-03-22
-     daily
-    
-        
-    
-     http://rhiever.github.io/tpot/examples/Boston_Example/
-     2017-03-22
-     daily
-    
-        
-    
-     http://rhiever.github.io/tpot/examples/Titanic_Kaggle_Example/
-     2017-03-22
-     daily
-    
-        
     
 
     
diff --git a/docs/support/index.html b/docs/support/index.html
index b5bbc0b9..dd5185c1 100644
--- a/docs/support/index.html
+++ b/docs/support/index.html
@@ -45,95 +45,50 @@
 
       
@@ -207,7 +162,7 @@
   

     
       
-           GitHub
+           GitHub
       
       
         « Previous
diff --git a/docs/using/index.html b/docs/using/index.html
index bab47183..2db911ce 100644
--- a/docs/using/index.html
+++ b/docs/using/index.html
@@ -45,105 +45,64 @@
 
       
@@ -504,7 +463,7 @@ 
TPOT with code

 

 
 
Once this code finishes running, tpot_exported_pipeline.py will contain the Python code for the optimized pipeline.

-
Check our examples to see TPOT applied to some specific data sets.

+
Check our examples to see TPOT applied to some specific data sets.

 

 
Scoring functions

 
TPOT makes use of sklearn.model_selection.cross_val_score for evaluating pipelines, and as such offers the same support for scoring functions. There are two ways to make use of scoring functions with TPOT:

@@ -595,7 +554,7 @@ 
Customizing TPOT's operators