diff --git a/autoPyTorch/evaluation/train_evaluator.py b/autoPyTorch/evaluation/train_evaluator.py index 1629451e2..fed527f2e 100644 --- a/autoPyTorch/evaluation/train_evaluator.py +++ b/autoPyTorch/evaluation/train_evaluator.py @@ -1,4 +1,6 @@ +import json from multiprocessing.queues import Queue +import os from typing import Any, Dict, List, Optional, Tuple, Union from ConfigSpace.configuration_space import Configuration @@ -195,12 +197,22 @@ def fit_predict_and_loss(self) -> None: # add learning curve of configurations to additional_run_info if isinstance(pipeline, TabularClassificationPipeline): - run_summary = pipeline.named_steps['trainer'].run_summary - split_types = ['train', 'val', 'test'] - additional_run_info['run_summary'] = dict() - for split_type in split_types: - additional_run_info['run_summary'][f'{split_type}_loss'] = run_summary.performance_tracker[f'{split_type}_loss'] - additional_run_info['run_summary'][f'{split_type}_metrics'] = run_summary.performance_tracker[f'{split_type}_metrics'] + if hasattr(pipeline.named_steps['trainer'], 'run_summary'): + run_summary = pipeline.named_steps['trainer'].run_summary + split_types = ['train', 'val', 'test'] + run_summary_dict = dict( + run_summary={}, + budget=self.budget, + seed=self.seed, + config_id=self.configuration.config_id, + num_run=self.num_run + ) + for split_type in split_types: + run_summary_dict['run_summary'][f'{split_type}_loss'] = run_summary.performance_tracker.get(f'{split_type}_loss', None) + run_summary_dict['run_summary'][f'{split_type}_metrics'] = run_summary.performance_tracker.get(f'{split_type}_metrics', None) + self.logger.debug(f"run_summary_dict {json.dumps(run_summary_dict)}") + with open(os.path.join(self.backend.temporary_directory, 'run_summary.txt'), 'a') as file: + file.write(f"{json.dumps(run_summary_dict)}\n") status = StatusType.SUCCESS diff --git a/autoPyTorch/pipeline/base_pipeline.py b/autoPyTorch/pipeline/base_pipeline.py index fa042e015..8bf3447ce 100644 --- a/autoPyTorch/pipeline/base_pipeline.py +++ b/autoPyTorch/pipeline/base_pipeline.py @@ -347,13 +347,13 @@ def _add_forbidden_conditions(self, cs): if cyclic_lr_name in available_schedulers: # disable snapshot ensembles and stochastic weight averaging snapshot_ensemble_hyperparameter = cs.get_hyperparameter(f'trainer:{trainer}:use_snapshot_ensemble') - if True in snapshot_ensemble_hyperparameter.choices: + if hasattr(snapshot_ensemble_hyperparameter, 'choices') and True in snapshot_ensemble_hyperparameter.choices: cs.add_forbidden_clause(ForbiddenAndConjunction( ForbiddenEqualsClause(snapshot_ensemble_hyperparameter, True), ForbiddenEqualsClause(cs.get_hyperparameter('lr_scheduler:__choice__'), cyclic_lr_name) )) swa_hyperparameter = cs.get_hyperparameter(f'trainer:{trainer}:use_stochastic_weight_averaging') - if True in swa_hyperparameter.choices: + if hasattr(swa_hyperparameter, 'choices') and True in swa_hyperparameter.choices: cs.add_forbidden_clause(ForbiddenAndConjunction( ForbiddenEqualsClause(swa_hyperparameter, True), ForbiddenEqualsClause(cs.get_hyperparameter('lr_scheduler:__choice__'), cyclic_lr_name) diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/__init__.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/__init__.py index 93439379b..eca46acb2 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/__init__.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/encoding/__init__.py @@ -1,7 +1,6 @@ import os from collections import OrderedDict from typing import Dict, List, Optional -import warnings import ConfigSpace.hyperparameters as CSH from ConfigSpace.configuration_space import ConfigurationSpace @@ -88,17 +87,14 @@ def get_hyperparameter_search_space(self, available_preprocessors, choice_hyperparameter.value_range)) if len(categorical_columns) == 0: - # assert len(choice_hyperparameter.value_range) == 1 - if 'NoEncoder' not in choice_hyperparameter.value_range: - warnings.warn("Provided {} in choices, however, the dataset " - "is incompatible with it, fixing it to `NoEncoder`".format(choice_hyperparameter.value_range)) - preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoEncoder'], - default_value='NoEncoder') - else: - preprocessor = CSH.CategoricalHyperparameter('__choice__', - choice_hyperparameter.value_range, - default_value=choice_hyperparameter.default_value) + assert len(choice_hyperparameter.value_range) == 1 + assert 'NoEncoder' in choice_hyperparameter.value_range, \ + "Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range) + + preprocessor = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: # add only no encoder to choice hyperparameters in case the dataset is only numerical if len(categorical_columns) == 0: diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/__init__.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/__init__.py index 4b56e3bc1..0e964ab56 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/__init__.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/feature_preprocessing/__init__.py @@ -1,7 +1,6 @@ import os from collections import OrderedDict from typing import Dict, List, Optional -import warnings import ConfigSpace.hyperparameters as CSH from ConfigSpace.configuration_space import ConfigurationSpace @@ -178,17 +177,13 @@ def get_hyperparameter_search_space(self, available_, choice_hyperparameter.value_range)) if len(numerical_columns) == 0: - # assert len(choice_hyperparameter.value_range) == 1 - if 'NoFeaturePreprocessor' not in choice_hyperparameter.value_range: - warnings.warn("Provided {} in choices, however, the dataset " - "is incompatible with it, fixing it to `NoFeaturePreprocessor`".format(choice_hyperparameter.value_range)) - preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoFeaturePreprocessor'], - default_value='NoFeaturePreprocessor') - else: - preprocessor = CSH.CategoricalHyperparameter('__choice__', - choice_hyperparameter.value_range, - default_value=choice_hyperparameter.default_value) + assert len(choice_hyperparameter.value_range) == 1 + assert 'NoFeaturePreprocessor' in choice_hyperparameter.value_range, \ + "Provided {} in choices, however, the dataset " \ + "is incompatible with it".format(choice_hyperparameter.value_range) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: # add only no feature preprocessor to choice hyperparameters in case the dataset is only categorical if len(numerical_columns) == 0: diff --git a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/__init__.py b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/__init__.py index 93806934e..d4d3ffeb5 100644 --- a/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/__init__.py +++ b/autoPyTorch/pipeline/components/preprocessing/tabular_preprocessing/scaling/__init__.py @@ -1,7 +1,6 @@ import os from collections import OrderedDict from typing import Dict, List, Optional -import warnings import ConfigSpace.hyperparameters as CSH from ConfigSpace.configuration_space import ConfigurationSpace @@ -96,18 +95,14 @@ def get_hyperparameter_search_space(self, available_scalers, choice_hyperparameter.value_range)) if len(numerical_columns) == 0: - # assert len(choice_hyperparameter.value_range) == 1 + assert len(choice_hyperparameter.value_range) == 1 if 'NoScaler' not in choice_hyperparameter.value_range: - warnings.warn("Provided {} in choices, however, the dataset " - "is incompatible with it, fixing it to `NoScaler`".format(choice_hyperparameter.value_range)) - preprocessor = CSH.CategoricalHyperparameter('__choice__', - ['NoScaler'], - default_value='NoScaler') - else: + raise ValueError("Provided {} in choices, however, the dataset " + "is incompatible with it".format(choice_hyperparameter.value_range)) - preprocessor = CSH.CategoricalHyperparameter('__choice__', - choice_hyperparameter.value_range, - default_value=choice_hyperparameter.default_value) + preprocessor = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: # add only no scaler to choice hyperparameters in case the dataset is only categorical if len(numerical_columns) == 0: diff --git a/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py b/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py index e2cdcadca..6939842f4 100644 --- a/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py +++ b/autoPyTorch/pipeline/components/setup/network_embedding/__init__.py @@ -1,7 +1,6 @@ import os from collections import OrderedDict from typing import Dict, List, Optional -import warnings import ConfigSpace.hyperparameters as CSH from ConfigSpace.configuration_space import ConfigurationSpace @@ -170,18 +169,13 @@ def get_hyperparameter_search_space( available_embedding, choice_hyperparameter.value_range)) if len(categorical_columns) == 0: - # assert len(choice_hyperparameter.value_range) == 1 + assert len(choice_hyperparameter.value_range) == 1 if 'NoEmbedding' not in choice_hyperparameter.value_range: - warnings.warn("Provided {} in choices, however, the dataset " - "is incompatible with it, fixing it to `NoEmbedding`".format(choice_hyperparameter.value_range)) - embedding = CSH.CategoricalHyperparameter('__choice__', - ['NoEmbedding'], - default_value='NoEmbedding') - else: - - embedding = CSH.CategoricalHyperparameter('__choice__', - choice_hyperparameter.value_range, - default_value=choice_hyperparameter.default_value) + raise ValueError("Provided {} in choices, however, the dataset " + "is incompatible with it".format(choice_hyperparameter.value_range)) + embedding = CSH.CategoricalHyperparameter('__choice__', + choice_hyperparameter.value_range, + default_value=choice_hyperparameter.default_value) else: if len(categorical_columns) == 0: