From 51aa88f7fb7a340ebe343cc798d1b0df11f018da Mon Sep 17 00:00:00 2001 From: dengdifan Date: Tue, 28 Jun 2022 17:21:03 +0200 Subject: [PATCH 1/7] docs for forecasting task --- README.md | 97 ++++++++++++++++++++++++++++++++++++++++++- docs/api.rst | 17 ++++++++ docs/dev.rst | 18 +++++++- docs/installation.rst | 16 +++++++ docs/manual.rst | 36 ++++++++++++++-- 5 files changed, 177 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9f7ae78ae..1fbbcdf82 100755 --- a/README.md +++ b/README.md @@ -4,8 +4,9 @@ Copyright (C) 2021 [AutoML Groups Freiburg and Hannover](http://www.automl.org/ While early AutoML frameworks focused on optimizing traditional ML pipelines and their hyperparameters, another trend in AutoML is to focus on neural architecture search. To bring the best of these two worlds together, we developed **Auto-PyTorch**, which jointly and robustly optimizes the network architecture and the training hyperparameters to enable fully automated deep learning (AutoDL). -Auto-PyTorch is mainly developed to support tabular data (classification, regression). +Auto-PyTorch is mainly developed to support tabular data (classification, regression) and time series data (forecasting). The newest features in Auto-PyTorch for tabular data are described in the paper ["Auto-PyTorch Tabular: Multi-Fidelity MetaLearning for Efficient and Robust AutoDL"](https://arxiv.org/abs/2006.13799) (see below for bibtex ref). +Details about Auto-PyTorch for multi-horizontal time series forecasting tasks can be found in the paper ["Efficient Automated Deep Learning for Time Series Forecasting"](https://arxiv.org/abs/2205.05511) (also see below for bibtex ref). Also, find the documentation [here](https://automl.github.io/Auto-PyTorch/master). @@ -27,7 +28,9 @@ In other words, we evaluate the portfolio on a provided data as initial configur Then API starts the following procedures: 1. **Validate input data**: Process each data type, e.g. encoding categorical data, so that Auto-Pytorch can handled. 2. **Create dataset**: Create a dataset that can be handled in this API with a choice of cross validation or holdout splits. -3. **Evaluate baselines** *1: Train each algorithm in the predefined pool with a fixed hyperparameter configuration and dummy model from `sklearn.dummy` that represents the worst possible performance. +3. **Evaluate baselines** + * ***Tabular dataset*** *1: Train each algorithm in the predefined pool with a fixed hyperparameter configuration and dummy model from `sklearn.dummy` that represents the worst possible performance. + * ***Time Series Forecasting dataset*** : Train a dummy predictor that repeats the last observed value in each series 4. **Search by [SMAC](https://github.com/automl/SMAC3)**:\ a. Determine budget and cut-off rules by [Hyperband](https://jmlr.org/papers/volume18/16-558/16-558.pdf)\ b. Sample a pipeline hyperparameter configuration *2 by SMAC\ @@ -50,6 +53,14 @@ pip install autoPyTorch ``` +Auto-PyTorch for Time Series Forecasting requires additional dependencies + +```sh + +pip install autoPyTorch[forecasting] + +``` + ### Manual Installation We recommend using Anaconda for developing as follows: @@ -70,6 +81,20 @@ python setup.py install ``` +Similarly, to install all the dependencies for Auto-PyTorch-TimeSeriesForecasting: + + +```sh + +git submodule update --init --recursive + +conda create -n auto-pytorch python=3.8 +conda activate auto-pytorch +conda install swig +pip install -e[forecasting] + +``` + ## Examples In a nutshell: @@ -105,6 +130,63 @@ score = api.score(y_pred, y_test) print("Accuracy score", score) ``` +For Time Series Forecasting Tasks +```py + +from autoPyTorch.api.time_series_forecasting import TimeSeriesForecastingTask + +# data and metric imports +from sktime.datasets import load_longley +targets, features = load_longley() + +# define the forecasting horizon +forecasting_horizon = 3 + +# each series represent an element in the List +# we take the last forecasting_horizon as test targets. The itme before that as training targets +# Normally the value to be forecasted should follow the training sets +y_train = [targets[: -forecasting_horizon]] +y_test = [targets[-forecasting_horizon:]] + +# same for features. For uni-variant models, X_train, X_test can be omitted +X_train = [features[: -forecasting_horizon]] +# Here x_test indicates the 'known future features': they are the features known previously, features that are unknown +# could be replaced with NAN or zeros (which will not be used by our networks). If no feature is known beforehand, +# we could also omit X_test +known_future_features = list(features.columns) +X_test = [features[-forecasting_horizon:]] + +start_times = [targets.index.to_timestamp()[0]] +freq = '1Y' + +# initialise Auto-PyTorch api +api = TimeSeriesForecastingTask() + +# Search for an ensemble of machine learning algorithms +api.search( + X_train=X_train, + y_train=y_train, + X_test=X_test, + optimize_metric='mean_MAPE_forecasting', + n_prediction_steps=forecasting_horizon, + memory_limit=16 * 1024, # Currently, forecasting models need much more memories than it actually requires + freq=freq, + start_times=start_times, + func_eval_time_limit_secs=50, + total_walltime_limit=60, + min_num_test_instances=1000, # proxy validation sets. This only works for the tasks with more than 1000 series + known_future_features=known_future_features, +) + +# our dataset could directly generate sequences for new datasets +test_sets = api.dataset.generate_test_seqs() + +# Calculate test accuracy +y_pred = api.predict(test_sets) +score = api.score(y_pred, y_test) +print("Forecasting score", score) +``` + For more examples including customising the search space, parellising the code, etc, checkout the `examples` folder ```sh @@ -163,6 +245,17 @@ Please refer to the branch `TPAMI.2021.3067763` to reproduce the paper *Auto-PyT } ``` +```bibtex +@article{deng-ecml22, + author = {Difan Deng and Florian Karl and Frank Hutter and Bernd Bischl and Marius Lindauer}, + title = {Efficient Automated Deep Learning for Time Series Forecasting}, + year = {2022}, + booktitle = {Machine Learning and Knowledge Discovery in Databases. Research Track + - European Conference, {ECML} {PKDD} 2022}, + url = {https://doi.org/10.48550/arXiv.2205.05511}, +} +``` + ## Contact Auto-PyTorch is developed by the [AutoML Groups of the University of Freiburg and Hannover](http://www.automl.org/). diff --git a/docs/api.rst b/docs/api.rst index 00ff11d08..f54dd1e90 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -25,6 +25,15 @@ Regression :members: :inherited-members: search, refit, predict, score +~~~~~~~~~~~~~~ +Time Series Forecasting +~~~~~~~~~~~~~~ + +.. autoclass:: autoPyTorch.api.time_series_forecasting.TimeSeriesForecastingTask + :members: + :inherited-members: search, refit, predict, score + + ========= Pipelines @@ -50,6 +59,14 @@ Tabular Regression .. autoclass:: autoPyTorch.pipeline.traditional_tabular_regression.TraditionalTabularRegressionPipeline :members: +~~~~~~~~~~~~~~~~~~ +Time Series Forecasting +~~~~~~~~~~~~~~~~~~ + +.. autoclass:: autoPyTorch.pipeline.time_series_forecasting.TimeSeriesForecastingPipeline + :members: + + ================= Steps in Pipeline ================= diff --git a/docs/dev.rst b/docs/dev.rst index a3c154cd7..2c297056c 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -60,10 +60,23 @@ handle column-reordering. Note that column-reordering shifts categorical columns to the earlier indices and it is activated only if one uses a ColumnTransformer. +Similar procedures can be found under time series forecasting tasks: + +#. `Feature Imputation `_ +#. `Feature scaling `_ +#. `Feature Encoding `_ +#. `Feature preprocessing `_ +#. `Target Imputation `_ +#. `Target Preprocessing `_ +#. `Target Scaling `_ +#. `Loss Types `_ +#. `Algorithm setup `_ +#. `Training `_ + Training of individual models ----------------------------- -Auto-PyTorch can fit 3 types of pipelines: +**Auto-PyTorch Tabular** can fit 3 types of pipelines: #. Dummy pipeline: Use sklearn.dummy to construct an estimator that predicts using simple rules such as most frequent class #. Traditional machine learning pipelines: Use LightGBM, CatBoost, RandomForest, ExtraTrees, K-Nearest-Neighbors, and SupportVectorMachines @@ -78,6 +91,9 @@ and data loaders required to perform the neural architecture search. After the training (fitting a pipeline), we use pickle to save it to disk as stated `here `_. +**Auto-PyTorch Time Series Forecasting** currently only allows Dummy pipelines and PyTorch neural networks. Traditional machine learning pipelines +will be introduced in the future iteration. + Optimization of pipeline ------------------------ diff --git a/docs/installation.rst b/docs/installation.rst index 10d0bbcba..f7e818593 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -25,6 +25,12 @@ PyPI Installation .. code:: bash pip install autoPyTorch +Auto-PyTorch for Time Series Forecasting requires additional dependencies + +.. code:: bash + pip install autoPyTorch[forecasting] + + Manual Installation ------------------- @@ -44,6 +50,16 @@ Manual Installation cat requirements.txt | xargs -n 1 -L 1 pip install python setup.py install +Similarly, Auto-PyTorch for time series forecasting requires additional dependencies + +.. code:: bash + git submodule update --init --recursive + + conda create -n auto-pytorch python=3.8 + conda activate auto-pytorch + conda install swig + pip install -e[forecasting] + Docker Image ============ diff --git a/docs/manual.rst b/docs/manual.rst index fabee8422..c32862111 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -18,23 +18,34 @@ Examples ======== * `Classification `_ * `Regression `_ +* `Forecasting `_ * `Customizing the search space `_ * `Changing the resampling strategy `_ * `Visualizing the results `_ Data validation =============== -For tabular tasks, *Auto-PyTorch* uses a feature and target validator on the input feature set and target set respectively. +For **tabular tasks**, *Auto-PyTorch* uses a feature and target validator on the input feature set and target set respectively. The feature validator checks whether the data is supported by *Auto-PyTorch* or not. Additionally, a sklearn column transformer is also used which imputes and ordinally encodes the categorical columns of the dataset. This ensures that no unseen category is found while fitting the data. -The target validator applies a label encoder on the target column. +The target validator applies a label encoder on the target column. + +For **time series forecasting tasks**, besides the functions described above, time series forecasting validators will also +check the information specify for time series forecasting tasks: it checks + * The index of the series that each data point belongs to + * if the dataset is uni-variant (only targets information is contained in the datasets) + * the sample frequency of the datasets + * the static features in the dataset, i.e., features that contain only one value within each series + +Time Series forecasting validator then transforms the features and targets into a `pd.DataFrame `_ +whose index is applied to identify the series that the time step belongs to. Data Preprocessing ================== -The tabular preprocessing pipeline in *Auto-PyTorch* consists of +The **tabular preprocessing pipeline** in *Auto-PyTorch* consists of #. `Imputation `_ #. `Encoding `_ @@ -47,7 +58,24 @@ The tabular preprocessing pipeline in *Auto-PyTorch* consists of Along with the choices, their corresponding hyperparameters are also tuned. A sklearn ColumnTransformer is created which includes a categorical pipeline and a numerical pipeline. These pipelines are made up of the relevant preprocessors chosen in the previous steps. The column transformer is compatible with `torchvision transforms `_ -and is therefore passed to the DataLoader. +and is therefore passed to the DataLoader. + +**time series forecasting pipeline** has two sorts of setup: + +- Uni-variant model only requires target transformations. They include *1: + #. `Target Imputation `_ + Choice of `linear`, `nearest`, `constant_zero`, `bfill` and `ffill` +- Multi-variant model contains target transformations (see above) and feature transformation. They include + #. `Imputation `_ + Choice of `linear`, `nearest`, `constant_zero`, `bfill` and `ffill` + #. `Scaling `_ + Choice of `standard`, `min_max`, `max_abs`, `mean_abs`, or no transformation *2 + #. `Encoding `_ + Choice of `OneHotEncoder` or no encoding. + +*1 Target scaling is considered as part of `setup `_ and the transform is done within each batch iteration + +*2 Scaling is transformed within each series Resource Allocation =================== From 6a969f18c0aa27979cfbdc9ee45352951ff25884 Mon Sep 17 00:00:00 2001 From: dengdifan Date: Tue, 28 Jun 2022 20:07:19 +0200 Subject: [PATCH 2/7] avoid directly import extra dependencies --- autoPyTorch/api/time_series_forecasting.py | 3 +- autoPyTorch/constants.py | 10 + autoPyTorch/evaluation/abstract_evaluator.py | 70 +---- autoPyTorch/evaluation/tae.py | 9 +- ...time_series_forecasting_train_evaluator.py | 2 +- autoPyTorch/evaluation/utils_extra.py | 72 +++++ .../components/training/metrics/metrics.py | 255 +++++++++--------- .../components/training/metrics/utils.py | 5 + autoPyTorch/utils/pipeline.py | 11 +- test/test_api/utils.py | 2 +- 10 files changed, 253 insertions(+), 186 deletions(-) create mode 100644 autoPyTorch/evaluation/utils_extra.py diff --git a/autoPyTorch/api/time_series_forecasting.py b/autoPyTorch/api/time_series_forecasting.py index b2221b45a..9498392e9 100644 --- a/autoPyTorch/api/time_series_forecasting.py +++ b/autoPyTorch/api/time_series_forecasting.py @@ -7,8 +7,7 @@ from autoPyTorch.api.base_task import BaseTask from autoPyTorch.automl_common.common.utils.backend import Backend from autoPyTorch.constants import MAX_WINDOW_SIZE_BASE, TASK_TYPES_TO_STRING, TIMESERIES_FORECASTING -from autoPyTorch.data.time_series_forecasting_validator import \ - TimeSeriesForecastingInputValidator +from autoPyTorch.data.time_series_forecasting_validator import TimeSeriesForecastingInputValidator from autoPyTorch.data.utils import ( DatasetCompressionSpec, get_dataset_compression_mapping diff --git a/autoPyTorch/constants.py b/autoPyTorch/constants.py index d7bb38b45..4a99011d5 100644 --- a/autoPyTorch/constants.py +++ b/autoPyTorch/constants.py @@ -54,7 +54,17 @@ CLASSIFICATION_OUTPUTS = [BINARY, MULTICLASS, MULTICLASSMULTIOUTPUT] REGRESSION_OUTPUTS = [CONTINUOUS, CONTINUOUSMULTIOUTPUT] + # Constants for Forecasting Tasks +# Exceptions +class ForecastingDependenciesNotInstalledError(ModuleNotFoundError): + def __init__(self, msg=None): + if msg is None: + msg = "Additional dependencies must be installed to work with time series forecasting" \ + "tasks! Please run \n pip install autoPyTorch[forecasting] \n to install the" \ + "corresponding dependencies!" + super().__init__(msg) + # The constant values for time series forecasting comes from # https://github.com/rakshitha123/TSForecasting/blob/master/experiments/deep_learning_experiments.py diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index f5f10f664..d8a511893 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -19,7 +19,11 @@ import autoPyTorch.pipeline.image_classification import autoPyTorch.pipeline.tabular_classification import autoPyTorch.pipeline.tabular_regression -import autoPyTorch.pipeline.time_series_forecasting +try: + import autoPyTorch.pipeline.time_series_forecasting + forecasting_dependencies_installed = True +except ModuleNotFoundError: + forecasting_dependencies_installed = False import autoPyTorch.pipeline.traditional_tabular_classification import autoPyTorch.pipeline.traditional_tabular_regression from autoPyTorch.automl_common.common.utils.backend import Backend @@ -27,6 +31,7 @@ CLASSIFICATION_TASKS, FORECASTING_BUDGET_TYPE, FORECASTING_TASKS, + ForecastingDependenciesNotInstalledError, IMAGE_TASKS, MULTICLASS, REGRESSION_TASKS, @@ -38,12 +43,16 @@ BaseDataset, BaseDatasetPropertiesType ) -from autoPyTorch.datasets.time_series_dataset import TimeSeriesSequence from autoPyTorch.evaluation.utils import ( DisableFileOutputParameters, VotingRegressorWrapper, convert_multioutput_multiclass_to_multilabel ) +try: + from autoPyTorch.evaluation.utils_extra import DummyTimeSeriesForecastingPipeline + forecasting_dependencies_installed = True +except ModuleNotFoundError: + forecasting_dependencies_installed = False from autoPyTorch.pipeline.base_pipeline import BasePipeline from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric from autoPyTorch.pipeline.components.training.metrics.utils import ( @@ -314,61 +323,6 @@ def get_default_pipeline_options() -> Dict[str, Any]: 'runtime': 1} -class DummyTimeSeriesForecastingPipeline(DummyClassificationPipeline): - """ - A wrapper class that holds a pipeline for dummy forecasting. For each series, it simply repeats the last element - in the training series - - - Attributes: - random_state (Optional[Union[int, np.random.RandomState]]): - Object that contains a seed and allows for reproducible results - init_params (Optional[Dict]): - An optional dictionary that is passed to the pipeline's steps. It complies - a similar function as the kwargs - n_prediction_steps (int): - forecasting horizon - """ - def __init__(self, config: Configuration, - random_state: Optional[Union[int, np.random.RandomState]] = None, - init_params: Optional[Dict] = None, - n_prediction_steps: int = 1, - ) -> None: - super(DummyTimeSeriesForecastingPipeline, self).__init__(config, random_state, init_params) - self.n_prediction_steps = n_prediction_steps - - def fit(self, X: Dict[str, Any], y: Any, - sample_weight: Optional[np.ndarray] = None) -> object: - self.n_prediction_steps = X['dataset_properties']['n_prediction_steps'] - y_train = subsampler(X['y_train'], X['train_indices']) - return DummyClassifier.fit(self, np.ones((y_train.shape[0], 1)), y_train, sample_weight) - - def _generate_dummy_forecasting(self, X: List[Union[TimeSeriesSequence, np.ndarray]]) -> List: - if isinstance(X[0], TimeSeriesSequence): - X_tail = [x.get_target_values(-1) for x in X] - else: - X_tail = [x[-1] for x in X] - return X_tail - - def predict_proba(self, X: Union[np.ndarray, pd.DataFrame], - batch_size: int = 1000) -> np.ndarray: - X_tail = self._generate_dummy_forecasting(X) - return np.tile(X_tail, (1, self.n_prediction_steps)).astype(np.float32).flatten() - - def predict(self, X: Union[np.ndarray, pd.DataFrame], - batch_size: int = 1000) -> np.ndarray: - X_tail = np.asarray(self._generate_dummy_forecasting(X)) - if X_tail.ndim == 1: - X_tail = np.expand_dims(X_tail, -1) - return np.tile(X_tail, (1, self.n_prediction_steps)).astype(np.float32).flatten() - - @staticmethod - def get_default_pipeline_options() -> Dict[str, Any]: - return {'budget_type': 'epochs', - 'epochs': 1, - 'runtime': 1} - - def fit_and_suppress_warnings(logger: PicklableClientLogger, pipeline: BaseEstimator, X: Dict[str, Any], y: Any ) -> BaseEstimator: @@ -543,6 +497,8 @@ def __init__(self, backend: Backend, self.predict_function = self._predict_proba elif self.task_type in FORECASTING_TASKS: if isinstance(self.configuration, int): + if not forecasting_dependencies_installed: + raise ForecastingDependenciesNotInstalledError self.pipeline_class = DummyTimeSeriesForecastingPipeline elif isinstance(self.configuration, str): raise ValueError("Only tabular classifications tasks " diff --git a/autoPyTorch/evaluation/tae.py b/autoPyTorch/evaluation/tae.py index b144da76f..1b701afab 100644 --- a/autoPyTorch/evaluation/tae.py +++ b/autoPyTorch/evaluation/tae.py @@ -25,6 +25,7 @@ from autoPyTorch.automl_common.common.utils.backend import Backend from autoPyTorch.constants import ( FORECASTING_BUDGET_TYPE, + ForecastingDependenciesNotInstalledError, STRING_TO_TASK_TYPES, TIMESERIES_FORECASTING, ) @@ -34,7 +35,11 @@ NoResamplingStrategyTypes ) from autoPyTorch.evaluation.test_evaluator import eval_test_function -from autoPyTorch.evaluation.time_series_forecasting_train_evaluator import forecasting_eval_train_function +try: + from autoPyTorch.evaluation.time_series_forecasting_train_evaluator import forecasting_eval_train_function + forecasting_dependencies_installed = True +except ModuleNotFoundError: + forecasting_dependencies_installed = False from autoPyTorch.evaluation.train_evaluator import eval_train_function from autoPyTorch.evaluation.utils import ( DisableFileOutputParameters, @@ -152,6 +157,8 @@ def __init__( self.resampling_strategy_args = dm.resampling_strategy_args if STRING_TO_TASK_TYPES.get(dm.task_type, -1) == TIMESERIES_FORECASTING: + if not forecasting_dependencies_installed: + raise ForecastingDependenciesNotInstalledError eval_function: Callable = forecasting_eval_train_function if isinstance(self.resampling_strategy, (HoldoutValTypes, CrossValTypes)): self.output_y_hat_optimization = output_y_hat_optimization diff --git a/autoPyTorch/evaluation/time_series_forecasting_train_evaluator.py b/autoPyTorch/evaluation/time_series_forecasting_train_evaluator.py index 729399321..0940d1e9a 100644 --- a/autoPyTorch/evaluation/time_series_forecasting_train_evaluator.py +++ b/autoPyTorch/evaluation/time_series_forecasting_train_evaluator.py @@ -13,9 +13,9 @@ from autoPyTorch.automl_common.common.utils.backend import Backend from autoPyTorch.constants import SEASONALITY_MAP -from autoPyTorch.evaluation.abstract_evaluator import DummyTimeSeriesForecastingPipeline from autoPyTorch.evaluation.train_evaluator import TrainEvaluator from autoPyTorch.evaluation.utils import DisableFileOutputParameters +from autoPyTorch.evaluation.utils_extra import DummyTimeSeriesForecastingPipeline from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric from autoPyTorch.pipeline.components.training.metrics.metrics import MASE_LOSSES from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates diff --git a/autoPyTorch/evaluation/utils_extra.py b/autoPyTorch/evaluation/utils_extra.py new file mode 100644 index 000000000..0201bacee --- /dev/null +++ b/autoPyTorch/evaluation/utils_extra.py @@ -0,0 +1,72 @@ +# The functions and classes implemented in this module all require extra requirements. +# We put them here to make it easier to be wrapped by try-except process +from typing import Any, Dict, List, Optional, Union + +from ConfigSpace import Configuration + +import numpy as np + +import pandas as pd + +from sklearn.dummy import DummyClassifier + +from autoPyTorch.datasets.time_series_dataset import TimeSeriesSequence +from autoPyTorch.utils.common import subsampler + + +class DummyTimeSeriesForecastingPipeline(DummyClassifier): + """ + A wrapper class that holds a pipeline for dummy forecasting. For each series, it simply repeats the last element + in the training series + + + Attributes: + random_state (Optional[Union[int, np.random.RandomState]]): + Object that contains a seed and allows for reproducible results + init_params (Optional[Dict]): + An optional dictionary that is passed to the pipeline's steps. It complies + a similar function as the kwargs + n_prediction_steps (int): + forecasting horizon + """ + def __init__(self, config: Configuration, + random_state: Optional[Union[int, np.random.RandomState]] = None, + init_params: Optional[Dict] = None, + n_prediction_steps: int = 1, + ) -> None: + self.config = config + self.init_params = init_params + self.random_state = random_state + super(DummyTimeSeriesForecastingPipeline, self).__init__(strategy="uniform") + self.n_prediction_steps = n_prediction_steps + + def fit(self, X: Dict[str, Any], y: Any, + sample_weight: Optional[np.ndarray] = None) -> object: + self.n_prediction_steps = X['dataset_properties']['n_prediction_steps'] + y_train = subsampler(X['y_train'], X['train_indices']) + return DummyClassifier.fit(self, np.ones((y_train.shape[0], 1)), y_train, sample_weight) + + def _generate_dummy_forecasting(self, X: List[Union[TimeSeriesSequence, np.ndarray]]) -> List: + if isinstance(X[0], TimeSeriesSequence): + X_tail = [x.get_target_values(-1) for x in X] + else: + X_tail = [x[-1] for x in X] + return X_tail + + def predict_proba(self, X: Union[np.ndarray, pd.DataFrame], + batch_size: int = 1000) -> np.ndarray: + X_tail = self._generate_dummy_forecasting(X) + return np.tile(X_tail, (1, self.n_prediction_steps)).astype(np.float32).flatten() + + def predict(self, X: Union[np.ndarray, pd.DataFrame], + batch_size: int = 1000) -> np.ndarray: + X_tail = np.asarray(self._generate_dummy_forecasting(X)) + if X_tail.ndim == 1: + X_tail = np.expand_dims(X_tail, -1) + return np.tile(X_tail, (1, self.n_prediction_steps)).astype(np.float32).flatten() + + @staticmethod + def get_default_pipeline_options() -> Dict[str, Any]: + return {'budget_type': 'epochs', + 'epochs': 1, + 'runtime': 1} diff --git a/autoPyTorch/pipeline/components/training/metrics/metrics.py b/autoPyTorch/pipeline/components/training/metrics/metrics.py index 51921dffb..5fa60a24d 100644 --- a/autoPyTorch/pipeline/components/training/metrics/metrics.py +++ b/autoPyTorch/pipeline/components/training/metrics/metrics.py @@ -5,7 +5,11 @@ import sklearn.metrics -import sktime.performance_metrics.forecasting as forecasting_metrics +try: + import sktime.performance_metrics.forecasting as forecasting_metrics + forecasting_dependencies_installed = True +except ModuleNotFoundError: + forecasting_dependencies_installed = False from smac.utils.constants import MAXINT @@ -52,6 +56,49 @@ sklearn.metrics.f1_score) +# Score functions that need decision values +roc_auc = make_metric('roc_auc', sklearn.metrics.roc_auc_score, needs_threshold=True) +average_precision = make_metric('average_precision', + sklearn.metrics.average_precision_score, + needs_threshold=True) +precision = make_metric('precision', + sklearn.metrics.precision_score) +recall = make_metric('recall', + sklearn.metrics.recall_score) + +# Score function for probabilistic classification +log_loss = make_metric('log_loss', + sklearn.metrics.log_loss, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + needs_proba=True) + +REGRESSION_METRICS = dict() +for scorer in [mean_absolute_error, mean_squared_error, root_mean_squared_error, + mean_squared_log_error, median_absolute_error, r2]: + REGRESSION_METRICS[scorer.name] = scorer + +CLASSIFICATION_METRICS = dict() + +for scorer in [accuracy, balanced_accuracy, roc_auc, average_precision, + log_loss]: + CLASSIFICATION_METRICS[scorer.name] = scorer + +for name, metric in [('precision', sklearn.metrics.precision_score), + ('recall', sklearn.metrics.recall_score), + ('f1', sklearn.metrics.f1_score)]: + globals()[name] = make_metric(name, metric) + CLASSIFICATION_METRICS[name] = globals()[name] + for average in ['macro', 'micro', 'samples', 'weighted']: + qualified_name = '{0}_{1}'.format(name, average) + globals()[qualified_name] = make_metric(qualified_name, + partial(metric, + pos_label=None, + average=average)) + CLASSIFICATION_METRICS[qualified_name] = globals()[qualified_name] + + # Standard Forecasting Scores # To avoid storing unnecessary scale values here, we scale all the values under @@ -97,125 +144,87 @@ def compute_mase_coefficient(past_target: Union[List, np.ndarray], sp: int) -> n ) -mean_MASE_forecasting = make_metric('mean_MASE_forecasting', - forecasting_metrics.mean_absolute_error, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - do_forecasting=True, - aggregation='mean', - ) - -median_MASE_forecasting = make_metric('median_MASE_forecasting', - forecasting_metrics.mean_absolute_error, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - do_forecasting=True, - aggregation='median', - ) - -MASE_LOSSES = [mean_MASE_forecasting, median_MASE_forecasting] - -mean_MAE_forecasting = make_metric('mean_MAE_forecasting', - forecasting_metrics.mean_absolute_error, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - do_forecasting=True, - aggregation='mean', - ) - -median_MAE_forecasting = make_metric('median_MAE_forecasting', - forecasting_metrics.mean_absolute_error, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - do_forecasting=True, - aggregation='median', - ) - -mean_MAPE_forecasting = make_metric('mean_MAPE_forecasting', - forecasting_metrics.mean_absolute_percentage_error, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - do_forecasting=True, - aggregation='mean', - ) - -median_MAPE_forecasting = make_metric('median_MAPE_forecasting', - forecasting_metrics.mean_absolute_percentage_error, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - do_forecasting=True, - aggregation='median', - ) - -mean_MSE_forecasting = make_metric('mean_MSE_forecasting', - forecasting_metrics.mean_squared_error, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - do_forecasting=True, - aggregation='mean', - ) - -median_MSE_forecasting = make_metric('median_MSE_forecasting', - forecasting_metrics.mean_squared_error, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - do_forecasting=True, - aggregation='median', - ) - -# Score functions that need decision values -roc_auc = make_metric('roc_auc', sklearn.metrics.roc_auc_score, needs_threshold=True) -average_precision = make_metric('average_precision', - sklearn.metrics.average_precision_score, - needs_threshold=True) -precision = make_metric('precision', - sklearn.metrics.precision_score) -recall = make_metric('recall', - sklearn.metrics.recall_score) - -# Score function for probabilistic classification -log_loss = make_metric('log_loss', - sklearn.metrics.log_loss, - optimum=0, - worst_possible_result=MAXINT, - greater_is_better=False, - needs_proba=True) - -REGRESSION_METRICS = dict() -for scorer in [mean_absolute_error, mean_squared_error, root_mean_squared_error, - mean_squared_log_error, median_absolute_error, r2]: - REGRESSION_METRICS[scorer.name] = scorer - -CLASSIFICATION_METRICS = dict() - -for scorer in [accuracy, balanced_accuracy, roc_auc, average_precision, - log_loss]: - CLASSIFICATION_METRICS[scorer.name] = scorer - -FORECASTING_METRICS = dict() -for scorer in [mean_MASE_forecasting, median_MASE_forecasting, - mean_MAE_forecasting, median_MAE_forecasting, - mean_MAPE_forecasting, median_MAPE_forecasting, - mean_MSE_forecasting, median_MSE_forecasting]: - FORECASTING_METRICS[scorer.name] = scorer - -for name, metric in [('precision', sklearn.metrics.precision_score), - ('recall', sklearn.metrics.recall_score), - ('f1', sklearn.metrics.f1_score)]: - globals()[name] = make_metric(name, metric) - CLASSIFICATION_METRICS[name] = globals()[name] - for average in ['macro', 'micro', 'samples', 'weighted']: - qualified_name = '{0}_{1}'.format(name, average) - globals()[qualified_name] = make_metric(qualified_name, - partial(metric, - pos_label=None, - average=average)) - CLASSIFICATION_METRICS[qualified_name] = globals()[qualified_name] +if forecasting_dependencies_installed: + mean_MASE_forecasting = make_metric('mean_MASE_forecasting', + forecasting_metrics.mean_absolute_error, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + do_forecasting=True, + aggregation='mean', + ) + + median_MASE_forecasting = make_metric('median_MASE_forecasting', + forecasting_metrics.mean_absolute_error, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + do_forecasting=True, + aggregation='median', + ) + + MASE_LOSSES = [mean_MASE_forecasting, median_MASE_forecasting] + + mean_MAE_forecasting = make_metric('mean_MAE_forecasting', + forecasting_metrics.mean_absolute_error, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + do_forecasting=True, + aggregation='mean', + ) + + median_MAE_forecasting = make_metric('median_MAE_forecasting', + forecasting_metrics.mean_absolute_error, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + do_forecasting=True, + aggregation='median', + ) + + mean_MAPE_forecasting = make_metric('mean_MAPE_forecasting', + forecasting_metrics.mean_absolute_percentage_error, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + do_forecasting=True, + aggregation='mean', + ) + + median_MAPE_forecasting = make_metric('median_MAPE_forecasting', + forecasting_metrics.mean_absolute_percentage_error, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + do_forecasting=True, + aggregation='median', + ) + + mean_MSE_forecasting = make_metric('mean_MSE_forecasting', + forecasting_metrics.mean_squared_error, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + do_forecasting=True, + aggregation='mean', + ) + + median_MSE_forecasting = make_metric('median_MSE_forecasting', + forecasting_metrics.mean_squared_error, + optimum=0, + worst_possible_result=MAXINT, + greater_is_better=False, + do_forecasting=True, + aggregation='median', + ) + + FORECASTING_METRICS = dict() + for scorer in [mean_MASE_forecasting, median_MASE_forecasting, + mean_MAE_forecasting, median_MAE_forecasting, + mean_MAPE_forecasting, median_MAPE_forecasting, + mean_MSE_forecasting, median_MSE_forecasting]: + FORECASTING_METRICS[scorer.name] = scorer +else: + MASE_LOSSES = [] + FORECASTING_METRICS = dict() diff --git a/autoPyTorch/pipeline/components/training/metrics/utils.py b/autoPyTorch/pipeline/components/training/metrics/utils.py index 80adfbe73..abb763e7e 100644 --- a/autoPyTorch/pipeline/components/training/metrics/utils.py +++ b/autoPyTorch/pipeline/components/training/metrics/utils.py @@ -6,6 +6,7 @@ from autoPyTorch.constants import ( CLASSIFICATION_TASKS, FORECASTING_TASKS, + ForecastingDependenciesNotInstalledError, REGRESSION_TASKS, STRING_TO_TASK_TYPES, TASK_TYPES, @@ -47,6 +48,8 @@ def get_supported_metrics(dataset_properties: Dict[str, Any]) -> Dict[str, autoP elif STRING_TO_TASK_TYPES[task_type] in CLASSIFICATION_TASKS: return CLASSIFICATION_METRICS elif STRING_TO_TASK_TYPES[task_type] in FORECASTING_TASKS: + if len(FORECASTING_METRICS) == 0: + raise ForecastingDependenciesNotInstalledError return FORECASTING_METRICS else: raise NotImplementedError(task_type) @@ -125,6 +128,8 @@ def calculate_score( ) -> Dict[str, float]: score_dict = dict() if task_type in FORECASTING_TASKS: + if len(MASE_LOSSES) == 0: + raise ForecastingDependenciesNotInstalledError cprediction = sanitize_array(prediction) for metric_ in metrics: if metric_ in MASE_LOSSES and 'mase_coefficient' in score_kwargs: diff --git a/autoPyTorch/utils/pipeline.py b/autoPyTorch/utils/pipeline.py index 570dbc0e8..d275baf8a 100644 --- a/autoPyTorch/utils/pipeline.py +++ b/autoPyTorch/utils/pipeline.py @@ -6,6 +6,7 @@ from autoPyTorch.constants import ( CLASSIFICATION_TASKS, FORECASTING_TASKS, + ForecastingDependenciesNotInstalledError, IMAGE_TASKS, REGRESSION_TASKS, STRING_TO_TASK_TYPES, @@ -14,7 +15,11 @@ from autoPyTorch.pipeline.image_classification import ImageClassificationPipeline from autoPyTorch.pipeline.tabular_classification import TabularClassificationPipeline from autoPyTorch.pipeline.tabular_regression import TabularRegressionPipeline -from autoPyTorch.pipeline.time_series_forecasting import TimeSeriesForecastingPipeline +try: + from autoPyTorch.pipeline.time_series_forecasting import TimeSeriesForecastingPipeline + forecasting_dependencies_installed = True +except ModuleNotFoundError: + forecasting_dependencies_installed = False from autoPyTorch.utils.common import FitRequirement from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates @@ -73,6 +78,8 @@ def get_dataset_requirements(info: Dict[str, Any], search_space_updates=search_space_updates ) else: + if not forecasting_dependencies_installed: + raise ForecastingDependenciesNotInstalledError return _get_forecasting_dataset_requirements(info, include if include is not None else {}, exclude if exclude is not None else {}, @@ -129,6 +136,8 @@ def _get_forecasting_dataset_requirements(info: Dict[str, Any], task_type = STRING_TO_TASK_TYPES[info['task_type']] if task_type in FORECASTING_TASKS: + if not forecasting_dependencies_installed: + raise ForecastingDependenciesNotInstalledError return TimeSeriesForecastingPipeline( dataset_properties=info, include=include, diff --git a/test/test_api/utils.py b/test/test_api/utils.py index 968fafe5b..bbee9a3c4 100644 --- a/test/test_api/utils.py +++ b/test/test_api/utils.py @@ -6,11 +6,11 @@ from autoPyTorch.evaluation.abstract_evaluator import ( DummyClassificationPipeline, DummyRegressionPipeline, - DummyTimeSeriesForecastingPipeline, fit_and_suppress_warnings ) from autoPyTorch.evaluation.time_series_forecasting_train_evaluator import TimeSeriesForecastingTrainEvaluator from autoPyTorch.evaluation.train_evaluator import TrainEvaluator +from autoPyTorch.evaluation.utils_extra import DummyTimeSeriesForecastingPipeline from autoPyTorch.pipeline.traditional_tabular_classification import TraditionalTabularClassificationPipeline From 2145b68bcbe62d01ca67487753b8a6ad55225ff3 Mon Sep 17 00:00:00 2001 From: Difan Deng <33290713+dengdifan@users.noreply.github.com> Date: Tue, 28 Jun 2022 20:08:57 +0200 Subject: [PATCH 3/7] Update docs/dev.rst Co-authored-by: Ravin Kohli <13005107+ravinkohli@users.noreply.github.com> --- docs/dev.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev.rst b/docs/dev.rst index 2c297056c..f1fec96c9 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -92,7 +92,7 @@ After the training (fitting a pipeline), we use pickle to save it to disk as stated `here `_. **Auto-PyTorch Time Series Forecasting** currently only allows Dummy pipelines and PyTorch neural networks. Traditional machine learning pipelines -will be introduced in the future iteration. +will be introduced in a future iteration. Optimization of pipeline ------------------------ From 2b8f6d1386dc97cba34c1d3d86d725a56bce3def Mon Sep 17 00:00:00 2001 From: dengdifan Date: Tue, 28 Jun 2022 20:18:17 +0200 Subject: [PATCH 4/7] make ForecastingDependenciesNotInstalledError a str message --- autoPyTorch/constants.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/autoPyTorch/constants.py b/autoPyTorch/constants.py index 4a99011d5..526c48ec5 100644 --- a/autoPyTorch/constants.py +++ b/autoPyTorch/constants.py @@ -1,3 +1,5 @@ +from typing import Optional + TABULAR_CLASSIFICATION = 1 IMAGE_CLASSIFICATION = 2 TABULAR_REGRESSION = 3 @@ -58,7 +60,7 @@ # Constants for Forecasting Tasks # Exceptions class ForecastingDependenciesNotInstalledError(ModuleNotFoundError): - def __init__(self, msg=None): + def __init__(self, msg: Optional[str] = None): if msg is None: msg = "Additional dependencies must be installed to work with time series forecasting" \ "tasks! Please run \n pip install autoPyTorch[forecasting] \n to install the" \ From 3db6626423047fe29512cff30cef37a90080207a Mon Sep 17 00:00:00 2001 From: dengdifan Date: Tue, 28 Jun 2022 20:18:45 +0200 Subject: [PATCH 5/7] make ForecastingDependenciesNotInstalledError a str message --- autoPyTorch/constants.py | 15 +++------------ autoPyTorch/evaluation/abstract_evaluator.py | 4 ++-- autoPyTorch/evaluation/tae.py | 4 ++-- .../pipeline/components/training/metrics/utils.py | 6 +++--- autoPyTorch/utils/pipeline.py | 6 +++--- 5 files changed, 13 insertions(+), 22 deletions(-) diff --git a/autoPyTorch/constants.py b/autoPyTorch/constants.py index 526c48ec5..bfd56d27f 100644 --- a/autoPyTorch/constants.py +++ b/autoPyTorch/constants.py @@ -1,5 +1,3 @@ -from typing import Optional - TABULAR_CLASSIFICATION = 1 IMAGE_CLASSIFICATION = 2 TABULAR_REGRESSION = 3 @@ -56,16 +54,9 @@ CLASSIFICATION_OUTPUTS = [BINARY, MULTICLASS, MULTICLASSMULTIOUTPUT] REGRESSION_OUTPUTS = [CONTINUOUS, CONTINUOUSMULTIOUTPUT] - -# Constants for Forecasting Tasks -# Exceptions -class ForecastingDependenciesNotInstalledError(ModuleNotFoundError): - def __init__(self, msg: Optional[str] = None): - if msg is None: - msg = "Additional dependencies must be installed to work with time series forecasting" \ - "tasks! Please run \n pip install autoPyTorch[forecasting] \n to install the" \ - "corresponding dependencies!" - super().__init__(msg) +ForecastingDependenciesNotInstalledMSG = "Additional dependencies must be installed to work with time series " \ + "forecasting tasks! Please run \n pip install autoPyTorch[forecasting] \n to "\ + "install the corresponding dependencies!" # The constant values for time series forecasting comes from diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index d8a511893..d20a96b75 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -31,7 +31,7 @@ CLASSIFICATION_TASKS, FORECASTING_BUDGET_TYPE, FORECASTING_TASKS, - ForecastingDependenciesNotInstalledError, + ForecastingDependenciesNotInstalledMSG, IMAGE_TASKS, MULTICLASS, REGRESSION_TASKS, @@ -498,7 +498,7 @@ def __init__(self, backend: Backend, elif self.task_type in FORECASTING_TASKS: if isinstance(self.configuration, int): if not forecasting_dependencies_installed: - raise ForecastingDependenciesNotInstalledError + raise ModuleNotFoundError(ForecastingDependenciesNotInstalledMSG) self.pipeline_class = DummyTimeSeriesForecastingPipeline elif isinstance(self.configuration, str): raise ValueError("Only tabular classifications tasks " diff --git a/autoPyTorch/evaluation/tae.py b/autoPyTorch/evaluation/tae.py index 1b701afab..b1650113c 100644 --- a/autoPyTorch/evaluation/tae.py +++ b/autoPyTorch/evaluation/tae.py @@ -25,7 +25,7 @@ from autoPyTorch.automl_common.common.utils.backend import Backend from autoPyTorch.constants import ( FORECASTING_BUDGET_TYPE, - ForecastingDependenciesNotInstalledError, + ForecastingDependenciesNotInstalledMSG, STRING_TO_TASK_TYPES, TIMESERIES_FORECASTING, ) @@ -158,7 +158,7 @@ def __init__( if STRING_TO_TASK_TYPES.get(dm.task_type, -1) == TIMESERIES_FORECASTING: if not forecasting_dependencies_installed: - raise ForecastingDependenciesNotInstalledError + raise ModuleNotFoundError(ForecastingDependenciesNotInstalledMSG) eval_function: Callable = forecasting_eval_train_function if isinstance(self.resampling_strategy, (HoldoutValTypes, CrossValTypes)): self.output_y_hat_optimization = output_y_hat_optimization diff --git a/autoPyTorch/pipeline/components/training/metrics/utils.py b/autoPyTorch/pipeline/components/training/metrics/utils.py index abb763e7e..e72c1afce 100644 --- a/autoPyTorch/pipeline/components/training/metrics/utils.py +++ b/autoPyTorch/pipeline/components/training/metrics/utils.py @@ -6,7 +6,7 @@ from autoPyTorch.constants import ( CLASSIFICATION_TASKS, FORECASTING_TASKS, - ForecastingDependenciesNotInstalledError, + ForecastingDependenciesNotInstalledMSG, REGRESSION_TASKS, STRING_TO_TASK_TYPES, TASK_TYPES, @@ -49,7 +49,7 @@ def get_supported_metrics(dataset_properties: Dict[str, Any]) -> Dict[str, autoP return CLASSIFICATION_METRICS elif STRING_TO_TASK_TYPES[task_type] in FORECASTING_TASKS: if len(FORECASTING_METRICS) == 0: - raise ForecastingDependenciesNotInstalledError + raise ModuleNotFoundError(ForecastingDependenciesNotInstalledMSG) return FORECASTING_METRICS else: raise NotImplementedError(task_type) @@ -129,7 +129,7 @@ def calculate_score( score_dict = dict() if task_type in FORECASTING_TASKS: if len(MASE_LOSSES) == 0: - raise ForecastingDependenciesNotInstalledError + raise ModuleNotFoundError(ForecastingDependenciesNotInstalledMSG) cprediction = sanitize_array(prediction) for metric_ in metrics: if metric_ in MASE_LOSSES and 'mase_coefficient' in score_kwargs: diff --git a/autoPyTorch/utils/pipeline.py b/autoPyTorch/utils/pipeline.py index d275baf8a..3a92e3cf3 100644 --- a/autoPyTorch/utils/pipeline.py +++ b/autoPyTorch/utils/pipeline.py @@ -6,7 +6,7 @@ from autoPyTorch.constants import ( CLASSIFICATION_TASKS, FORECASTING_TASKS, - ForecastingDependenciesNotInstalledError, + ForecastingDependenciesNotInstalledMSG, IMAGE_TASKS, REGRESSION_TASKS, STRING_TO_TASK_TYPES, @@ -79,7 +79,7 @@ def get_dataset_requirements(info: Dict[str, Any], ) else: if not forecasting_dependencies_installed: - raise ForecastingDependenciesNotInstalledError + raise ModuleNotFoundError(ForecastingDependenciesNotInstalledMSG) return _get_forecasting_dataset_requirements(info, include if include is not None else {}, exclude if exclude is not None else {}, @@ -137,7 +137,7 @@ def _get_forecasting_dataset_requirements(info: Dict[str, Any], if task_type in FORECASTING_TASKS: if not forecasting_dependencies_installed: - raise ForecastingDependenciesNotInstalledError + raise ModuleNotFoundError(ForecastingDependenciesNotInstalledMSG) return TimeSeriesForecastingPipeline( dataset_properties=info, include=include, From 12ba700f120f755a96a9b5b5a17e3c3879d04b51 Mon Sep 17 00:00:00 2001 From: dengdifan Date: Wed, 29 Jun 2022 13:40:49 +0200 Subject: [PATCH 6/7] update readme and examples --- README.md | 9 ++++++--- examples/20_basics/example_time_series_forecasting.py | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1fbbcdf82..377d5dcba 100755 --- a/README.md +++ b/README.md @@ -142,8 +142,11 @@ targets, features = load_longley() # define the forecasting horizon forecasting_horizon = 3 -# each series represent an element in the List -# we take the last forecasting_horizon as test targets. The itme before that as training targets +# Dataset optimized by APT-TS can be a list of np.ndarray/ pd.DataFrame where each series represents an element in the +# list, or a single pd.DataFrame that records the series +# index information: to which series the timestep belongs? This id can be stored as the DataFrame's index or a separate +# column +# Within each series, we take the last forecasting_horizon as test targets. The items before that as training targets # Normally the value to be forecasted should follow the training sets y_train = [targets[: -forecasting_horizon]] y_test = [targets[-forecasting_horizon:]] @@ -169,7 +172,7 @@ api.search( X_test=X_test, optimize_metric='mean_MAPE_forecasting', n_prediction_steps=forecasting_horizon, - memory_limit=16 * 1024, # Currently, forecasting models need much more memories than it actually requires + memory_limit=16 * 1024, # Currently, forecasting models use much more memories freq=freq, start_times=start_times, func_eval_time_limit_secs=50, diff --git a/examples/20_basics/example_time_series_forecasting.py b/examples/20_basics/example_time_series_forecasting.py index e4c3d9ae2..affaf5ffe 100644 --- a/examples/20_basics/example_time_series_forecasting.py +++ b/examples/20_basics/example_time_series_forecasting.py @@ -25,8 +25,11 @@ forecasting_horizon = 3 -# each series represent an element in the List -# we take the last forecasting_horizon as test targets. The itme before that as training targets +# Dataset optimized by APT-TS can be a list of np.ndarray/ pd.DataFrame where each series represents an element in the +# list, or a single pd.DataFrame that records the series +# index information: to which series the timestep belongs? This id can be stored as the DataFrame's index or a separate +# column +# Within each series, we take the last forecasting_horizon as test targets. The items before that as training targets # Normally the value to be forecasted should follow the training sets y_train = [targets[: -forecasting_horizon]] y_test = [targets[-forecasting_horizon:]] @@ -57,7 +60,7 @@ X_test=X_test, optimize_metric='mean_MASE_forecasting', n_prediction_steps=forecasting_horizon, - memory_limit=16 * 1024, # Currently, forecasting models need much more memories than it actually requires + memory_limit=16 * 1024, # Currently, forecasting models use much more memories freq=freq, start_times=start_times, func_eval_time_limit_secs=50, From be290e113183865d85934421fbf13caa999c890f Mon Sep 17 00:00:00 2001 From: dengdifan Date: Wed, 29 Jun 2022 18:09:36 +0200 Subject: [PATCH 7/7] add explanation for univariant models in example --- README.md | 2 +- examples/20_basics/example_time_series_forecasting.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 377d5dcba..d64eff179 100755 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ forecasting_horizon = 3 y_train = [targets[: -forecasting_horizon]] y_test = [targets[-forecasting_horizon:]] -# same for features. For uni-variant models, X_train, X_test can be omitted +# same for features. For uni-variant models, X_train, X_test can be omitted and set as None X_train = [features[: -forecasting_horizon]] # Here x_test indicates the 'known future features': they are the features known previously, features that are unknown # could be replaced with NAN or zeros (which will not be used by our networks). If no feature is known beforehand, diff --git a/examples/20_basics/example_time_series_forecasting.py b/examples/20_basics/example_time_series_forecasting.py index affaf5ffe..a7adba025 100644 --- a/examples/20_basics/example_time_series_forecasting.py +++ b/examples/20_basics/example_time_series_forecasting.py @@ -34,7 +34,7 @@ y_train = [targets[: -forecasting_horizon]] y_test = [targets[-forecasting_horizon:]] -# same for features. For uni-variant models, X_train, X_test can be omitted +# same for features. For uni-variant models, X_train, X_test can be omitted and set as None X_train = [features[: -forecasting_horizon]] # Here x_test indicates the 'known future features': they are the features known previously, features that are unknown # could be replaced with NAN or zeros (which will not be used by our networks). If no feature is known beforehand,