renamed train_metric to optimize_metric

improved handling of metrics and logs
improved return result of fit
added training_time as budget time
updated benchmark summary
plotting fixes

Author: urbanmatthias

autoPyTorch/__init__.py

@@ -3,5 +3,6 @@
 sys.path.append(hpbandster)
 
 from autoPyTorch.core.autonet_classes import AutoNetClassification, AutoNetMultilabel, AutoNetRegression
+from autoPyTorch.data_management.data_manager import DataManager
 from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdates
 from autoPyTorch.core.ensemble import AutoNetEnsemble

autoPyTorch/components/ensembles/ensemble_selection.py

@@ -7,12 +7,11 @@
 
 
 class EnsembleSelection(AbstractEnsemble):
-    def __init__(self, ensemble_size, metric, minimize,
+    def __init__(self, ensemble_size, metric,
                  sorted_initialization_n_best=0, only_consider_n_best=0,
                  bagging=False, mode='fast'):
         self.ensemble_size = ensemble_size
-        self.metric = metric
-        self.minimize = 1 if minimize else -1
+        self.metric = metric.get_loss_value
         self.sorted_initialization_n_best = sorted_initialization_n_best
         self.only_consider_n_best = only_consider_n_best
         self.bagging = bagging
@@ -56,7 +55,7 @@ def _fast(self, predictions, labels):
                 ensemble.append(predictions[idx])
                 order.append(idx)
                 ensemble_ = np.array(ensemble).mean(axis=0)
-                ensemble_performance = self.metric(ensemble_, labels) * self.minimize
+                ensemble_performance = self.metric(ensemble_, labels)
                 trajectory.append(ensemble_performance)
             ensemble_size -= self.sorted_initialization_n_best
 
@@ -82,7 +81,7 @@ def _fast(self, predictions, labels):
                     continue
                 fant_ensemble_prediction[:,:] = weighted_ensemble_prediction + \
                                              (1. / float(s + 1)) * pred
-                scores[j] = self.metric(fant_ensemble_prediction, labels) * self.minimize
+                scores[j] = self.metric(fant_ensemble_prediction, labels)
             all_best = np.argwhere(scores == np.nanmin(scores)).flatten()
             best = np.random.choice(all_best)
             ensemble.append(predictions[best])
@@ -113,7 +112,7 @@ def _slow(self, predictions, labels):
                 ensemble.append(predictions[idx])
                 order.append(idx)
                 ensemble_ = np.array(ensemble).mean(axis=0)
-                ensemble_performance = self.metric(ensemble_, labels) * self.minimize
+                ensemble_performance = self.metric(ensemble_, labels)
                 trajectory.append(ensemble_performance)
             ensemble_size -= self.sorted_initialization_n_best
 
@@ -129,7 +128,7 @@ def _slow(self, predictions, labels):
                     continue
                 ensemble.append(pred)
                 ensemble_prediction = np.mean(np.array(ensemble), axis=0)
-                scores[j] = self.metric(ensemble_prediction, labels) * self.minimize
+                scores[j] = self.metric(ensemble_prediction, labels)
                 ensemble.pop()
             best = np.nanargmin(scores)
             ensemble.append(predictions[best])
@@ -160,7 +159,7 @@ def _sorted_initialization(self, predictions, labels, n_best):
         perf = np.zeros([predictions.shape[0]])
 
         for idx, prediction in enumerate(predictions):
-            perf[idx] = self.metric(prediction, labels) * self.minimize
+            perf[idx] = self.metric(prediction, labels)
 
         indices = np.argsort(perf)[:n_best]
         return indices

autoPyTorch/components/metrics/balanced_accuracy.py

@@ -4,11 +4,8 @@
 from sklearn.metrics.classification import _check_targets, type_of_target
 
 
-def balanced_accuracy(y_pred, y_true):
-    return _balanced_accuracy(np.argmax(y_true, axis=1), np.argmax(y_pred, axis=1)) * 100
+def balanced_accuracy(solution, prediction):
 
-
-def _balanced_accuracy(solution, prediction):
     y_type, solution, prediction = _check_targets(solution, prediction)
 
     if y_type not in ["binary", "multiclass", 'multilabel-indicator']:
@@ -65,4 +62,4 @@ def _balanced_accuracy(solution, prediction):
     else:
         raise ValueError(y_type)
 
-    return np.mean(bac)  # average over all classes
+    return np.mean(bac)  # average over all classes

autoPyTorch/components/metrics/pac_score.py

@@ -4,11 +4,7 @@
 from sklearn.metrics.classification import _check_targets, type_of_target
 
 
-def pac_metric(y_pred, y_true):
-    return _pac_score(y_true, y_pred) * 100
-
-
-def _pac_score(solution, prediction):
+def pac_metric(solution, prediction):
     """
     Probabilistic Accuracy based on log_loss metric.
     We assume the solution is in {0, 1} and prediction in [0, 1].

autoPyTorch/components/metrics/standard_metrics.py

@@ -2,22 +2,20 @@
 import numpy as np
 
 # classification metrics
-def accuracy(y_pred, y_true):
-    return np.mean((undo_ohe(y_true) == undo_ohe(y_pred))) * 100
 
-def auc_metric(y_pred, y_true):
-    return (2 * metrics.roc_auc_score(y_true, y_pred) - 1) * 100
+
+def accuracy(y_true, y_pred):
+    return np.mean(y_true == y_pred)
+
+
+def auc_metric(y_true, y_pred):
+    return (2 * metrics.roc_auc_score(y_true, y_pred) - 1)
 
 
 # multilabel metric
-def multilabel_accuracy(y_pred, y_true):
-    return np.mean(y_true == (y_pred > 0.5)) * 100
+def multilabel_accuracy(y_true, y_pred):
+    return np.mean(y_true == (y_pred > 0.5))
 
 # regression metric
-def mean_distance(y_pred, y_true):
+def mean_distance(y_true, y_pred):
     return np.mean(np.abs(y_true - y_pred))
-
-def undo_ohe(y):
-    if len(y.shape) == 1:
-        return(y)
-    return np.argmax(y, axis=1)

autoPyTorch/components/training/budget_types.py

@@ -21,9 +21,9 @@ def on_batch_end(self, **kwargs):
 
     # OVERRIDE
     def on_epoch_end(self, trainer, **kwargs):
-        elapsed = time.time() - self.start_time
+        elapsed = time.time() - trainer.fit_start_time
         trainer.model.budget_trained = elapsed
-        trainer.logger.debug("Budget used: " + str(elapsed) + "/" + str(self.end_time - self.start_time))
+        trainer.logger.debug("Budget used: " + str(elapsed) + "/" + str(trainer.budget - self.compensate))
 
         if time.time() >= self.end_time:
             trainer.logger.debug("Budget exhausted!")
@@ -47,4 +47,33 @@ def on_epoch_end(self, trainer, epoch, **kwargs):
         if epoch >= self.target:
             trainer.logger.debug("Budget exhausted!")
             return True
-        return False
+        return False
+
+class BudgetTypeTrainingTime(BaseTrainingTechnique):
+    default_min_budget = 120
+    default_max_budget = 6000
+
+    # OVERRIDE
+    def set_up(self, trainer, pipeline_config, **kwargs):
+        super(BudgetTypeTrainingTime, self).set_up(trainer, pipeline_config)
+        self.end_time = trainer.budget + time.time()
+        self.start_time = time.time()
+
+        if self.start_time >= self.end_time:
+            raise Exception("Budget exhausted before training started")
+
+    # OVERRIDE
+    def on_batch_end(self, **kwargs):
+        return time.time() >= self.end_time
+
+    # OVERRIDE
+    def on_epoch_end(self, trainer, **kwargs):
+        elapsed = time.time() - self.start_time
+        trainer.model.budget_trained = elapsed
+        trainer.logger.debug("Budget used: " + str(elapsed) +
+                             "/" + str(self.end_time - self.start_time))
+
+        if time.time() >= self.end_time:
+            trainer.logger.debug("Budget exhausted!")
+            return True
+        return False

autoPyTorch/components/training/early_stopping.py

@@ -11,8 +11,8 @@ class EarlyStopping(BaseTrainingTechnique):
     def set_up(self, trainer, pipeline_config, **kwargs):
         super(EarlyStopping, self).set_up(trainer, pipeline_config)
         self.reset_parameters = pipeline_config["early_stopping_reset_parameters"]
-        self.minimize = pipeline_config["minimize"]
         self.patience = pipeline_config["early_stopping_patience"]
+        self.loss_transform = trainer.metrics[0].loss_transform
 
         # does not work with e.g. cosine anealing with warm restarts
         if hasattr(trainer, "lr_scheduler") and not trainer.lr_scheduler.allows_early_stopping:
@@ -21,8 +21,6 @@ def set_up(self, trainer, pipeline_config, **kwargs):
         # initialize current best performance to +/- infinity
         if trainer.model.current_best_epoch_performance is None:
             trainer.model.current_best_epoch_performance = float("inf")
-            if not self.minimize:
-                trainer.model.current_best_epoch_performance = -float("inf")
 
         trainer.logger.debug("Using Early stopping with patience: " + str(self.patience))
         trainer.logger.debug("Reset Parameters to parameters with best validation performance: " + str(self.reset_parameters))
@@ -35,11 +33,10 @@ def on_epoch_end(self, trainer, log, **kwargs):
             return False
         if self.reset_parameters and (not hasattr(trainer, "lr_scheduler") or not trainer.lr_scheduler.snapshot_before_restart):
             log["best_parameters"] = False
-        current_performance = log["val_" + trainer.metrics[0]]
+        current_performance = self.loss_transform(log["val_" + trainer.metrics[0]])
 
         # new best performance
-        if ((self.minimize and current_performance < trainer.model.current_best_epoch_performance) or
-            (not self.minimize and current_performance > trainer.model.current_best_epoch_performance)):
+        if current_performance < trainer.model.current_best_epoch_performance:
             trainer.model.num_epochs_no_progress = 0
             trainer.model.current_best_epoch_performance = current_performance
             trainer.logger.debug("New best performance!")

autoPyTorch/components/training/trainer.py

@@ -66,10 +66,10 @@ def on_epoch_start(self, log, epoch):
     def on_epoch_end(self, log, epoch):
         return any([t.on_epoch_end(trainer=self, log=log, epoch=epoch) for t in self.training_techniques])
 
-    def final_eval(self, opt_metric_name, logs, train_loader, valid_loader, minimize, best_over_epochs, refit):
+    def final_eval(self, opt_metric_name, logs, train_loader, valid_loader, best_over_epochs, refit):
         # select log
         if best_over_epochs:
-            final_log = (min if minimize else max)(logs, key=lambda log: log[opt_metric_name])
+            final_log = min(logs, key=lambda log: self.metrics[0].loss_transform(log[opt_metric_name]))
         else:
             final_log = None
             for t in self.training_techniques:
@@ -87,10 +87,10 @@ def final_eval(self, opt_metric_name, logs, train_loader, valid_loader, minimize
 
             for i, metric in enumerate(self.metrics):
                 if valid_metric_results:
-                    final_log['val_' + metric.__name__] = valid_metric_results[i]
+                    final_log['val_' + metric.name] = valid_metric_results[i]
             if self.eval_additional_logs_on_snapshot and not refit:
                     for additional_log in self.log_functions:
-                        final_log[additional_log.__name__] = additional_log(self.model, None)
+                        final_log[additional_log.name] = additional_log(self.model, None)
         return final_log
 
     def train(self, epoch, train_loader):

autoPyTorch/core/api.py

@@ -4,6 +4,7 @@
 
 
 import numpy as np
+import scipy.sparse
 import torch
 import torch.nn as nn
 import copy
@@ -21,6 +22,7 @@
 from autoPyTorch.utils.config.config_file_parser import ConfigFileParser
 
 class AutoNet():
+    """Find an optimal neural network given a ML-task using BOHB"""
     preset_folder_name = None
 
     def __init__(self, config_preset="medium_cs", pipeline=None, **autonet_config):
@@ -34,6 +36,7 @@ def __init__(self, config_preset="medium_cs", pipeline=None, **autonet_config):
         self.base_config = autonet_config
         self.autonet_config = None
         self.fit_result = None
+        self.dataset_info = None
 
         if config_preset is not None:
             parser = self.get_autonet_config_file_parser()
@@ -70,10 +73,11 @@ def get_current_autonet_config(self):
         return self.pipeline.get_pipeline_config(**self.base_config)
 
     def get_hyperparameter_search_space(self, X_train=None, Y_train=None, X_valid=None, Y_valid=None, **autonet_config):
-        """Return hyperparameter search space of Auto-PyTorch. Does depend on the dataset and the configuration.!
+        """Return hyperparameter search space of Auto-PyTorch. Does depend on the dataset and the configuration!
+        You can either pass the dataset and the configuration or use dataset and configuration of last fit call.
         
         Keyword Arguments:
-            X_train {array} -- Training data.
+            X_train {array} -- Training data. ConfigSpace depends on Training data.
             Y_train {array} -- Targets of training data.
             X_valid {array} -- Validation data. Will be ignored if cv_splits > 1. (default: {None})
             Y_valid {array} -- Validation data. Will be ignored if cv_splits > 1. (default: {None})
@@ -82,8 +86,8 @@ def get_hyperparameter_search_space(self, X_train=None, Y_train=None, X_valid=No
         Returns:
             ConfigurationSpace -- The configuration space that should be optimized.
         """
-
-        dataset_info = None
+        X_train, Y_train, X_valid, Y_valid = self.check_data_array_types(X_train, Y_train, X_valid, Y_valid)
+        dataset_info = self.dataset_info
         pipeline_config = dict(self.base_config, **autonet_config) if autonet_config else \
             self.get_current_autonet_config()
         if X_train is not None and Y_train is not None:
@@ -129,21 +133,22 @@ def fit(self, X_train, Y_train, X_valid=None, Y_valid=None, refit=True, **autone
         
         Returns:
             optimized_hyperparameter_config -- The best found hyperparameter config.
-            final_metric_score --  The final score of the specified train metric.
             **autonet_config -- Configure AutoNet for your needs. You can also configure AutoNet in the constructor(). Call print_help() for more info.
         """
+        X_train, Y_train, X_valid, Y_valid = self.check_data_array_types(X_train, Y_train, X_valid, Y_valid)
         self.autonet_config = self.pipeline.get_pipeline_config(**dict(self.base_config, **autonet_config))
 
         self.fit_result = self.pipeline.fit_pipeline(pipeline_config=self.autonet_config,
                                                      X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid)
+        self.dataset_info = self.pipeline[CreateDatasetInfo.get_name()].fit_output["dataset_info"]
         self.pipeline.clean()
 
         if not self.fit_result["optimized_hyperparameter_config"]:
             raise RuntimeError("No models fit during training, please retry with a larger max_runtime.")
 
         if (refit):
             self.refit(X_train, Y_train, X_valid, Y_valid)
-        return self.fit_result["optimized_hyperparameter_config"], self.fit_result['final_metric_score']
+        return self.fit_result
 
     def refit(self, X_train, Y_train, X_valid=None, Y_valid=None, hyperparameter_config=None, autonet_config=None, budget=None, rescore=False):
         """Refit AutoNet to given hyperparameters. This will skip hyperparameter search.
@@ -163,6 +168,7 @@ def refit(self, X_train, Y_train, X_valid=None, Y_valid=None, hyperparameter_con
         Raises:
             ValueError -- No hyperparameter config available
         """
+        X_train, Y_train, X_valid, Y_valid = self.check_data_array_types(X_train, Y_train, X_valid, Y_valid)
         if (autonet_config is None):
             autonet_config = self.autonet_config
         if (autonet_config is None):
@@ -182,9 +188,8 @@ def refit(self, X_train, Y_train, X_valid=None, Y_valid=None, hyperparameter_con
                       'budget': budget,
                       'rescore': rescore}
 
-        result = self.pipeline.fit_pipeline(pipeline_config=autonet_config, refit=refit_data,
-                                    X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid)
-        return result["final_metric_score"]
+        return self.pipeline.fit_pipeline(pipeline_config=autonet_config, refit=refit_data,
+                                          X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid)
 
     def predict(self, X, return_probabilities=False):
         """Predict the targets for a data matrix X.
@@ -200,6 +205,7 @@ def predict(self, X, return_probabilities=False):
         """
 
         # run predict pipeline
+        X, = self.check_data_array_types(X)
         autonet_config = self.autonet_config or self.base_config
         Y_pred = self.pipeline.predict_pipeline(pipeline_config=autonet_config, X=X)['Y']
 
@@ -208,8 +214,8 @@ def predict(self, X, return_probabilities=False):
         result = OHE.reverse_transform_y(Y_pred, OHE.fit_output['y_one_hot_encoder'])
         return result if not return_probabilities else (result, Y_pred)
 
-    def score(self, X_test, Y_test):
-        """Calculate the sore on test data using the specified train_metric
+    def score(self, X_test, Y_test, return_loss_value=False):
+        """Calculate the sore on test data using the specified optimize_metric
         
         Arguments:
             X_test {array} -- The test data matrix.
@@ -220,6 +226,7 @@ def score(self, X_test, Y_test):
         """
 
         # run predict pipeline
+        X_test, Y_test = self.check_data_array_types(X_test, Y_test)
         autonet_config = self.autonet_config or self.base_config
         self.pipeline.predict_pipeline(pipeline_config=autonet_config, X=X_test)
         Y_pred = self.pipeline[OptimizationAlgorithm.get_name()].predict_output['Y']
@@ -228,5 +235,19 @@ def score(self, X_test, Y_test):
         OHE = self.pipeline[OneHotEncoding.get_name()]
         Y_test = OHE.transform_y(Y_test, OHE.fit_output['y_one_hot_encoder'])
 
-        metric = self.pipeline[MetricSelector.get_name()].fit_output['train_metric']
+        metric = self.pipeline[MetricSelector.get_name()].fit_output['optimize_metric']
+        if return_loss_value:
+            return metric.get_loss_value(Y_pred, Y_test)
         return metric(Y_pred, Y_test)
+    
+    def check_data_array_types(self, *arrays):
+        result = []
+        for array in arrays:
+            if array is None or scipy.sparse.issparse(array):
+                result.append(array)
+                continue
+            
+            result.append(np.asanyarray(array))
+            if not result[-1].shape:
+                raise RuntimeError("Given data-array is of unexpected type %s. Please pass numpy arrays instead." % type(array))
+        return result