Added cross-entropy as optimization metric

Author: Lucas Zimmer

autoPyTorch/components/metrics/__init__.py

@@ -1,4 +1,3 @@
 from autoPyTorch.components.metrics.balanced_accuracy import balanced_accuracy
 from autoPyTorch.components.metrics.pac_score import pac_metric
-from autoPyTorch.components.metrics.standard_metrics import accuracy, auc_metric, mean_distance, multilabel_accuracy
-from autoPyTorch.components.metrics.standard_metrics import top1, top3, top5
+from autoPyTorch.components.metrics.standard_metrics import accuracy, auc_metric, mean_distance, multilabel_accuracy, cross_entropy, top1, top3, top5

autoPyTorch/components/metrics/standard_metrics.py

@@ -8,6 +8,12 @@ def accuracy(y_true, y_pred):
 def auc_metric(y_true, y_pred):
     return (2 * metrics.roc_auc_score(y_true, y_pred) - 1)
 
+def cross_entropy(y_true, y_pred):
+    if y_true==1:
+        return -np.log(y_pred)
+    else:
+        return -np.log(1-y_pred)
+
 def top1(y_pred, y_true):
     return topN(y_pred, y_true, 1)
 

autoPyTorch/core/autonet_classes/autonet_feature_classification.py

@@ -19,7 +19,7 @@ def _apply_default_pipeline_settings(pipeline):
 
         import torch.nn as nn
         from sklearn.model_selection import StratifiedKFold
-        from autoPyTorch.components.metrics import accuracy, auc_metric, pac_metric, balanced_accuracy
+        from autoPyTorch.components.metrics import accuracy, auc_metric, pac_metric, balanced_accuracy, cross_entropy
         from autoPyTorch.components.preprocessing.loss_weight_strategies import LossWeightStrategyWeighted
 
         AutoNetFeatureData._apply_default_pipeline_settings(pipeline)
@@ -41,6 +41,8 @@ def _apply_default_pipeline_settings(pipeline):
                                    requires_target_class_labels=False)
         metric_selector.add_metric('balanced_accuracy', balanced_accuracy, loss_transform=True,
                                    requires_target_class_labels=True)
+        metric_selector.add_metric('cross_entropy', cross_entropy, loss_transform=True,
+                                   requires_target_class_labels=False)
 
         resample_selector = pipeline[ResamplingStrategySelector.get_name()]
         resample_selector.add_over_sampling_method('random', RandomOverSamplingWithReplacement)

autoPyTorch/core/autonet_classes/autonet_image_classification.py

@@ -12,7 +12,7 @@ def _apply_default_pipeline_settings(pipeline):
         from autoPyTorch.pipeline.nodes.image.cross_validation_indices import CrossValidationIndices
         from autoPyTorch.pipeline.nodes.image.loss_module_selector_indices import LossModuleSelectorIndices
         from autoPyTorch.pipeline.nodes.image.network_selector_datasetinfo import NetworkSelectorDatasetInfo
-        from autoPyTorch.components.metrics.standard_metrics import accuracy
+        from autoPyTorch.components.metrics import accuracy, auc_metric, pac_metric, balanced_accuracy, cross_entropy
         from autoPyTorch.components.preprocessing.loss_weight_strategies import LossWeightStrategyWeighted
 
         AutoNetImageData._apply_default_pipeline_settings(pipeline)
@@ -25,7 +25,16 @@ def _apply_default_pipeline_settings(pipeline):
         loss_selector.add_loss_module('cross_entropy_weighted', nn.CrossEntropyLoss, LossWeightStrategyWeighted(), True)
 
         metric_selector = pipeline[MetricSelector.get_name()]
-        metric_selector.add_metric('accuracy', accuracy)
+        metric_selector.add_metric('accuracy', accuracy, loss_transform=True,
+                                   requires_target_class_labels=False)
+        metric_selector.add_metric('auc_metric', auc_metric, loss_transform=True,
+                                   requires_target_class_labels=False)
+        metric_selector.add_metric('pac_metric', pac_metric, loss_transform=True,
+                                   requires_target_class_labels=False)
+        metric_selector.add_metric('balanced_accuracy', balanced_accuracy, loss_transform=True,
+                                   requires_target_class_labels=True)
+        metric_selector.add_metric('cross_entropy', cross_entropy, loss_transform=True,
+                                   requires_target_class_labels=False)
 
         train_node = pipeline[SimpleTrainNode.get_name()]
         train_node.default_minimize_value = False

autoPyTorch/core/hpbandster_extensions/bohb_multi_kde_ext.py

@@ -1,4 +1,4 @@
 lr_scheduler=[cosine_annealing, step]
 networks=[resnet, mobilenet]
-batch_loss_computation_techniques=[mixup]
+batch_loss_computation_techniques=[standard, mixup]
 optimizer=[adamw, sgd]

autoPyTorch/core/presets/image_classification/medium_cs.txt

@@ -1,4 +1,4 @@
 lr_scheduler=[cosine_annealing, step]
 networks=[resnet, mobilenet]
-batch_loss_computation_techniques=[mixup]
+batch_loss_computation_techniques=[standard, mixup]
 optimizer=[adamw, sgd]

autoPyTorch/core/presets/image_classification_multiple_datasets/medium_cs.txt

@@ -22,7 +22,6 @@
 from autoPyTorch.utils.config.config_option import ConfigOption, to_bool
 
 from autoPyTorch.core.hpbandster_extensions.bohb_ext import BOHBExt
-from autoPyTorch.core.hpbandster_extensions.bohb_multi_kde_ext import BOHBMultiKDEExt
 from autoPyTorch.core.hpbandster_extensions.hyperband_ext import HyperBandExt
 from autoPyTorch.core.worker_no_timelimit import ModuleWorkerNoTimeLimit
 
@@ -68,7 +67,6 @@ def __init__(self, optimization_pipeline_nodes):
         self.algorithms = dict()
         self.algorithms["bohb"] = BOHBExt
         self.algorithms["hyperband"] = HyperBandExt
-        self.algorithms["bohb_multi_kde"] = BOHBMultiKDEExt
 
         self.logger = logging.getLogger('autonet')