Network Cleanup (#81)

* removed old supported_tasks dictionary from heads, added some docstrings and some small fixes

* removed old supported_tasks attribute and updated doc strings in base backbone and base head components

* removed old supported_tasks attribute from network backbones

* put time series backbones in separate files, add doc strings and refactored search space arguments

* split image networks into separate files, add doc strings and refactor search space

* fix typo

* add an intial simple backbone test similar to the network head test

* fix flake8

* fixed imports in backbones and heads

* added new network backbone and head tests

* enabled tests for adding custom backbones and heads, added required properties to base head and base backbone

Author: bastiscode

autoPyTorch/pipeline/components/setup/network_backbone/ConvNetImageBackbone.py

@@ -0,0 +1,122 @@
+from typing import Any, Dict, List, Optional, Tuple
+
+import ConfigSpace as CS
+from ConfigSpace.configuration_space import ConfigurationSpace
+from ConfigSpace.hyperparameters import (
+    CategoricalHyperparameter,
+    UniformIntegerHyperparameter
+)
+
+from torch import nn
+
+from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent
+from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations
+
+
+class ConvNetImageBackbone(NetworkBackboneComponent):
+    """
+    Standard Convolutional Neural Network backbone for images
+    """
+
+    def __init__(self, **kwargs: Any):
+        super().__init__(**kwargs)
+        self.bn_args = {"eps": 1e-5, "momentum": 0.1}
+
+    def _get_layer_size(self, w: int, h: int) -> Tuple[int, int]:
+        cw = ((w - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
+              // self.config["conv_kernel_stride"]) + 1
+        ch = ((h - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
+              // self.config["conv_kernel_stride"]) + 1
+        cw, ch = cw // self.config["pool_size"], ch // self.config["pool_size"]
+        return cw, ch
+
+    def _add_layer(self, layers: List[nn.Module], in_filters: int, out_filters: int) -> None:
+        layers.append(nn.Conv2d(in_filters, out_filters,
+                                kernel_size=self.config["conv_kernel_size"],
+                                stride=self.config["conv_kernel_stride"],
+                                padding=self.config["conv_kernel_padding"]))
+        layers.append(nn.BatchNorm2d(out_filters, **self.bn_args))
+        layers.append(_activations[self.config["activation"]]())
+        layers.append(nn.MaxPool2d(kernel_size=self.config["pool_size"], stride=self.config["pool_size"]))
+
+    def build_backbone(self, input_shape: Tuple[int, ...]) -> nn.Module:
+        channels, iw, ih = input_shape
+        layers: List[nn.Module] = []
+        init_filter = self.config["conv_init_filters"]
+        self._add_layer(layers, channels, init_filter)
+
+        cw, ch = self._get_layer_size(iw, ih)
+        for i in range(2, self.config["num_layers"] + 1):
+            cw, ch = self._get_layer_size(cw, ch)
+            if cw == 0 or ch == 0:
+                break
+            self._add_layer(layers, init_filter, init_filter * 2)
+            init_filter *= 2
+        backbone = nn.Sequential(*layers)
+        self.backbone = backbone
+        return backbone
+
+    @staticmethod
+    def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
+        return {
+            'shortname': 'ConvNetImageBackbone',
+            'name': 'ConvNetImageBackbone',
+            'handles_tabular': False,
+            'handles_image': True,
+            'handles_time_series': False,
+        }
+
+    @staticmethod
+    def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
+                                        num_layers: Tuple[Tuple, int] = ((2, 8), 4),
+                                        num_init_filters: Tuple[Tuple, int] = ((16, 64), 32),
+                                        activation: Tuple[Tuple, str] = (tuple(_activations.keys()),
+                                                                         list(_activations.keys())[0]),
+                                        kernel_size: Tuple[Tuple, int] = ((3, 5), 3),
+                                        stride: Tuple[Tuple, int] = ((1, 3), 1),
+                                        padding: Tuple[Tuple, int] = ((2, 3), 2),
+                                        pool_size: Tuple[Tuple, int] = ((2, 3), 2)
+                                        ) -> ConfigurationSpace:
+        cs = CS.ConfigurationSpace()
+
+        min_num_layers, max_num_layers = num_layers[0]
+        cs.add_hyperparameter(UniformIntegerHyperparameter('num_layers',
+                                                           lower=min_num_layers,
+                                                           upper=max_num_layers,
+                                                           default_value=num_layers[1]))
+
+        cs.add_hyperparameter(CategoricalHyperparameter('activation',
+                                                        choices=activation[0],
+                                                        default_value=activation[1]))
+
+        min_init_filters, max_init_filters = num_init_filters[0]
+        cs.add_hyperparameter(UniformIntegerHyperparameter('conv_init_filters',
+                                                           lower=min_init_filters,
+                                                           upper=max_init_filters,
+                                                           default_value=num_init_filters[1]))
+
+        min_kernel_size, max_kernel_size = kernel_size[0]
+        cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_size',
+                                                           lower=min_kernel_size,
+                                                           upper=max_kernel_size,
+                                                           default_value=kernel_size[1]))
+
+        min_stride, max_stride = stride[0]
+        cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_stride',
+                                                           lower=min_stride,
+                                                           upper=max_stride,
+                                                           default_value=stride[1]))
+
+        min_padding, max_padding = padding[0]
+        cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_padding',
+                                                           lower=min_padding,
+                                                           upper=max_padding,
+                                                           default_value=padding[1]))
+
+        min_pool_size, max_pool_size = pool_size[0]
+        cs.add_hyperparameter(UniformIntegerHyperparameter('pool_size',
+                                                           lower=min_pool_size,
+                                                           upper=max_pool_size,
+                                                           default_value=pool_size[1]))
+
+        return cs

autoPyTorch/pipeline/components/setup/network_backbone/image.py renamed to autoPyTorch/pipeline/components/setup/network_backbone/DenseNetImageBackone.py

@@ -1,7 +1,6 @@
-import logging
 import math
 from collections import OrderedDict
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, Optional, Tuple
 
 import ConfigSpace as CS
 from ConfigSpace.configuration_space import ConfigurationSpace
@@ -16,103 +15,7 @@
 from torch.nn import functional as F
 
 from autoPyTorch.pipeline.components.setup.network_backbone.base_network_backbone import NetworkBackboneComponent
-
-_activations: Dict[str, nn.Module] = {
-    "relu": nn.ReLU,
-    "tanh": nn.Tanh,
-    "sigmoid": nn.Sigmoid
-}
-
-
-class ConvNetImageBackbone(NetworkBackboneComponent):
-    supported_tasks = {"image_classification", "image_regression"}
-
-    def __init__(self, **kwargs: Any):
-        super().__init__(**kwargs)
-        self.bn_args = {"eps": 1e-5, "momentum": 0.1}
-
-    def _get_layer_size(self, w: int, h: int) -> Tuple[int, int]:
-        cw = ((w - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
-              // self.config["conv_kernel_stride"]) + 1
-        ch = ((h - self.config["conv_kernel_size"] + 2 * self.config["conv_kernel_padding"])
-              // self.config["conv_kernel_stride"]) + 1
-        cw, ch = cw // self.config["pool_size"], ch // self.config["pool_size"]
-        return cw, ch
-
-    def _add_layer(self, layers: List[nn.Module], in_filters: int, out_filters: int) -> None:
-        layers.append(nn.Conv2d(in_filters, out_filters,
-                                kernel_size=self.config["conv_kernel_size"],
-                                stride=self.config["conv_kernel_stride"],
-                                padding=self.config["conv_kernel_padding"]))
-        layers.append(nn.BatchNorm2d(out_filters, **self.bn_args))
-        layers.append(_activations[self.config["activation"]]())
-        layers.append(nn.MaxPool2d(kernel_size=self.config["pool_size"], stride=self.config["pool_size"]))
-
-    def build_backbone(self, input_shape: Tuple[int, ...]) -> nn.Module:
-        channels, iw, ih = input_shape
-        layers: List[nn.Module] = []
-        init_filter = self.config["conv_init_filters"]
-        self._add_layer(layers, channels, init_filter)
-
-        cw, ch = self._get_layer_size(iw, ih)
-        for i in range(2, self.config["num_layers"] + 1):
-            cw, ch = self._get_layer_size(cw, ch)
-            if cw == 0 or ch == 0:
-                logging.info("> reduce network size due to too small layers.")
-                break
-            self._add_layer(layers, init_filter, init_filter * 2)
-            init_filter *= 2
-        backbone = nn.Sequential(*layers)
-        self.backbone = backbone
-        return backbone
-
-    @staticmethod
-    def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
-        return {
-            'shortname': 'ConvNetImageBackbone',
-            'name': 'ConvNetImageBackbone',
-            'handles_tabular': False,
-            'handles_image': True,
-            'handles_time_series': False,
-        }
-
-    @staticmethod
-    def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None,
-                                        min_num_layers: int = 2,
-                                        max_num_layers: int = 5,
-                                        min_init_filters: int = 16,
-                                        max_init_filters: int = 64,
-                                        min_kernel_size: int = 2,
-                                        max_kernel_size: int = 5,
-                                        min_stride: int = 1,
-                                        max_stride: int = 3,
-                                        min_padding: int = 2,
-                                        max_padding: int = 3,
-                                        min_pool_size: int = 2,
-                                        max_pool_size: int = 3) -> ConfigurationSpace:
-        cs = CS.ConfigurationSpace()
-
-        cs.add_hyperparameter(UniformIntegerHyperparameter('num_layers',
-                                                           lower=min_num_layers,
-                                                           upper=max_num_layers))
-        cs.add_hyperparameter(CategoricalHyperparameter('activation',
-                                                        choices=list(_activations.keys())))
-        cs.add_hyperparameter(UniformIntegerHyperparameter('conv_init_filters',
-                                                           lower=min_init_filters,
-                                                           upper=max_init_filters))
-        cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_size',
-                                                           lower=min_kernel_size,
-                                                           upper=max_kernel_size))
-        cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_stride',
-                                                           lower=min_stride,
-                                                           upper=max_stride))
-        cs.add_hyperparameter(UniformIntegerHyperparameter('conv_kernel_padding',
-                                                           lower=min_padding,
-                                                           upper=max_padding))
-        cs.add_hyperparameter(UniformIntegerHyperparameter('pool_size',
-                                                           lower=min_pool_size,
-                                                           upper=max_pool_size))
-        return cs
+from autoPyTorch.pipeline.components.setup.network_backbone.utils import _activations
 
 
 class _DenseLayer(nn.Sequential):
@@ -177,7 +80,9 @@ def __init__(self,
 
 
 class DenseNetBackbone(NetworkBackboneComponent):
-    supported_tasks = {"image_classification", "image_regression"}
+    """
+    Dense Net Backbone for images (see https://arxiv.org/pdf/1608.06993.pdf)
+    """
 
     def __init__(self, **kwargs: Any):
         super().__init__(**kwargs)
@@ -247,39 +152,55 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None) -> Dict[
         }
 
     @staticmethod
-    def get_hyperparameter_search_space(dataset_properties: Optional[Dict[str, str]] = None,
-                                        min_growth_rate: int = 12,
-                                        max_growth_rate: int = 40,
-                                        min_num_blocks: int = 3,
-                                        max_num_blocks: int = 4,
-                                        min_num_layers: int = 4,
-                                        max_num_layers: int = 64) -> ConfigurationSpace:
+    def get_hyperparameter_search_space(dataset_properties: Optional[Dict] = None,
+                                        num_blocks: Tuple[Tuple, int] = ((3, 4), 3),
+                                        num_layers: Tuple[Tuple, int] = ((4, 64), 16),
+                                        growth_rate: Tuple[Tuple, int] = ((12, 40), 20),
+                                        activation: Tuple[Tuple, str] = (tuple(_activations.keys()),
+                                                                         list(_activations.keys())[0]),
+                                        use_dropout: Tuple[Tuple, bool] = ((True, False), False),
+                                        dropout: Tuple[Tuple, float] = ((0, 0.5), 0.2)
+                                        ) -> ConfigurationSpace:
         cs = CS.ConfigurationSpace()
+
+        min_growth_rate, max_growth_rate = growth_rate[0]
         growth_rate_hp = UniformIntegerHyperparameter('growth_rate',
                                                       lower=min_growth_rate,
-                                                      upper=max_growth_rate)
+                                                      upper=max_growth_rate,
+                                                      default_value=growth_rate[1])
         cs.add_hyperparameter(growth_rate_hp)
 
+        min_num_blocks, max_num_blocks = num_blocks[0]
         blocks_hp = UniformIntegerHyperparameter('blocks',
                                                  lower=min_num_blocks,
-                                                 upper=max_num_blocks)
+                                                 upper=max_num_blocks,
+                                                 default_value=num_blocks[1])
         cs.add_hyperparameter(blocks_hp)
 
         activation_hp = CategoricalHyperparameter('activation',
-                                                  choices=list(_activations.keys()))
+                                                  choices=activation[0],
+                                                  default_value=activation[1])
         cs.add_hyperparameter(activation_hp)
 
-        use_dropout = CategoricalHyperparameter('use_dropout', choices=[True, False])
+        use_dropout = CategoricalHyperparameter('use_dropout',
+                                                choices=use_dropout[0],
+                                                default_value=use_dropout[1])
+
+        min_dropout, max_dropout = dropout[0]
         dropout = UniformFloatHyperparameter('dropout',
-                                             lower=0.0,
-                                             upper=1.0)
+                                             lower=min_dropout,
+                                             upper=max_dropout,
+                                             default_value=dropout[1])
+
         cs.add_hyperparameters([use_dropout, dropout])
         cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True))
 
         for i in range(1, max_num_blocks + 1):
+            min_num_layers, max_num_layers = num_layers[0]
             layer_hp = UniformIntegerHyperparameter('layer_in_block_%d' % i,
                                                     lower=min_num_layers,
-                                                    upper=max_num_layers)
+                                                    upper=max_num_layers,
+                                                    default_value=num_layers[1])
             cs.add_hyperparameter(layer_hp)
 
             if i > min_num_blocks: