Merge branch 'refactor_development' of github.com:automl/Auto-PyTorch into refactoring-base-dataset_splitting-functions

[fix] Update before merging

Author: nabenabe0928

autoPyTorch/evaluation/abstract_evaluator.py

@@ -54,6 +54,23 @@
 
 
 class MyTraditionalTabularClassificationPipeline(BaseEstimator):
+    """
+    A wrapper class that holds a pipeline for traditional classification.
+    Estimators like CatBoost, and Random Forest are considered traditional machine
+    learning models and are fitted before neural architecture search.
+
+    This class is an interface to fit a pipeline containing a traditional machine
+    learning model, and is the final object that is stored for inference.
+
+    Attributes:
+        dataset_properties (Dict[str, Any]):
+            A dictionary containing dataset specific information
+        random_state (Optional[Union[int, np.random.RandomState]]):
+            Object that contains a seed and allows for reproducible results
+        init_params  (Optional[Dict]):
+            An optional dictionary that is passed to the pipeline's steps. It complies
+            a similar function as the kwargs
+    """
     def __init__(self, config: str,
                  dataset_properties: Dict[str, Any],
                  random_state: Optional[Union[int, np.random.RandomState]] = None,
@@ -98,6 +115,21 @@ def get_default_pipeline_options() -> Dict[str, Any]:
 
 
 class DummyClassificationPipeline(DummyClassifier):
+    """
+    A wrapper class that holds a pipeline for dummy classification.
+
+    A wrapper over DummyClassifier of scikit learn. This estimator is considered the
+    worst performing model. In case of failure, at least this model will be fitted.
+
+    Attributes:
+        dataset_properties (Dict[str, Any]):
+            A dictionary containing dataset specific information
+        random_state (Optional[Union[int, np.random.RandomState]]):
+            Object that contains a seed and allows for reproducible results
+        init_params  (Optional[Dict]):
+            An optional dictionary that is passed to the pipeline's steps. It complies
+            a similar function as the kwargs
+    """
     def __init__(self, config: Configuration,
                  random_state: Optional[Union[int, np.random.RandomState]] = None,
                  init_params: Optional[Dict] = None
@@ -148,6 +180,21 @@ def get_default_pipeline_options() -> Dict[str, Any]:
 
 
 class DummyRegressionPipeline(DummyRegressor):
+    """
+    A wrapper class that holds a pipeline for dummy regression.
+
+    A wrapper over DummyRegressor of scikit learn. This estimator is considered the
+    worst performing model. In case of failure, at least this model will be fitted.
+
+    Attributes:
+        dataset_properties (Dict[str, Any]):
+            A dictionary containing dataset specific information
+        random_state (Optional[Union[int, np.random.RandomState]]):
+            Object that contains a seed and allows for reproducible results
+        init_params  (Optional[Dict]):
+            An optional dictionary that is passed to the pipeline's steps. It complies
+            a similar function as the kwargs
+    """
     def __init__(self, config: Configuration,
                  random_state: Optional[Union[int, np.random.RandomState]] = None,
                  init_params: Optional[Dict] = None) -> None:
@@ -351,7 +398,7 @@ def _get_pipeline(self) -> BaseEstimator:
         if isinstance(self.configuration, int):
             pipeline = self.pipeline_class(config=self.configuration,
                                            random_state=np.random.RandomState(self.seed),
-                                           init_params=self.fit_dictionary)
+                                           init_params=self._init_params)
         elif isinstance(self.configuration, Configuration):
             pipeline = self.pipeline_class(config=self.configuration,
                                            dataset_properties=self.dataset_properties,
@@ -364,7 +411,7 @@ def _get_pipeline(self) -> BaseEstimator:
             pipeline = self.pipeline_class(config=self.configuration,
                                            dataset_properties=self.dataset_properties,
                                            random_state=np.random.RandomState(self.seed),
-                                           init_params=self.fit_dictionary)
+                                           init_params=self._init_params)
         else:
             raise ValueError("Invalid configuration entered")
         return pipeline

autoPyTorch/pipeline/base_pipeline.py

@@ -398,24 +398,47 @@ def _check_search_space_updates(self, include: Optional[Dict[str, Any]],
                 raise ValueError("Unknown node name. Expected update node name to be in {} "
                                  "got {}".format(self.named_steps.keys(), update.node_name))
             node = self.named_steps[update.node_name]
+            # if node is a choice module
             if hasattr(node, 'get_components'):
                 split_hyperparameter = update.hyperparameter.split(':')
 
+                # check if component is not present in include
                 if include is not None and update.node_name in include.keys():
                     if split_hyperparameter[0] not in include[update.node_name]:
                         raise ValueError("Not found {} in include".format(split_hyperparameter[0]))
 
+                # check if component is present in exclude
                 if exclude is not None and update.node_name in exclude.keys():
                     if split_hyperparameter[0] in exclude[update.node_name]:
                         raise ValueError("Found {} in exclude".format(split_hyperparameter[0]))
 
                 components = node.get_components()
-                if split_hyperparameter[0] not in components.keys():
+                # if hyperparameter is __choice__, check if
+                # the components in the value range of search space update
+                # are in components of the choice module
+                if split_hyperparameter[0] == '__choice__':
+                    for choice in update.value_range:
+                        if include is not None and update.node_name in include.keys():
+                            if choice not in include[update.node_name]:
+                                raise ValueError("Not found {} in include".format(choice))
+                        if exclude is not None and update.node_name in exclude.keys():
+                            if choice in exclude[update.node_name]:
+                                raise ValueError("Found {} in exclude".format(choice))
+                        if choice not in components.keys():
+                            raise ValueError("Unknown hyperparameter for choice {}. "
+                                             "Expected update hyperparameter "
+                                             "to be in {} got {}".format(node.__class__.__name__,
+                                                                         components.keys(), choice))
+                # check if the component whose hyperparameter
+                # needs to be updated is in components of the
+                # choice module
+                elif split_hyperparameter[0] not in components.keys():
                     raise ValueError("Unknown hyperparameter for choice {}. "
                                      "Expected update hyperparameter "
                                      "to be in {} got {}".format(node.__class__.__name__,
                                                                  components.keys(), split_hyperparameter[0]))
                 else:
+                    # check if hyperparameter is in the search space of the component
                     component = components[split_hyperparameter[0]]
                     if split_hyperparameter[1] not in component. \
                             get_hyperparameter_search_space(dataset_properties=self.dataset_properties):

autoPyTorch/pipeline/components/base_choice.py

@@ -1,6 +1,7 @@
+import re
 import warnings
 from collections import OrderedDict
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional
 
 from ConfigSpace.configuration_space import Configuration, ConfigurationSpace
 
@@ -9,7 +10,8 @@
 from sklearn.utils import check_random_state
 
 from autoPyTorch.pipeline.components.base_component import autoPyTorchComponent
-from autoPyTorch.utils.common import FitRequirement
+from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace
+from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdate
 
 
 class autoPyTorchChoice(object):
@@ -49,7 +51,7 @@ def __init__(self,
         # self.set_hyperparameters(self.configuration)
         self.choice: Optional[autoPyTorchComponent] = None
 
-        self._cs_updates: Dict[str, Tuple] = dict()
+        self._cs_updates: Dict[str, HyperparameterSearchSpaceUpdate] = dict()
 
     def get_fit_requirements(self) -> Optional[List[FitRequirement]]:
         if self.choice is not None:
@@ -247,35 +249,35 @@ def _check_dataset_properties(self, dataset_properties: Dict[str, Any]) -> None:
         """
         assert isinstance(dataset_properties, dict), "dataset_properties must be a dictionary"
 
-    def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tuple],
-                                   default_value: Union[int, float, str], log: bool = False) -> None:
-        """Allows the user to update a hyperparameter
+    def _apply_search_space_update(self, hyperparameter_search_space_update: HyperparameterSearchSpaceUpdate) -> None:
+        """
+        Applies search space update to the class
 
-        Arguments:
-            name {string} -- name of hyperparameter
-            new_value_range {List[?] -- value range can be either lower, upper or a list of possible conditionals
-            log {bool} -- is hyperparameter logscale
+        Args:
+            hyperparameter_search_space_update (HyperparameterSearchSpaceUpdate):
+                Search Space update for the current autoPyTorchChoice module
         """
 
-        if len(new_value_range) == 0:
-            raise ValueError("The new value range needs at least one value")
-        self._cs_updates[name] = tuple([new_value_range, default_value, log])
+        self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update
 
-    def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, Tuple]:
+    def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, HyperparameterSearchSpace]:
         """Get the search space updates with the given prefix
 
-        Keyword Arguments:
-            prefix {str} -- Only return search space updates with given prefix (default: {None})
+        Args:
+            prefix (str):
+                Only return search space updates with given prefix (default: {None})
 
         Returns:
-            dict -- Mapping of search space updates. Keys don't contain the prefix.
+            Dict[str, HyperparameterSearchSpace]:
+                Mapping of search space updates. Keys don't contain the prefix.
         """
-        if prefix is None:
-            return self._cs_updates
-        result: Dict[str, Tuple] = dict()
 
-        # iterate over all search space updates of this node and filter the ones out, that have the given prefix
+        result: Dict[str, HyperparameterSearchSpace] = dict()
+
+        # iterate over all search space updates of this node and keep the ones that have the given prefix
         for key in self._cs_updates.keys():
-            if key.startswith(prefix):
-                result[key[len(prefix) + 1:]] = self._cs_updates[key]
+            if prefix is None:
+                result[key] = self._cs_updates[key].get_search_space()
+            elif re.search(f'^{prefix}', key) is not None:
+                result[key[len(prefix) + 1:]] = self._cs_updates[key].get_search_space(remove_prefix=prefix)
         return result

autoPyTorch/pipeline/components/base_component.py

@@ -4,19 +4,20 @@
 import sys
 import warnings
 from collections import OrderedDict
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional
 
 from ConfigSpace.configuration_space import Configuration, ConfigurationSpace
 
 from sklearn.base import BaseEstimator
 
-from autoPyTorch.utils.common import FitRequirement
+from autoPyTorch.utils.common import FitRequirement, HyperparameterSearchSpace
+from autoPyTorch.utils.hyperparameter_search_space_update import HyperparameterSearchSpaceUpdate
 
 
 def find_components(
-        package: str,
-        directory: str,
-        base_class: BaseEstimator
+    package: str,
+    directory: str,
+    base_class: BaseEstimator
 ) -> Dict[str, BaseEstimator]:
     """Utility to find component on a given directory,
     that inherit from base_class
@@ -34,8 +35,7 @@ def find_components(
             module = importlib.import_module(full_module_name)
 
             for member_name, obj in inspect.getmembers(module):
-                if inspect.isclass(obj) and issubclass(obj, base_class) and \
-                        obj != base_class:
+                if inspect.isclass(obj) and issubclass(obj, base_class) and obj != base_class:
                     # TODO test if the obj implements the interface
                     # Keep in mind that this only instantiates the ensemble_wrapper,
                     # but not the real target classifier
@@ -96,7 +96,7 @@ class autoPyTorchComponent(BaseEstimator):
     def __init__(self) -> None:
         super().__init__()
         self._fit_requirements: List[FitRequirement] = list()
-        self._cs_updates: Dict[str, Tuple] = dict()
+        self._cs_updates: Dict[str, HyperparameterSearchSpaceUpdate] = dict()
 
     @classmethod
     def get_required_properties(cls) -> Optional[List[str]]:
@@ -140,7 +140,7 @@ def get_properties(dataset_properties: Optional[Dict[str, str]] = None
 
     @staticmethod
     def get_hyperparameter_search_space(
-            dataset_properties: Optional[Dict[str, str]] = None
+        dataset_properties: Optional[Dict[str, str]] = None
     ) -> ConfigurationSpace:
         """Return the configuration space of this classification algorithm.
 
@@ -253,8 +253,7 @@ def __str__(self) -> str:
         name = self.get_properties()['name']
         return "autoPyTorch.pipeline %s" % name
 
-    def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tuple],
-                                   default_value: Union[int, float, str], log: bool = False) -> None:
+    def _apply_search_space_update(self, hyperparameter_search_space_update: HyperparameterSearchSpaceUpdate) -> None:
         """Allows the user to update a hyperparameter
 
         Arguments:
@@ -263,26 +262,18 @@ def _apply_search_space_update(self, name: str, new_value_range: Union[List, Tup
             log {bool} -- is hyperparameter logscale
         """
 
-        if len(new_value_range) == 0:
-            raise ValueError("The new value range needs at least one value")
-        self._cs_updates[name] = tuple([new_value_range, default_value, log])
+        self._cs_updates[hyperparameter_search_space_update.hyperparameter] = hyperparameter_search_space_update
 
-    def _get_search_space_updates(self, prefix: Optional[str] = None) -> Dict[str, Tuple]:
-        """Get the search space updates with the given prefix
-
-        Keyword Arguments:
-            prefix {str} -- Only return search space updates with given prefix (default: {None})
+    def _get_search_space_updates(self) -> Dict[str, HyperparameterSearchSpace]:
+        """Get the search space updates
 
         Returns:
             dict -- Mapping of search space updates. Keys don't contain the prefix.
         """
-        if prefix is None:
-            return self._cs_updates
-        result: Dict[str, Tuple] = dict()
+
+        result: Dict[str, HyperparameterSearchSpace] = dict()
 
         # iterate over all search space updates of this node and keep the ones that have the given prefix
         for key in self._cs_updates.keys():
-            if key.startswith(prefix):
-                # different for autopytorch component as the hyperparameter
-                result[key[len(prefix):]] = self._cs_updates[key]
+            result[key] = self._cs_updates[key].get_search_space()
         return result

autoPyTorch/pipeline/components/preprocessing/image_preprocessing/normalise/base_normalizer_choice.py

@@ -76,10 +76,21 @@ def get_hyperparameter_search_space(self,
                     default = default_
                     break
 
-        preprocessor = CSH.CategoricalHyperparameter('__choice__',
-                                                     list(available_preprocessors.keys()),
-                                                     default_value=default)
-
+        updates = self._get_search_space_updates()
+        if '__choice__' in updates.keys():
+            choice_hyperparameter = updates['__choice__']
+            if not set(choice_hyperparameter.value_range).issubset(available_preprocessors):
+                raise ValueError("Expected given update for {} to have "
+                                 "choices in {} got {}".format(self.__class__.__name__,
+                                                               available_preprocessors,
+                                                               choice_hyperparameter.value_range))
+            preprocessor = CSH.CategoricalHyperparameter('__choice__',
+                                                         choice_hyperparameter.value_range,
+                                                         default_value=choice_hyperparameter.default_value)
+        else:
+            preprocessor = CSH.CategoricalHyperparameter('__choice__',
+                                                         list(available_preprocessors.keys()),
+                                                         default_value=default)
         cs.add_hyperparameter(preprocessor)
 
         # add only child hyperparameters of early_preprocessor choices