Cocktail hotfixes (#245)

* Fixes for the development branch and regularization cocktails

* Update implementation

* Fix unit tests temporarily

* Implementation update and bug fixes

* Removing unecessary code

* Addressing Ravin's comments

[refactor] Address Shuhei's comments

[refactor] Address Shuhei's comments

[refactor] Address Shuhei's comments

[refactor] Address Shuhei's comments

Author: ArlindKadra

autoPyTorch/api/base_task.py

@@ -401,7 +401,7 @@ def set_pipeline_config(self, **pipeline_config_kwargs: Any) -> None:
             None
         """
         unknown_keys = []
-        for option, value in pipeline_config_kwargs.items():
+        for option in pipeline_config_kwargs.keys():
             if option in self.pipeline_options.keys():
                 pass
             else:

autoPyTorch/api/tabular_classification.py

@@ -379,7 +379,16 @@ def search(
             y_test=y_test,
             resampling_strategy=self.resampling_strategy,
             resampling_strategy_args=self.resampling_strategy_args,
-            dataset_name=dataset_name)
+            dataset_name=dataset_name
+        )
+
+        if not isinstance(self.resampling_strategy, (CrossValTypes, HoldoutValTypes)):
+            raise ValueError(
+                'Hyperparameter optimization requires a validation split. '
+                'Expected `self.resampling_strategy` to be either '
+                '(CrossValTypes, HoldoutValTypes), but got {}'.format(self.resampling_strategy)
+            )
+
 
         return self._search(
             dataset=self.dataset,
@@ -420,23 +429,23 @@ def predict(
             raise ValueError("predict() is only supported after calling search. Kindly call first "
                              "the estimator search() method.")
 
-        X_test = self.InputValidator.feature_validator.transform(X_test)
+        X_test = self.input_validator.feature_validator.transform(X_test)
         predicted_probabilities = super().predict(X_test, batch_size=batch_size,
                                                   n_jobs=n_jobs)
 
-        if self.InputValidator.target_validator.is_single_column_target():
+        if self.input_validator.target_validator.is_single_column_target():
             predicted_indexes = np.argmax(predicted_probabilities, axis=1)
         else:
             predicted_indexes = (predicted_probabilities > 0.5).astype(int)
 
         # Allow to predict in the original domain -- that is, the user is not interested
         # in our encoded values
-        return self.InputValidator.target_validator.inverse_transform(predicted_indexes)
+        return self.input_validator.target_validator.inverse_transform(predicted_indexes)
 
     def predict_proba(self,
                       X_test: Union[np.ndarray, pd.DataFrame, List],
                       batch_size: Optional[int] = None, n_jobs: int = 1) -> np.ndarray:
-        if self.InputValidator is None or not self.InputValidator._is_fitted:
+        if self.input_validator is None or not self.input_validator._is_fitted:
             raise ValueError("predict() is only supported after calling search. Kindly call first "
                              "the estimator search() method.")
         X_test = self.InputValidator.feature_validator.transform(X_test)

autoPyTorch/api/tabular_regression.py

@@ -377,7 +377,16 @@ def search(
             y_test=y_test,
             resampling_strategy=self.resampling_strategy,
             resampling_strategy_args=self.resampling_strategy_args,
-            dataset_name=dataset_name)
+            dataset_name=dataset_name
+        )
+
+        if not isinstance(self.resampling_strategy, (CrossValTypes, HoldoutValTypes)):
+            raise ValueError(
+                'Hyperparameter optimization requires a validation split. '
+                'Expected `self.resampling_strategy` to be either '
+                '(CrossValTypes, HoldoutValTypes), but got {}'.format(self.resampling_strategy)
+            )
+
 
         return self._search(
             dataset=self.dataset,
@@ -404,14 +413,14 @@ def predict(
             batch_size: Optional[int] = None,
             n_jobs: int = 1
     ) -> np.ndarray:
-        if self.InputValidator is None or not self.InputValidator._is_fitted:
+        if self.input_validator is None or not self.input_validator._is_fitted:
             raise ValueError("predict() is only supported after calling search. Kindly call first "
                              "the estimator search() method.")
 
-        X_test = self.InputValidator.feature_validator.transform(X_test)
+        X_test = self.input_validator.feature_validator.transform(X_test)
         predicted_values = super().predict(X_test, batch_size=batch_size,
                                            n_jobs=n_jobs)
 
         # Allow to predict in the original domain -- that is, the user is not interested
         # in our encoded values
-        return self.InputValidator.target_validator.inverse_transform(predicted_values)
+        return self.input_validator.target_validator.inverse_transform(predicted_values)

autoPyTorch/data/tabular_feature_validator.py

@@ -391,7 +391,7 @@ def _get_columns_to_encode(
         feat_type = []
 
         # Make sure each column is a valid type
-        for i, column in enumerate(X.columns):
+        for column in X.columns:
             if X[column].dtype.name in ['category', 'bool']:
 
                 transformed_columns.append(column)
@@ -512,7 +512,7 @@ def infer_objects(self, X: pd.DataFrame) -> pd.DataFrame:
                     X[key] = X[key].astype(dtype.name)
                 except Exception as e:
                     # Try inference if possible
-                    self.logger.warning(f"Tried to cast column {key} to {dtype} caused {e}")
+                    self.logger.warning(f'Casting the column {key} to {dtype} caused the exception {e}')
                     pass
         else:
             X = X.infer_objects()

autoPyTorch/evaluation/fit_evaluator.py

@@ -58,10 +58,12 @@ def __init__(self, backend: Backend, queue: Queue,
             pipeline_config=pipeline_config,
             search_space_updates=search_space_updates
         )
-        assert isinstance(self.datamanager.resampling_strategy, NoResamplingStrategyTypes),\
-            "This Evaluator is used for fitting a pipeline on the whole dataset. " \
-            "Expected 'self.resampling_strategy' to be" \
-            " 'NoResamplingStrategyTypes' got {}".format(self.datamanager.resampling_strategy)
+        if not isinstance(self.datamanager.resampling_strategy, NoResamplingStrategyTypes):
+            raise ValueError(
+                "FitEvaluator needs to be fitted on the whole dataset and resampling_strategy "
+                "must be `NoResamplingStrategyTypes`, but got {}".format(
+                    self.datamanager.resampling_strategy
+                ))
 
         self.splits = self.datamanager.splits
         self.Y_target: Optional[np.ndarray] = None

autoPyTorch/evaluation/tae.py

@@ -184,10 +184,10 @@ def __init__(
         elif isinstance(self.resampling_strategy, NoResamplingStrategyTypes):
             eval_function = autoPyTorch.evaluation.fit_evaluator.eval_function
         else:
-            raise ValueError("Unknown resampling strategy specified."
-                             "Expected resampling strategy to be in "
-                             "'(HoldoutValTypes, CrossValTypes, NoResamplingStrategyTypes)"
-                             "got {}".format(self.resampling_strategy))
+            raise ValueError("resampling strategy must be in "
+                             "(HoldoutValTypes, CrossValTypes, NoResamplingStrategyTypes), "
+                             "but got {}.".format(self.resampling_strategy)
+            )
 
         self.worst_possible_result = cost_for_crash
 
@@ -336,6 +336,7 @@ def run(
         info: Optional[List[RunValue]]
         additional_run_info: Dict[str, Any]
         try:
+            # By default, self.ta is fit_predict_try_except_decorator
             obj = pynisher.enforce_limits(**pynisher_arguments)(self.ta)
             obj(**obj_kwargs)
         except Exception as e:

autoPyTorch/evaluation/train_evaluator.py

@@ -149,11 +149,13 @@ def __init__(self, backend: Backend, queue: Queue,
             pipeline_config=pipeline_config,
             search_space_updates=search_space_updates
         )
-        assert isinstance(self.datamanager.resampling_strategy, (CrossValTypes, HoldoutValTypes)),\
-            "This Evaluator is used for HPO Search. " \
-            "Val Split is required for HPO search. " \
-            "Expected 'self.resampling_strategy' in" \
-            " '(CrossValTypes, HoldoutValTypes)' got {}".format(self.datamanager.resampling_strategy)
+
+        if not isinstance(self.datamanager.resampling_strategy, (CrossValTypes, HoldoutValTypes)):
+            raise ValueError(
+                'TrainEvaluator expect to have (CrossValTypes, HoldoutValTypes) as '
+                'resampling_strategy, but got {}'.format(self.datamanager.resampling_strategy)
+            )
+
 
         self.splits = self.datamanager.splits
         if self.splits is None:

autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py

@@ -64,7 +64,8 @@ def _add_group(self, in_features: int, out_features: int,
             out_features (int): output dimensionality for the current block
             blocks_per_group (int): Number of ResNet per group
             last_block_index (int): block index for shake regularization
-            dropout (bool): whether or not use dropout
+            dropout (None, float): dropout value for the group. If none,
+                no dropout is applied.
         """
         blocks = list()
         for i in range(blocks_per_group):
@@ -180,9 +181,7 @@ def get_hyperparameter_search_space(
 
         if skip_connection_flag:
 
-            shake_drop_prob_flag = False
-            if 'shake-drop' in multi_branch_choice.value_range:
-                shake_drop_prob_flag = True
+            shake_drop_prob_flag = 'shake-drop' in multi_branch_choice.value_range
 
             mb_choice = get_hyperparameter(multi_branch_choice, CategoricalHyperparameter)
             cs.add_hyperparameter(mb_choice)
@@ -290,13 +289,21 @@ def _build_block(self, in_features: int, out_features: int) -> nn.Module:
             if self.config['use_batch_norm']:
                 layers.append(nn.BatchNorm1d(in_features))
             layers.append(self.activation())
+        elif not self.config['use_skip_connection']:
+            # if start norm is not None and skip connection is False
+            # we will never apply the start_norm for the first layer in the block,
+            # which is why we should account for this case.
+            if self.config['use_batch_norm']:
+                layers.append(nn.BatchNorm1d(in_features))
+            layers.append(self.activation())
+
         layers.append(nn.Linear(in_features, out_features))
 
         if self.config['use_batch_norm']:
             layers.append(nn.BatchNorm1d(out_features))
         layers.append(self.activation())
 
-        if self.config["use_dropout"]:
+        if self.dropout is not None:
             layers.append(nn.Dropout(self.dropout))
         layers.append(nn.Linear(out_features, out_features))
 
@@ -321,6 +328,7 @@ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
             if self.config["use_skip_connection"]:
                 residual = self.shortcut(x)
 
+        # TODO make the below code better
         if self.config["use_skip_connection"]:
             if self.config["multi_branch_choice"] == 'shake-shake':
                 x1 = self.layers(x)

autoPyTorch/pipeline/components/setup/network_backbone/ShapedResNetBackbone.py

@@ -31,11 +31,13 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential:
         out_features = self.config["output_dim"]
 
         # use the get_shaped_neuron_counts to update the number of units
-        neuron_counts = get_shaped_neuron_counts(self.config['resnet_shape'],
-                                                 in_features,
-                                                 out_features,
-                                                 self.config['max_units'],
-                                                 self.config['num_groups'] + 2)[:-1]
+        neuron_counts = get_shaped_neuron_counts(
+            shape=self.config['resnet_shape'],
+            in_feat=in_features,
+            out_feat=out_features,
+            max_neurons=self.config['max_units'],
+            layer_count=self.config['num_groups'] + 2,
+        )[:-1]
         self.config.update(
             {"num_units_%d" % (i): num for i, num in enumerate(neuron_counts)}
         )
@@ -45,12 +47,12 @@ def build_backbone(self, input_shape: Tuple[int, ...]) -> torch.nn.Sequential:
             # n_units for the architecture, since, it is mostly implemented for the
             # output layer, which is part of the head and not of the backbone.
             dropout_shape = get_shaped_neuron_counts(
-                self.config['dropout_shape'], 0, 0, 1000, self.config['num_groups']
-            )
-
-            dropout_shape = [
-                dropout / 1000 * self.config["max_dropout"] for dropout in dropout_shape
-            ]
+                shape=self.config['dropout_shape'],
+                in_feat=0,
+                out_feat=0,
+                max_neurons=self.config["max_dropout"],
+                layer_count=self.config['num_groups'] + 1,
+            )[:-1]
 
             self.config.update(
                 {"dropout_%d" % (i + 1): dropout for i, dropout in enumerate(dropout_shape)}

autoPyTorch/pipeline/components/setup/network_head/no_head.py

@@ -20,7 +20,7 @@ class NoHead(NetworkHeadComponent):
     """
 
     def build_head(self, input_shape: Tuple[int, ...], output_shape: Tuple[int, ...]) -> nn.Module:
-        layers = [nn.Flatten()]
+        layers = []
         in_features = np.prod(input_shape).item()
         out_features = np.prod(output_shape).item()
         layers.append(_activations[self.config["activation"]]())
@@ -34,8 +34,8 @@ def get_properties(dataset_properties: Optional[Dict[str, Any]] = None) -> Dict[
             'shortname': 'NoHead',
             'name': 'NoHead',
             'handles_tabular': True,
-            'handles_image': True,
-            'handles_time_series': True,
+            'handles_image': False,
+            'handles_time_series': False,
         }
 
     @staticmethod