Make sure the performance of pipeline is at least 0.8

Author: franchuterivera

autoPyTorch/pipeline/components/setup/lr_scheduler/base_scheduler_choice.py

@@ -138,13 +138,14 @@ def get_hyperparameter_search_space(
             raise ValueError("No scheduler found")
 
         if default is None:
-            defaults = ['no_LRScheduler',
-                        'LambdaLR',
-                        'StepLR',
-                        'ExponentialLR',
-                        'CosineAnnealingLR',
-                        'ReduceLROnPlateau'
-                        ]
+            defaults = [
+                'ReduceLROnPlateau',
+                'CosineAnnealingLR',
+                'no_LRScheduler',
+                'LambdaLR',
+                'StepLR',
+                'ExponentialLR',
+            ]
             for default_ in defaults:
                 if default_ in available_schedulers:
                     default = default_

test/test_pipeline/components/test_setup_networks.py

@@ -32,6 +32,10 @@ def test_pipeline_fit(self, fit_dictionary, backbone, head):
         assert backbone == config.get('network_backbone:__choice__', None)
         assert head == config.get('network_head:__choice__', None)
         pipeline.set_hyperparameters(config)
+
+        # Need more epochs to make sure validation performance is met
+        fit_dictionary['epochs'] = 100
+
         pipeline.fit(fit_dictionary)
 
         # To make sure we fitted the model, there should be a
@@ -44,9 +48,10 @@ def test_pipeline_fit(self, fit_dictionary, backbone, head):
         assert run_summary.total_parameter_count > 0
         assert 'accuracy' in run_summary.performance_tracker['train_metrics'][1]
 
-        # Commented out the next line as some pipelines are not
-        # achieving this accuracy with default configuration and 10 epochs
-        # To be added once we fix the search space
-        # assert run_summary.performance_tracker['val_metrics'][fit_dictionary['epochs']]['accuracy'] >= 0.8
+        # Make sure default pipeline achieves a good score for dummy datasets
+        assert run_summary.performance_tracker[
+            'val_metrics'
+        ][fit_dictionary['epochs']]['accuracy'] >= 0.8, run_summary.performance_tracker['val_metrics']
+
         # Make sure a network was fit
         assert isinstance(pipeline.named_steps['network'].get_network(), torch.nn.Module)