66from autoPyTorch .ensemble .abstract_ensemble import AbstractEnsemble
77from autoPyTorch .pipeline .base_pipeline import BasePipeline
88from autoPyTorch .pipeline .components .training .metrics .base import autoPyTorchMetric
9- from autoPyTorch .pipeline .components .training .metrics .utils import calculate_score
9+ from autoPyTorch .pipeline .components .training .metrics .utils import calculate_loss
1010
1111
1212class EnsembleSelection (AbstractEnsemble ):
@@ -39,6 +39,24 @@ def fit(
3939 labels : np .ndarray ,
4040 identifiers : List [Tuple [int , int , float ]],
4141 ) -> AbstractEnsemble :
42+ """
43+ Builds a ensemble given the individual models out of fold predictions.
44+ Fundamentally, defines a set of weights on how to perform a soft-voting
45+ aggregation of the models in the given identifiers.
46+
47+ Args:
48+ predictions (List[np.array]):
49+ A list of individual model predictions of shape (n_datapoints, n_targets)
50+ corresponding to the OutOfFold estimate of the ground truth
51+ labels (np.ndarray):
52+ The ground truth targets of shape (n_datapoints, n_targets)
53+ identifiers: List[Tuple[int, int, float]]
54+ A list of model identifiers, each with the form
55+ (seed, number of run, budget)
56+
57+ Returns:
58+ A copy of self
59+ """
4260 self .ensemble_size = int (self .ensemble_size )
4361 if self .ensemble_size < 1 :
4462 raise ValueError ('Ensemble size cannot be less than one!' )
@@ -53,7 +71,20 @@ def _fit(
5371 predictions : List [np .ndarray ],
5472 labels : np .ndarray ,
5573 ) -> None :
56- """Fast version of Rich Caruana's ensemble selection method."""
74+ """
75+ Fast version of Rich Caruana's ensemble selection method.
76+
77+ For more details, please check the paper
78+ "Ensemble Selection from Library of Models" by R Caruana (2004)
79+
80+ Args:
81+ predictions (List[np.array]):
82+ A list of individual model predictions of shape (n_datapoints, n_targets)
83+ corresponding to the OutOfFold estimate of the ground truth
84+ identifiers (List[Tuple[int, int, float]]):
85+ A list of model identifiers, each with the form
86+ (seed, number of run, budget)
87+ """
5788 self .num_input_models_ = len (predictions )
5889
5990 ensemble = [] # type: List[np.ndarray]
@@ -71,60 +102,47 @@ def _fit(
71102 dtype = np .float64 ,
72103 )
73104 for i in range (ensemble_size ):
74- scores = np .zeros (
105+ losses = np .zeros (
75106 (len (predictions )),
76107 dtype = np .float64 ,
77108 )
78109 s = len (ensemble )
79- if s == 0 :
80- weighted_ensemble_prediction .fill (0.0 )
81- else :
82- weighted_ensemble_prediction .fill (0.0 )
83- for pred in ensemble :
84- np .add (
85- weighted_ensemble_prediction ,
86- pred ,
87- out = weighted_ensemble_prediction ,
88- )
89- np .multiply (
90- weighted_ensemble_prediction ,
91- 1 / s ,
92- out = weighted_ensemble_prediction ,
93- )
94- np .multiply (
110+ if s > 0 :
111+ np .add (
95112 weighted_ensemble_prediction ,
96- ( s / float ( s + 1 )) ,
113+ ensemble [ - 1 ] ,
97114 out = weighted_ensemble_prediction ,
98115 )
99116
117+ # Memory-efficient averaging!
100118 for j , pred in enumerate (predictions ):
101- # Memory-efficient averaging!
102- fant_ensemble_prediction .fill (0.0 )
119+ # fant_ensemble_prediction is the prediction of the current ensemble
120+ # and should be ([predictions[selected_prev_iterations] + predictions[j])/(s+1)
121+ # We overwrite the contents of fant_ensemble_prediction
122+ # directly with weighted_ensemble_prediction + new_prediction and then scale for avg
103123 np .add (
104- fant_ensemble_prediction ,
105124 weighted_ensemble_prediction ,
125+ pred ,
106126 out = fant_ensemble_prediction
107127 )
108- np .add (
128+ np .multiply (
109129 fant_ensemble_prediction ,
110- (1. / float (s + 1 )) * pred ,
130+ (1. / float (s + 1 )),
111131 out = fant_ensemble_prediction
112132 )
113133
114- # Calculate score is versatile and can return a dict of score
115- # when all_scoring_functions=False, we know it will be a float
116- score = calculate_score (
134+ # Calculate loss is versatile and can return a dict of slosses
135+ losses [j ] = calculate_loss (
117136 metrics = [self .metric ],
118137 target = labels ,
119138 prediction = fant_ensemble_prediction ,
120139 task_type = self .task_type ,
121- )
122- scores [j ] = self .metric ._optimum - score [self .metric .name ]
140+ )[self .metric .name ]
123141
124- all_best = np .argwhere (scores == np .nanmin (scores )).flatten ()
142+ all_best = np .argwhere (losses == np .nanmin (losses )).flatten ()
125143 best = self .random_state .choice (all_best )
126144 ensemble .append (predictions [best ])
127- trajectory .append (scores [best ])
145+ trajectory .append (losses [best ])
128146 order .append (best )
129147
130148 # Handle special case
@@ -133,9 +151,15 @@ def _fit(
133151
134152 self .indices_ = order
135153 self .trajectory_ = trajectory
136- self .train_score_ = trajectory [- 1 ]
154+ self .train_loss_ = trajectory [- 1 ]
137155
138156 def _calculate_weights (self ) -> None :
157+ """
158+ Calculates the contribution each of the individual models
159+ should have, in the final ensemble soft voting. It does so by
160+ a frequency counting scheme. In particular, how many times a model
161+ was used during hill climbing optimization.
162+ """
139163 ensemble_members = Counter (self .indices_ ).most_common ()
140164 weights = np .zeros (
141165 (self .num_input_models_ ,),
@@ -151,6 +175,19 @@ def _calculate_weights(self) -> None:
151175 self .weights_ = weights
152176
153177 def predict (self , predictions : Union [np .ndarray , List [np .ndarray ]]) -> np .ndarray :
178+ """
179+ Given a list of predictions from the individual model, this method
180+ aggregates the predictions using a soft voting scheme with the weights
181+ found during training.
182+
183+ Args:
184+ predictions (List[np.ndarray]):
185+ A list of predictions from the individual base models.
186+
187+ Returns:
188+ average (np.array): Soft voting predictions of ensemble models, using
189+ the weights found during ensemble selection (self._weights)
190+ """
154191
155192 average = np .zeros_like (predictions [0 ], dtype = np .float64 )
156193 tmp_predictions = np .empty_like (predictions [0 ], dtype = np .float64 )
@@ -191,6 +228,19 @@ def get_models_with_weights(
191228 self ,
192229 models : Dict [Any , BasePipeline ]
193230 ) -> List [Tuple [float , BasePipeline ]]:
231+ """
232+ Handy function to tag the provided input models with a given weight.
233+
234+ Args:
235+ models (List[Tuple[float, BasePipeline]]):
236+ A dictionary that maps a model's name to it's actual python object.
237+
238+ Returns:
239+ output (List[Tuple[float, BasePipeline]]):
240+ each model with the related weight, sorted by ascending
241+ performance. Notice that ensemble selection solves a minimization
242+ problem.
243+ """
194244 output = []
195245 for i , weight in enumerate (self .weights_ ):
196246 if weight > 0.0 :
@@ -203,6 +253,15 @@ def get_models_with_weights(
203253 return output
204254
205255 def get_selected_model_identifiers (self ) -> List [Tuple [int , int , float ]]:
256+ """
257+ After training of ensemble selection, not all models will be used.
258+ Some of them will have zero weight. This procedure filters this models
259+ out.
260+
261+ Returns:
262+ output (List[Tuple[int, int, float]]):
263+ The models actually used by ensemble selection
264+ """
206265 output = []
207266
208267 for i , weight in enumerate (self .weights_ ):
@@ -213,4 +272,11 @@ def get_selected_model_identifiers(self) -> List[Tuple[int, int, float]]:
213272 return output
214273
215274 def get_validation_performance (self ) -> float :
275+ """
276+ Returns the best optimization performance seen during hill climbing
277+
278+ Returns:
279+ (float):
280+ best ensemble training performance
281+ """
216282 return self .trajectory_ [- 1 ]
0 commit comments