Add HPO for xgboost

Author: frederikhoengaard

python/src/lazylearn/pipeline/pipeline.py

@@ -74,3 +74,4 @@ def __init__(self):
         self.holdout_features_df: DataFrame = None
         self.holdout_targets: Series = None
         self.holdout_score: float = None
+        self.regressor = None

python/src/lazylearn/regression/models/xgboost/xgb.py

@@ -4,6 +4,7 @@
 from regression.models.xgboost.xgb_regressor_steps.regressor_step import (
     XGBRegressorStep,
 )
+from regression.models.xgboost.xgb_regressor_steps.hpo_step import HyperParameterOptimizationStep  # noqa
 
 
 class XGBRegressionRunner:
@@ -32,7 +33,9 @@ def fit(self):
 
         self.pipeline.add(OrdinalConverter(cat_vars=cat_vars))
 
-        self.pipeline.add(XGBRegressorStep(random_state=self.random_state))
+        self.pipeline.add(HyperParameterOptimizationStep())
+
+        self.pipeline.add(XGBRegressorStep())
 
         self.pipeline.fit()
 

python/src/lazylearn/regression/models/xgboost/xgb_regressor_steps/hpo_step.py

@@ -0,0 +1,38 @@
+from xgboost import XGBRegressor
+
+from pipeline.pipeline import PipelineStep, RegressionPipeline
+from sklearn.model_selection import RandomizedSearchCV, KFold
+
+
+class HyperParameterOptimizationStep(PipelineStep):
+    def __init__(self, n_splits=5, random_state=None):
+        self.n_splits = n_splits
+        self.random_state = random_state
+        self.param_grid = {
+            'max_depth': [3, 4, 5, 6, 7, 8, 9, 10],
+            'learning_rate': [0.001, 0.01, 0.1, 0.2, 0.3],
+            'subsample': [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
+            'colsample_bytree': [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
+            'colsample_bylevel': [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
+            'min_child_weight': [0.5, 1.0, 3.0, 5.0, 7.0, 10.0],
+            'gamma': [0, 0.25, 0.5, 1.0],
+            'n_estimators': [100, 200, 300, 500, 1000]
+        }
+
+    def fit(self, pipeline: RegressionPipeline):
+        xgbtuned = XGBRegressor()
+
+        cv = KFold(n_splits=self.n_splits)  # time series cross validation split
+        xgbtunedreg = RandomizedSearchCV(
+            xgbtuned,
+            param_distributions=self.param_grid,
+            scoring='neg_mean_squared_error',
+            n_iter=20,
+            n_jobs=-1,
+            cv=cv,
+            verbose=1,
+        )
+        pipeline.regressor = xgbtunedreg
+
+    def predict(self, pipeline: RegressionPipeline):
+        pass

python/src/lazylearn/regression/models/xgboost/xgb_regressor_steps/regressor_step.py

@@ -1,25 +1,20 @@
 from pipeline.pipeline import PipelineStep, RegressionPipeline
-from xgboost import XGBRegressor
 
 
 class XGBRegressorStep(PipelineStep):
-    def __init__(self, random_state=None):
-        self.regressor = XGBRegressor(
-            n_estimators=1000, random_state=random_state
-        )  # noqa
 
     def fit(self, pipeline: RegressionPipeline):
         pipeline.feature_list = [
             item for item in pipeline.feature_list if item != pipeline.target
         ]
         print("Fitting XGBRegressor")
-        self.regressor.fit(
+        pipeline.regressor.fit(
             X=pipeline.train_features_df[pipeline.feature_list],
             y=pipeline.train_targets,
         )  # noqa
         print("XGBRegressor fitted!")
 
     def predict(self, pipeline: RegressionPipeline):
-        pipeline.tmp_pred = self.regressor.predict(
+        pipeline.tmp_pred = pipeline.regressor.predict(
             X=pipeline.tmp_test[pipeline.feature_list]
         )