Merge branch 'develop' into main

Author: manujosephv

README.md

@@ -69,6 +69,7 @@ For complete Documentation with tutorials visit []
 * [TabNet: Attentive Interpretable Tabular Learning](https://arxiv.org/abs/1908.07442) is another model coming out of Google Research which uses Sparse Attention in multiple steps of decision making to model the output.
 * [Mixture Density Networks](https://publications.aston.ac.uk/id/eprint/373/1/NCRG_94_004.pdf) is a regression model which uses gaussian components to approximate the target function and  provide a probabilistic prediction out of the box.
 * [AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks](https://arxiv.org/abs/1810.11921) is a model which tries to learn interactions between the features in an automated way and create a better representation and then use this representation in downstream task
+* [TabTransformer](https://arxiv.org/abs/2012.06678) is an adaptation of the Transformer model for Tabular Data which creates contextual representations for categorical features.
 
 To implement new models, see the [How to implement new models tutorial](https://github.com/manujosephv/pytorch_tabular/blob/main/docs/04-Implementing%20New%20Architectures.ipynb). It covers basic as well as advanced architectures.
 
@@ -112,9 +113,9 @@ loaded_model = TabularModel.load_from_checkpoint("examples/basic")
 ```
 ## Blogs
 
-[PyTorch Tabular – A Framework for Deep Learning for Tabular Data](https://deep-and-shallow.com/2021/01/27/pytorch-tabular-a-framework-for-deep-learning-for-tabular-data/)
-[Neural Oblivious Decision Ensembles(NODE) – A State-of-the-Art Deep Learning Algorithm for Tabular Data](https://deep-and-shallow.com/2021/02/25/neural-oblivious-decision-ensemblesnode-a-state-of-the-art-deep-learning-algorithm-for-tabular-data/)
-[Mixture Density Networks: Probabilistic Regression for Uncertainty Estimation](https://deep-and-shallow.com/2021/03/20/mixture-density-networks-probabilistic-regression-for-uncertainty-estimation/)
+- [PyTorch Tabular – A Framework for Deep Learning for Tabular Data](https://deep-and-shallow.com/2021/01/27/pytorch-tabular-a-framework-for-deep-learning-for-tabular-data/)
+- [Neural Oblivious Decision Ensembles(NODE) – A State-of-the-Art Deep Learning Algorithm for Tabular Data](https://deep-and-shallow.com/2021/02/25/neural-oblivious-decision-ensemblesnode-a-state-of-the-art-deep-learning-algorithm-for-tabular-data/)
+- [Mixture Density Networks: Probabilistic Regression for Uncertainty Estimation](https://deep-and-shallow.com/2021/03/20/mixture-density-networks-probabilistic-regression-for-uncertainty-estimation/)
 
 ## Future Roadmap(Contributions are Welcome)
 
@@ -124,8 +125,15 @@ loaded_model = TabularModel.load_from_checkpoint("examples/basic")
 4. Add Fourier Encoding for cyclic time variables
 5. Integrate Optuna Hyperparameter Tuning
 6. Add Text and Image Modalities for mixed modal problems
-7. Integrate Wide and Deep model
-8. Integrate TabTransformer
+7. Add Variable Importance
+8. Integrate SHAP for interpretability
+        
+**DL Models**
+     
+9. [DNF-Net: A Neural Architecture for Tabular Data](https://www.semanticscholar.org/paper/DNF-Net%3A-A-Neural-Architecture-for-Tabular-Data-Abutbul-Elidan/99c49f3a917815eed2144bfb5d064623ff09ade5)
+10. [Attention augmented differentiable forest for tabular data](https://www.semanticscholar.org/paper/Attention-augmented-differentiable-forest-for-data-Chen/57990b40affc5f34f4029dab39bc78e44e7d3b10)
+11. [XBNet : An Extremely Boosted Neural Network](https://arxiv.org/abs/2106.05239v2)
+12. [Revisiting Deep Learning Models for Tabular Data](https://arxiv.org/abs/2106.11959)
 ## Citation
 If you use PyTorch Tabular for a scientific publication, we would appreciate citations to the published software and the following paper:
 

examples/to_test_classification.py

@@ -1,3 +1,5 @@
+from pytorch_tabular.models.tab_transformer.config import TabTransformerConfig
+from pytorch_tabular.models.ft_transformer.config import FTTransformerConfig
 import torch
 import numpy as np
 from torch.functional import norm
@@ -88,7 +90,7 @@
     continuous_feature_transform=None,#"quantile_normal",
     normalize_continuous_features=False,
 )
-model_config = CategoryEmbeddingModelConfig(task="classification", metrics=["f1","accuracy"], metrics_params=[{"num_classes":num_classes},{}])
+# model_config = CategoryEmbeddingModelConfig(task="classification", metrics=["f1","accuracy"], metrics_params=[{"num_classes":num_classes},{}])
 # model_config = NodeConfig(
 #     task="classification",
 #     depth=4,
@@ -97,7 +99,25 @@
 #     metrics=["f1", "accuracy"],
 #     metrics_params=[{"num_classes": num_classes, "average": "macro"}, {}],
 # )
-trainer_config = TrainerConfig(gpus=-1, auto_select_gpus=True, fast_dev_run=False, max_epochs=5, batch_size=1024)
+model_config = TabTransformerConfig(
+    task="classification",
+    metrics=["f1", "accuracy"],
+    share_embedding = True,
+    share_embedding_strategy="add",
+    shared_embedding_fraction=0.25,
+    metrics_params=[{"num_classes": num_classes, "average": "macro"}, {}],
+)
+# model_config = FTTransformerConfig(
+#     task="classification",
+#     metrics=["f1", "accuracy"],
+#     # embedding_initialization=None,
+#     embedding_bias=True,
+#     share_embedding = True,
+#     share_embedding_strategy="fraction",
+#     shared_embedding_fraction=0.25,
+#     metrics_params=[{"num_classes": num_classes, "average": "macro"}, {}],
+# )
+trainer_config = TrainerConfig(gpus=-1, auto_select_gpus=True, fast_dev_run=True, max_epochs=5, batch_size=512)
 experiment_config = ExperimentConfig(project_name="PyTorch Tabular Example", 
                                      run_name="node_forest_cov", 
                                      exp_watch="gradients", 
@@ -127,11 +147,18 @@
     # loss=cust_loss,
     train_sampler=sampler)
 
-result = tabular_model.evaluate(test)
-print(result)
-# test.drop(columns=target_name, inplace=True)
+from pytorch_tabular.categorical_encoders import CategoricalEmbeddingTransformer
+transformer = CategoricalEmbeddingTransformer(tabular_model)
+train_transform = transformer.fit_transform(train)
+# test_transform = transformer.transform(test)
+# ft = tabular_model.model.feature_importance()
+# result = tabular_model.evaluate(test)
+# print(result)
+# test.drop(columns=ta6rget_name, inplace=True)
 # pred_df = tabular_model.predict(test)
+# print(pred_df.head())
 # pred_df.to_csv("output/temp2.csv")
 # tabular_model.save_model("test_save")
 # new_model = TabularModel.load_from_checkpoint("test_save")
-# result = new_model.evaluate(test)
+# result = new_model.evaluate(test)
+# print(result)

pytorch_tabular/config/config.py

@@ -479,22 +479,22 @@ def read_from_yaml(filename: str = "config/optimizer_config.yml"):
 class ExperimentRunManager:
     def __init__(
         self,
-        exp_version_manager: str = ".tmp/exp_version_manager.yml",
+        exp_version_manager: str = ".pt_tmp/exp_version_manager.yml",
     ) -> None:
         """The manages the versions of the experiments based on the name. It is a simple dictionary(yaml) based lookup.
         Primary purpose is to avoid overwriting of saved models while runing the training without changing the experiment name.
 
         Args:
             exp_version_manager (str, optional): The path of the yml file which acts as version control.
-            Defaults to ".tmp/exp_version_manager.yml".
+            Defaults to ".pt_tmp/exp_version_manager.yml".
         """
         super().__init__()
         self._exp_version_manager = exp_version_manager
         if os.path.exists(exp_version_manager):
             self.exp_version_manager = OmegaConf.load(exp_version_manager)
         else:
             self.exp_version_manager = OmegaConf.create({})
-            os.makedirs(".tmp", exist_ok=True)
+            os.makedirs(".pt_tmp", exist_ok=True)
             with open(self._exp_version_manager, "w") as file:
                 OmegaConf.save(config=self.exp_version_manager, f=file)
 

pytorch_tabular/models/__init__.py

@@ -12,6 +12,8 @@
     AutoIntMDNConfig
 )
 from .autoint import AutoIntConfig, AutoIntModel
+from .tab_transformer import TabTransformerConfig, TabTransformerModel
+from .ft_transformer import FTTransformerConfig, FTTransformerModel
 from .base_model import BaseModel
 from . import category_embedding, node, mixture_density, tabnet, autoint
 
@@ -33,9 +35,14 @@
     "AutoIntMDNConfig",
     "AutoIntConfig",
     "AutoIntModel",
+    "TabTransformerConfig", 
+    "TabTransformerModel",
+    "FTTransformerConfig", 
+    "FTTransformerModel",
     "category_embedding",
     "node",
     "mixture_density",
     "tabnet",
     "autoint",
+    "tab_transformer"
 ]

pytorch_tabular/models/autoint/autoint.py

@@ -46,24 +46,24 @@ def _build_network(self):
         # Deep Layers
         _curr_units = self.hparams.embedding_dim
         if self.hparams.deep_layers:
-            activation = getattr(nn, self.hparams.activation)
             # Linear Layers
             layers = []
             for units in self.hparams.layers.split("-"):
                 layers.extend(
                     _linear_dropout_bn(
-                        self.hparams,
+                        self.hparams.activation,
+                        self.hparams.initialization,
+                        self.hparams.use_batch_norm,
                         _curr_units,
                         int(units),
-                        activation,
                         self.hparams.dropout,
                     )
                 )
                 _curr_units = int(units)
             self.linear_layers = nn.Sequential(*layers)
         # Projection to Multi-Headed Attention Dims
         self.attn_proj = nn.Linear(_curr_units, self.hparams.attn_embed_dim)
-        _initialize_layers(self.hparams, self.attn_proj)
+        _initialize_layers(self.hparams.activation, self.hparams.initialization, self.attn_proj)
         # Multi-Headed Attention Layers
         self.self_attns = nn.ModuleList(
             [
@@ -152,7 +152,7 @@ def _build_network(self):
         self.output_layer = nn.Linear(
             self.backbone.output_dim, self.hparams.output_dim
         )  # output_dim auto-calculated from other config
-        _initialize_layers(self.hparams, self.output_layer)
+        _initialize_layers(self.hparams.activation, self.hparams.initialization, self.output_layer)
 
     def forward(self, x: Dict):
         x = self.backbone(x)
@@ -165,3 +165,9 @@ def forward(self, x: Dict):
                 y_min, y_max = self.hparams.target_range[i]
                 y_hat[:, i] = y_min + nn.Sigmoid()(y_hat[:, i]) * (y_max - y_min)
         return {"logits": y_hat, "backbone_features": x}
+    
+    def extract_embedding(self):
+        if len(self.hparams.categorical_cols) > 0:
+            return self.backbone.cat_embedding_layers
+        else:
+            raise ValueError("Model has been trained with no categorical feature and therefore can't be used as a Categorical Encoder")

pytorch_tabular/models/base_model.py

@@ -121,7 +121,7 @@ def calculate_metrics(self, y, y_hat, tag):
         for metric, metric_str, metric_params in zip(
             self.metrics, self.hparams.metrics, self.hparams.metrics_params
         ):
-            if (self.hparams.task == "regression") and (self.hparams.output_dim > 1):
+            if (self.hparams.task == "regression"):
                 _metrics = []
                 for i in range(self.hparams.output_dim):
                     if (

pytorch_tabular/models/category_embedding/category_embedding_model.py

@@ -24,7 +24,6 @@ def __init__(self, config: DictConfig, **kwargs):
         self._build_network()
 
     def _build_network(self):
-        activation = getattr(nn, self.hparams.activation)
         # Linear Layers
         layers = []
         _curr_units = self.embedding_cat_dim + self.hparams.continuous_dim
@@ -33,10 +32,11 @@ def _build_network(self):
         for units in self.hparams.layers.split("-"):
             layers.extend(
                 _linear_dropout_bn(
-                    self.hparams,
+                    self.hparams.activation,
+                    self.hparams.initialization,
+                    self.hparams.use_batch_norm,
                     _curr_units,
                     int(units),
-                    activation,
                     self.hparams.dropout,
                 )
             )
@@ -69,7 +69,7 @@ def _build_network(self):
         self.output_layer = nn.Linear(
             self.backbone.output_dim, self.hparams.output_dim
         )  # output_dim auto-calculated from other config
-        _initialize_layers(self.hparams, self.output_layer)
+        _initialize_layers(self.hparams.activation, self.hparams.initialization, self.output_layer)
 
     def unpack_input(self, x: Dict):
         continuous_data, categorical_data = x["continuous"], x["categorical"]