ConceptMetrics: metrics refactor + GroupConfig: flexible/modular init of metrics and losses.

Author: gdefe

conceptarium/conf/_default.yaml

@@ -1,7 +1,8 @@
 defaults:
   - dataset: asia
   - model: cbm_joint
-  - loss: _default
+  - loss: standard
+  - metrics: standard
   - _self_
 
 # =============================================================

conceptarium/conf/loss/_default.yaml

@@ -0,0 +1,13 @@
+# =============================================================
+# Loss settings
+# =============================================================
+_target_: "torch_concepts.nn.ConceptLoss"
+
+fn_collection:
+  _target_: "torch_concepts.nn.modules.utils.GroupConfig"
+  binary: 
+    _target_: "torch.nn.BCEWithLogitsLoss"
+  categorical: 
+    _target_: "torch.nn.CrossEntropyLoss"
+  # continuous: 
+    # ... not supported yet

conceptarium/conf/loss/standard.yaml

@@ -1,15 +1,16 @@
+# =============================================================
+# Loss settings
+# =============================================================
 _target_: "torch_concepts.nn.WeightedConceptLoss"
 
 weight: 0.8 # weight applied to concepts, (1-weight) applied to task
-task_names: ${model.task_names}
-fn_collection:
-  discrete:
-    binary: 
-      path: "torch.nn.BCEWithLogitsLoss"
-      kwargs: {}
-    categorical: 
-      path: "torch.nn.CrossEntropyLoss"
-      kwargs: {}
+task_names: ${dataset.default_task_names}
 
+fn_collection:
+  _target_: "torch_concepts.nn.modules.utils.GroupConfig"
+  binary: 
+    _target_: "torch.nn.BCEWithLogitsLoss"
+  categorical: 
+    _target_: "torch.nn.CrossEntropyLoss"
   # continuous: 
-    # ... not supported yet
+    # ... not supported yet

conceptarium/conf/loss/weighted.yaml

@@ -0,0 +1,24 @@
+# =============================================================
+# Metrics settings
+# =============================================================
+_target_: "torch_concepts.nn.ConceptMetrics"
+
+# tracking of summary metrics for each concept type
+summary_metrics: true
+# tracking of metrics for each individual concept 
+# `true` for all concepts, list of concept names, or `false` for none
+# ${dataset.default_task_names} for tracking tasks individually
+perconcept_metrics: true
+
+fn_collection:
+  _target_: "torch_concepts.nn.modules.utils.GroupConfig"
+  binary: 
+    accuracy: 
+      _target_: "torchmetrics.classification.BinaryAccuracy"
+  categorical: 
+    accuracy:
+      - _target_: "hydra.utils.get_class"
+        path: "torchmetrics.classification.MulticlassAccuracy"
+      - average: 'micro'
+  # continuous: 
+    # ... not supported yet

conceptarium/conf/metrics/standard.yaml

@@ -1,5 +1,4 @@
 defaults:
-  - metrics: _default
   - _self_
 
 
@@ -53,16 +52,10 @@ optim_kwargs:
 #   factor: 0.2
 
 
+# TODO: implement this
 # =============================================================
-# Metrics settings
+# Training settings
 # =============================================================
-# tracking of summary metrics for each concept type
-summary_metrics: true
-# tracking of metrics for each individual concept 
-# `true` for all concepts, list of concept names, or `false` for none
-perconcept_metrics: ${dataset.default_task_names}
-
-# TODO: implement this
 # train_interv_prob: 0.1
 # test_interv_policy: nodes_true # levels_true, levels_pred, nodes_true, nodes_pred, random
-# test_interv_noise: 0.
+# test_interv_noise: 0.

conceptarium/conf/model/_commons.yaml

@@ -0,0 +1,11 @@
+defaults:
+  - _commons
+  - _self_
+  
+_target_: "torch_concepts.nn.ConceptBottleneckModel_Independent"
+
+task_names: ${dataset.default_task_names}
+
+inference: 
+  _target_: "torch_concepts.nn.DeterministicInference"
+  _partial_: true

conceptarium/conf/model/cbm_indep.yaml

@@ -9,23 +9,22 @@ hydra:
     # standard grid search
     params:
       seed: 1
-      dataset: asia
+      dataset: asia, sachs, insurance
       model: cbm
-      # loss: weighted
-      # loss.weight: 0.99
+      #loss: standard, weighted
 
 model:
-  summary_metrics: true
-  perconcept_metrics: true # or ${dataset.default_task_names}
-  # train_interv_prob: 0.8
-  # test_interv_noise: 0.8   # for bndatasets only
   optim_kwargs:
-    lr: 0.001
+    lr: 0.01
+
+metrics:
+  summary_metrics: true
+  perconcept_metrics: true #${dataset.default_task_names}
 
 trainer:
   logger: null
   max_epochs: 200
-  patience: 30
+  patience: 20
 
 matmul_precision: medium
 

conceptarium/conf/model/metrics/_default.yaml

@@ -44,7 +44,10 @@ def main(cfg: DictConfig) -> None:
     # ----------------------------------
     logger.info("----------------------INIT MODEL-------------------------------------")
     loss = instantiate(cfg.loss, annotations=datamodule.annotations, _convert_="all")
-    model = instantiate(cfg.model, annotations=datamodule.annotations, loss=loss, _convert_="all")
+    logger.info(loss)
+    metrics = instantiate(cfg.metrics, annotations=datamodule.annotations, _convert_="all")
+    logger.info(metrics)
+    model = instantiate(cfg.model, annotations=datamodule.annotations, loss=loss, metrics=metrics, _convert_="all")
 
     logger.info("----------------------BEGIN TRAINING---------------------------------")
     try: