MisconfigurationException: Could not find the LightningModule attribute for the torchmetrics.Metric logged. You can fix this by setting an attribute for the metric in your LightningModule.

[adrian-jaques-b]

Throws this Exception. IDK what to do. Can anyone help me?

Code:

import enum
import os
import torch
import torchmetrics

from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import EarlyStopping


from torch.utils.data import Dataset, DataLoader
import pytorch_lightning as pl

from dataclasses import dataclass, asdict
import pytorch_lightning_spells as pls
import typer
from transformers import T5ForConditionalGeneration, T5Tokenizer

from Cord19Dataset import Cord19Dataset, DATASET_DIR, Parts
from t2t import BaseConfig, T5BaseModel, masked_cross_entropy_loss
from pathlib import Path
import pandas as pd


class Corpus(enum.Enum):
    CORD19 = "cord19"

CACHE_DIR = 

@dataclass
class Config(BaseConfig):
    dataset: Corpus = Corpus.CORD19

from pathlib import Path
import pandas as pd




class T5Model(T5BaseModel):
    def __init__(self, config: Config, **kwargs):
        model = T5ForConditionalGeneration.from_pretrained(config.base_t5_model)
        tokenizer = T5Tokenizer.from_pretrained(config.base_t5_model)
        super().__init__(config, model, tokenizer)
        self.config = config
        # log the config values
        self.save_hyperparameters(asdict(config))
        self.train_dataset = Cord19Dataset(Parts.TRAIN)
        print("Train dataset: ", len(self.train_dataset))
        self.valid_dataset = Cord19Dataset(Parts.VALID)
        print("Valid dataset: ", len(self.valid_dataset))
        
print(Cord19Dataset(Parts.TRAIN))

def main(
        t5_model: str = "t5-small", lr: float = 1e-4,  # 3^4
        epochs: int = 5, fp16: bool = False,
        #dataset: Corpus = Corpus.CORD19, batch_size: int = 16,
        dataset: Corpus = Corpus.CORD19, batch_size: int = 8,
        max_len: int = 64, grad_accu: int = 1,
        num_gpus: int = 1
      
):
    pl.seed_everything(int(os.environ.get("SEED", 738)))
    config = Config(
        base_t5_model=t5_model,
        learning_rate=lr,
        epochs=epochs,
        dataset=dataset,
        max_len=max_len,
        grad_accu=grad_accu,
        batch_size=batch_size,
        fp16=fp16,
        weight_decay=0,
        num_gpus=num_gpus,
        loss_fn=masked_cross_entropy_loss
    )

    pl_module = T5Model(config)

    callbacks = [
        pl.callbacks.ModelCheckpoint(
            dirpath=str(DATASET_DIR / "model_checkpoints"),
            monitor='val_loss',
            mode="min",
            filename='{step:06d}-{val_loss:.4f}',
            save_top_k=1,
            save_last=False
        ),
        pl.callbacks.LearningRateMonitor(logging_interval='step'),
    ]
    trainer = pl.Trainer(
        accelerator='dp' if num_gpus > 1 else None,
        # amp_backend="apex", amp_level='O2',
        precision=16 if config.fp16 else 32,
        gpus=config.num_gpus,
        val_check_interval=0.25,
        gradient_clip_val=10,
        max_epochs=epochs,
        # max_steps=steps,
        callbacks=callbacks,
        accumulate_grad_batches=grad_accu,
        # auto_scale_batch_size='power' if batch_size is None else None,
        logger=[
            pl.loggers.TensorBoardLogger(str(DATASET_DIR / "tb_logs"), name=""),
            pls.loggers.ScreenLogger(),
            # pl.loggers.WandbLogger(project="t5-paraphrase")
        ],
        log_every_n_steps=100
    )

    trainer.fit(pl_module)

#     pl_module.model.save_pretrained(CACHE_DIR / f"{config.base_t5_model}_last")
#     pl_module.tokenizer.save_pretrained(CACHE_DIR / f"{config.base_t5_model}_last")
#     print("Last model saved")

    assert isinstance(callbacks[0], pl.callbacks.ModelCheckpoint)
    print(callbacks[0].best_model_path)
    pl_module = T5Model.load_from_checkpoint(
        callbacks[0].best_model_path,
        config=config
    )
    pl_module.model.save_pretrained(DATASET_DIR / f"{config.base_t5_model}_best")
    pl_module.tokenizer.save_pretrained(DATASET_DIR / f"{config.base_t5_model}_best")
    print("Best model saved")
    pl_module.model.save_pretrained(path+t5+"model/")
    pl_module.tokenizer.save_pretrained(path+t5+"tokenizer/")

This gives me:

Global seed set to 738
Some weights of the model checkpoint at t5-small were not used when initializing T5ForConditionalGeneration: ['decoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weight']
- This IS expected if you are initializing T5ForConditionalGeneration from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing T5ForConditionalGeneration from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
/opt/anaconda/envs/venv/lib/python3.7/site-packages/deprecate/deprecation.py:115: LightningDeprecationWarning: The `Accuracy` was deprecated since v1.3.0 in favor of `torchmetrics.classification.accuracy.Accuracy`. It will be removed in v1.5.0.
  stream(template_mgs % msg_args)
100%|████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 175.62it/s]
100%|█████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 24.88it/s]
Loading train dataset into memory...
Used File: ['dataset_1.jbl', 'dataset_2.jbl', 'dataset_3.jbl', 'dataset_4.jbl', 'dataset_5.jbl', 'dataset_6.jbl']

/opt/anaconda/envs/venv/lib/python3.7/[...]Pretraining-T5-PyTorch-Lightning/model_checkpoints exists and is not empty.
  rank_zero_warn(f"Checkpoint directory {dirpath} exists and is not empty.")

GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [1]
Valid dataset:  124
/opt/anaconda/envs/venv/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:134: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
  "https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate", UserWarning)

  | Name  | Type                       | Params
-----------------------------------------------------
0 | model | T5ForConditionalGeneration | 60.5 M
-----------------------------------------------------
60.5 M    Trainable params
0         Non-trainable params
60.5 M    Total params
242.026   Total estimated model params size (MB)
Steps per epochs: 46
Validation sanity check: 0%
0/2 [00:00<?, ?it/s]
---------------------------------------------------------------------------
MisconfigurationException                 Traceback (most recent call last)
<ipython-input-10-b1e689bd5891> in <module>
      2 # if __name__ == "__main__":
      3 #     typer.run(main)
----> 4 main()

<ipython-input-9-576858e8663d> in main(t5_model, lr, epochs, fp16, dataset, batch_size, max_len, grad_accu, num_gpus)
     56     )
     57 
---> 58     trainer.fit(pl_module)
     59 
     60 #     pl_module.model.save_pretrained(CACHE_DIR / f"{config.base_t5_model}_last")

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloaders, val_dataloaders, datamodule, train_dataloader)
    551         self.checkpoint_connector.resume_start()
    552 
--> 553         self._run(model)
    554 
    555         assert self.state.stopped

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _run(self, model)
    916 
    917         # dispatch `start_training` or `start_evaluating` or `start_predicting`
--> 918         self._dispatch()
    919 
    920         # plugin will finalized fitting (e.g. ddp_spawn will load trained model)

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _dispatch(self)
    984             self.accelerator.start_predicting(self)
    985         else:
--> 986             self.accelerator.start_training(self)
    987 
    988     def run_stage(self):

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/accelerators/accelerator.py in start_training(self, trainer)
     90 
     91     def start_training(self, trainer: "pl.Trainer") -> None:
---> 92         self.training_type_plugin.start_training(trainer)
     93 
     94     def start_evaluating(self, trainer: "pl.Trainer") -> None:

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in start_training(self, trainer)
    159     def start_training(self, trainer: "pl.Trainer") -> None:
    160         # double dispatch to initiate the training loop
--> 161         self._results = trainer.run_stage()
    162 
    163     def start_evaluating(self, trainer: "pl.Trainer") -> None:

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in run_stage(self)
    994         if self.predicting:
    995             return self._run_predict()
--> 996         return self._run_train()
    997 
    998     def _pre_training_routine(self):

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _run_train(self)
   1029             self.progress_bar_callback.disable()
   1030 
-> 1031         self._run_sanity_check(self.lightning_module)
   1032 
   1033         # enable train mode

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in _run_sanity_check(self, ref_model)
   1113             # run eval step
   1114             with torch.no_grad():
-> 1115                 self._evaluation_loop.run()
   1116 
   1117             self.on_sanity_check_end()

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/loops/base.py in run(self, *args, **kwargs)
    109             try:
    110                 self.on_advance_start(*args, **kwargs)
--> 111                 self.advance(*args, **kwargs)
    112                 self.on_advance_end()
    113                 self.iteration_count += 1

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py in advance(self, *args, **kwargs)
    109 
    110         dl_outputs = self.epoch_loop.run(
--> 111             dataloader_iter, self.current_dataloader_idx, dl_max_batches, self.num_dataloaders
    112         )
    113 

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/loops/base.py in run(self, *args, **kwargs)
    109             try:
    110                 self.on_advance_start(*args, **kwargs)
--> 111                 self.advance(*args, **kwargs)
    112                 self.on_advance_end()
    113                 self.iteration_count += 1

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py in advance(self, dataloader_iter, dataloader_idx, dl_max_batches, num_dataloaders)
    109         with self.trainer.profiler.profile("evaluation_step_and_end"):
    110             output = self.evaluation_step(batch, batch_idx, dataloader_idx)
--> 111             output = self.evaluation_step_end(output)
    112 
    113         self.batch_progress.increment_processed()

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py in evaluation_step_end(self, *args, **kwargs)
    159         """Calls the `{validation/test}_step_end` hook"""
    160         hook_name = "test_step_end" if self.trainer.testing else "validation_step_end"
--> 161         output = self.trainer.call_hook(hook_name, *args, **kwargs)
    162         return output
    163 

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in call_hook(self, hook_name, *args, **kwargs)
   1222             if is_overridden(hook_name, model_ref):
   1223                 hook_fx = getattr(model_ref, hook_name)
-> 1224                 output = hook_fx(*args, **kwargs)
   1225 
   1226             # call the accelerator hook

/home/[...]Pretraining-T5-PyTorch-Lightning/t2t/__init__.py in validation_step_end(self, outputs)
    227                 outputs['target']['ids'].view(-1).cpu()
    228             )
--> 229             self.log("val_" + name, metric)
    230 
    231     def _should_log(self, flag):

/opt/anaconda/envs/venv/lib/python3.7/site-packages/pytorch_lightning/core/lightning.py in log(self, name, value, prog_bar, logger, on_step, on_epoch, reduce_fx, tbptt_reduce_fx, tbptt_pad_token, enable_graph, sync_dist, sync_dist_op, sync_dist_group, add_dataloader_idx, batch_size, metric_attribute, rank_zero_only)
    432                 if not self._metric_attributes:
    433                     raise MisconfigurationException(
--> 434                         "Could not find the `LightningModule` attribute for the `torchmetrics.Metric` logged."
    435                         " You can fix this by setting an attribute for the metric in your `LightningModule`."
    436                     )

MisconfigurationException: Could not find the `LightningModule` attribute for the `torchmetrics.Metric` logged. You can fix this by setting an attribute for the metric in your `LightningModule`.

[rohitgr7]

hey @JaquJaqu
can you reformat your code for better readability and also add the exact error with traceback?

[adrian-jaques-b]

Thank you, I didn't notice the chaos.. just wrote my first issue ever :)

[adrian-jaques-b]

ok well i got it. Seems like I accidentally updated the module...