Fine-tuning ECAPA-TDNN
Hi, I am trying to fine-tune the ECAPA-TDNN model, following the recommendations made in the examples for fine-tuning the titanet-large. At some point during training, the loss is infinite and then everything turns into "not a number", including the embeddings. I already clip the gradients, as is done in the example that I follow. Could you check the demo below and tell me what's wrong? Thank you very much.

The packages that I install in the conda environment with python 3.9.16 are:

- torch==2.0.0
- torchvision==0.15.1
- torchaudio==2.0.1
- nemo_toolkit[all]==1.17.0
- resampy==0.4.2
- speechbrain==0.5.14
- openai-whisper==20230314

Here is a minimal dataset with 10 samples that I use to reproduce the error.

[ls-dev-clean-2.txt](https://github.com/NVIDIA/NeMo/files/11770634/ls-dev-clean-2.txt)

And the code:

```python
from omegaconf import OmegaConf
from nemo.collections.asr.models.label_models import EncDecSpeakerLabelModel
import pytorch_lightning as pl
from pytorch_lightning import seed_everything
from torchmetrics import Accuracy
from omegaconf import OmegaConf


seed_everything(42)


def finetune_nemo():
    """
    Fine-tune the model on the given datafiles. Store the resulting model
    in the given experiment folder. The decoder of the model is modified to
    output the number of speakers given in `n_speakers`. Its parameters are
    initialized again, meaning that the decoder is trained from scratch. Save
    the model parameters in the experiment folder.
    """

    datafiles = ["ls-dev-clean-2.txt"]
    n_speakers = 3
    device = "cpu"
    batch_size = 2
    num_workers = 0

    cfg = OmegaConf.create(
        {
            "trainer": {
                "accelerator": "cpu",
                "max_epochs": 5,
                "gradient_clip_val": 1.0,
                "logger": False,
                "deterministic": True,
                "max_steps": -1,
            },
            "optim": {
                "name": "adamw",
                "lr": 0.0001,
                "weight_decay": 0.0002,
                "sched": {
                    "name": "CosineAnnealing",
                    "warmup_ratio": 0.1,
                    "min_lr": 0.0,
                },
            },
        }
    )

    trainer = pl.Trainer(**cfg.trainer)

    model = EncDecSpeakerLabelModel.from_pretrained(model_name="ecapa_tdnn")

    # update training data config
    OmegaConf.set_struct(model.cfg, False)
    model.cfg.train_ds.update(
        {
            "manifest_filepath": datafiles,
            "batch_size": batch_size,
            "num_workers": num_workers,
            "drop_last": True,
        }
    )
    if "noise" in model.cfg.train_ds.augmentor:
        del model.cfg.train_ds.augmentor.noise
    model.setup_training_data(model.cfg.train_ds)

    # initialize decoder with new number of classes
    decoder_cfg = model.cfg.decoder
    decoder_cfg.num_classes = n_speakers
    model.decoder = EncDecSpeakerLabelModel.from_config_dict(decoder_cfg)
    model.decoder.final.bias = None

    # initialize optimizer and update accuracy metric
    model._macro_accuracy = Accuracy(
        task="multiclass", num_classes=n_speakers, average="macro"
    )
    model.setup_optimization(cfg.optim)

    # train model
    model.train()
    trainer.fit(model)
    model.eval()
    model.to(device)

    # save model
    model.save_to("spkid.nemo")


if __name__ == "__main__":
    finetune_nemo()

```