Replicating the Fine-Tune Whisper For Multilingual ASR with 🤗 Transformers on MPS
Hello! I’m new to Automatic Speech Recognition and I found this incredibly helpful tutorial toFine-Tune Whisper For Multilingual ASR withTransformers.I’m trying to replicate the tutorial on an M2 Ultra to fine-tune on a language not yet seen by Whisper but which is linguistically similar to about 3 of the 96 languages it was pretained on.I, however, cannot get my model to start training and from what I can tell so far, it seems it’s got something to do with configuring the mps correctly?Below is the script I’m running, the stack trace and the details of my system and the Transformers library version I am using:# Setting up environment

# !pip install --upgrade pip
# !pip install --upgrade datasets[audio] transformers accelerate evaluate jiwer tensorboard gradio

# Importing BembaSpeech dataset that's been uploaded to huggingface private repo.
# Logging in to huggingface

# from huggingface_hub import notebook_login

# notebook_login()

# Importing BembaSpeech dataset that's available locally.
import pandas as pd
from datasets import load_dataset, DatasetDict, Dataset, Audio

csv_file_path = "/Users/user/Desktop/BembaSpeech-master/bem/metadata.csv"
df = pd.read_csv(csv_file_path)

df_dict = df.to_dict(orient='list')
bemba_speech = DatasetDict()

# Type casting audio column from string to audio file
bemba_speech["train"] = Dataset.from_dict(df_dict).cast_column("audio", Audio())

# Performing randomised 90:10 train-test split
bemba_speech = bemba_speech['train'].train_test_split(test_size=0.1)

#Verifying dataset uploaded correctly

print(bemba_speech)
print("Dataset Loaded.")

#Setting up the Whisper Feature Extractor

from transformers import WhisperFeatureExtractor

feature_extractor = WhisperFeatureExtractor.from_pretrained("openai/whisper-small")
print("Feature Extractor Loaded.")

#Setting up the Whisper Tokenizer

from transformers import WhisperTokenizer

tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-small", task="transcribe")
print("Tokenizer Loaded.")

#Verifying that Tokenizer is recognising the language correctly

input_str = bemba_speech["train"][0]["sentence"]
labels = tokenizer(input_str).input_ids
decoded_with_special = tokenizer.decode(labels, skip_special_tokens=False)
decoded_str = tokenizer.decode(labels, skip_special_tokens=True)

print(f"Input:                 {input_str}")
print(f"Decoded w/ special:    {decoded_with_special}")
print(f"Decoded w/out special: {decoded_str}")
print(f"Are equal:             {input_str == decoded_str}")

# Note: Closest Bantu languages to Bemba: Swahili, Shona and Lingala

from transformers import WhisperProcessor

processor = WhisperProcessor.from_pretrained("openai/whisper-small", task="transcribe")
print("Whisper Processor Set Up.")
print(bemba_speech["train"][0])


def prepare_dataset(batch):
    # load and resample audio data from 48 to 16kHz
    audio = batch["audio"]

    # compute log-Mel input features from input audio array
    batch["input_features"] = feature_extractor(audio["array"], sampling_rate=audio["sampling_rate"]).input_features[0]

    # encode target text to label ids
    batch["labels"] = tokenizer(batch["sentence"]).input_ids
    return batch


bemba_speech = bemba_speech.map(prepare_dataset, remove_columns=bemba_speech.column_names["train"], num_proc=1)

from transformers import WhisperForConditionalGeneration

model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small")

model.generation_config.task = "transcribe"

model.generation_config.forced_decoder_ids = None
print("Whisper Model Loaded.")

import torch

from dataclasses import dataclass
from typing import Any, Dict, List, Union

# device = torch.device("mps")
# model.to(device)

# print(f"Model is on device: {next(model.parameters()).device}")

@dataclass
class DataCollatorSpeechSeq2SeqWithPadding:
    processor: Any
    decoder_start_token_id: int

    def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:
        # split inputs and labels since they have to be of different lengths and need different padding methods
        # first treat the audio inputs by simply returning torch tensors
        input_features = [{"input_features": feature["input_features"]} for feature in features]
        batch = self.processor.feature_extractor.pad(input_features, return_tensors="pt")

        # get the tokenized label sequences
        label_features = [{"input_ids": feature["labels"]} for feature in features]
        # pad the labels to max length
        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors="pt")

        # print(f"input_features type: {type(input_features)}, shape: {[len(f) for f in input_features]}")
        # print(f"label_features type: {type(label_features)}, shape: {[len(f['input_ids']) for f in label_features]}")
        # print(f"Padded batch input_features type: {type(batch['input_features'])}, shape: {batch['input_features'].shape}")
        # print(f"Padded batch labels_batch type: {type(labels_batch['input_ids'])}, shape: {labels_batch['input_ids'].shape}")

        # replace padding with -100 to ignore loss correctly
        labels = labels_batch["input_ids"].masked_fill(labels_batch.attention_mask.ne(1), -100)

        # print(f"Labels after masked_fill type: {type(labels)}, shape: {labels.shape}")

        # if bos token is appended in previous tokenization step,
        # cut bos token here as it's append later anyways
        if (labels[:, 0] == self.decoder_start_token_id).all().cpu().item():
            labels = labels[:, 1:]

        batch["labels"] = labels

        return batch

data_collator = DataCollatorSpeechSeq2SeqWithPadding(
    processor=processor,
    decoder_start_token_id=model.config.decoder_start_token_id,
)

import evaluate

metric = evaluate.load("wer")

def compute_metrics(pred):
    pred_ids = pred.predictions
    label_ids = pred.label_ids

    # replace -100 with the pad_token_id
    label_ids[label_ids == -100] = tokenizer.pad_token_id

    # we do not want to group tokens when computing the metrics
    pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True)
    label_str = tokenizer.batch_decode(label_ids, skip_special_tokens=True)

    wer = 100 * metric.compute(predictions=pred_str, references=label_str)

    return {"wer": wer}

from transformers import Seq2SeqTrainingArguments

training_args = Seq2SeqTrainingArguments(
    output_dir="./whisper-small-bem",  # change to a repo name of your choice
    per_device_train_batch_size=16,
    gradient_accumulation_steps=1,  # increase by 2x for every 2x decrease in batch size
    learning_rate=1e-5,
    warmup_steps=500,
    max_steps=5000,
    gradient_checkpointing=True,
    fp16=False,
    evaluation_strategy="steps",
    per_device_eval_batch_size=8,
    predict_with_generate=True,
    generation_max_length=225,
    save_steps=1000,
    eval_steps=1000,
    logging_steps=25,
    report_to=["tensorboard"],
    load_best_model_at_end=True,
    metric_for_best_model="wer",
    greater_is_better=False,
    push_to_hub=True,
)

from transformers import Seq2SeqTrainer

trainer = Seq2SeqTrainer(
    args=training_args,
    model=model,
    train_dataset=bemba_speech["train"],
    eval_dataset=bemba_speech["test"],
    data_collator=data_collator,
    compute_metrics=compute_metrics,
    tokenizer=processor.feature_extractor,
)
print("Training Arguments Defined.")

trainer.train()/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/transformers/training_args.py:1494: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(
Training Arguments Defined.
max_steps is given, it will override any value given in num_train_epochs
  0%|          | 0/5000 [00:00<?, ?it/s]Traceback (most recent call last):
  File "/Users/user/PycharmProjects/pythonProject/.venv/whisperbem.py", line 198, in <module>
    trainer.train()
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/transformers/trainer.py", line 1923, in train
    return inner_training_loop(
           ^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/transformers/trainer.py", line 2268, in _inner_training_loop
    tr_loss_step = self.training_step(model, inputs)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/transformers/trainer.py", line 3307, in training_step
    loss = self.compute_loss(model, inputs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/transformers/trainer.py", line 3338, in compute_loss
    outputs = model(**inputs)
              ^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/transformers/models/whisper/modeling_whisper.py", line 1753, in forward
    outputs = self.model(
              ^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/transformers/models/whisper/modeling_whisper.py", line 1611, in forward
    encoder_outputs = self.encoder(
                      ^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/transformers/models/whisper/modeling_whisper.py", line 1174, in forward
    inputs_embeds = nn.functional.gelu(self.conv1(input_features))
                                       ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/conv.py", line 310, in forward
    return self._conv_forward(input, self.weight, self.bias)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/PycharmProjects/pythonProject/.venv/lib/python3.12/site-packages/torch/nn/modules/conv.py", line 306, in _conv_forward
    return F.conv1d(input, weight, bias, self.stride,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Mismatched Tensor types in NNPack convolutionOutput
  0%|          | 0/5000 [00:01<?, ?it/s]- `transformers` version: 4.42.4
- Platform: macOS-14.5-arm64-arm-64bit
- Python version: 3.12.0
- Huggingface_hub version: 0.23.5
- Safetensors version: 0.4.3
- Accelerate version: 0.32.1
- Accelerate config:    not found
- PyTorch version (GPU?): 2.3.1 (False)
- Tensorflow version (GPU?): not installed (NA)
- Flax version (CPU?/GPU?/TPU?): not installed (NA)
- Jax version: not installed
- JaxLib version: not installed
- Using distributed or parallel set-up in script?: <fill in>I tried to run this first in Google Colab and I got to training with no major issues. The only reason I cannot complete this fine-tuning exercise in Colab is because only the free version is available in my region and the session typically times out at around 3 hours.Thank you so much for your time!