Error in fine-tuning BERT
As a follow-up from myprevious question, I am trying to fine-tune a model, but I am getting an error:IndexError: tuple index out of range.I am trying to classify individual sentences with a binary classification.  I am usingtransformersversion 4.2.1 anddatasetsversion 1.2.1The dataset(s) are .csv files with two columns: “sentence” and “label”.  The following is the code that led to the error - if anyone can help identify my error, please let me knowimport numpy as np
from transformers import TrainingArguments, Trainer
from transformers import BertTokenizer, BertForSequenceClassification
from datasets import load_dataset, load_metric

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)

dataset = load_dataset('csv', data_files={'train': "train_data.csv",
                                          'test':  "test_data.csv"})

metric = load_metric('f1', 'accuracy')

encoded_dataset = dataset.map(lambda x: tokenizer(x['sentence'], padding=True, truncation=True), batched=True,load_from_cache_file=False)

batch_size = 16
args = TrainingArguments(
    "test_20210201_1200",
    evaluation_strategy = "epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    num_train_epochs=5,
    weight_decay=0.01,
    seed=18,
    label_names='label',
    load_best_model_at_end=True,
    metric_for_best_model='f1',
)

def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    predictions = np.argmax(predictions, axis=1)
    return metric.compute(predictions=predictions, references=labels)

trainer = Trainer(
    model,
    args,
    train_dataset=encoded_dataset['train'],
    eval_dataset=encoded_dataset['test'],
    tokenizer=tokenizer,
    compute_metrics=compute_metrics
)All of that runs with no problem.  However, I get the following error next:trainer.train()---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-2-3435b262f1ae> in <module>
----> 1 trainer.train()

/usr/local/bin/miniconda3/envs/tfhub/lib/python3.8/site-packages/transformers/trainer.py in train(self, model_path, trial)
    933 
    934             self.control = self.callback_handler.on_epoch_end(self.args, self.state, self.control)
--> 935             self._maybe_log_save_evaluate(tr_loss, model, trial, epoch)
    936 
    937             if self.args.tpu_metrics_debug or self.args.debug:

/usr/local/bin/miniconda3/envs/tfhub/lib/python3.8/site-packages/transformers/trainer.py in _maybe_log_save_evaluate(self, tr_loss, model, trial, epoch)
   1002         metrics = None
   1003         if self.control.should_evaluate:
-> 1004             metrics = self.evaluate()
   1005             self._report_to_hp_search(trial, epoch, metrics)
   1006 

/usr/local/bin/miniconda3/envs/tfhub/lib/python3.8/site-packages/transformers/trainer.py in evaluate(self, eval_dataset, ignore_keys, metric_key_prefix)
   1440         start_time = time.time()
   1441 
-> 1442         output = self.prediction_loop(
   1443             eval_dataloader,
   1444             description="Evaluation",

/usr/local/bin/miniconda3/envs/tfhub/lib/python3.8/site-packages/transformers/trainer.py in prediction_loop(self, dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix)
   1569                 losses_host = losses if losses_host is None else torch.cat((losses_host, losses), dim=0)
   1570             if logits is not None:
-> 1571                 preds_host = logits if preds_host is None else nested_concat(preds_host, logits, padding_index=-100)
   1572             if labels is not None:
   1573                 labels_host = labels if labels_host is None else nested_concat(labels_host, labels, padding_index=-100)

/usr/local/bin/miniconda3/envs/tfhub/lib/python3.8/site-packages/transformers/trainer_pt_utils.py in nested_concat(tensors, new_tensors, padding_index)
     83     ), f"Expected `tensors` and `new_tensors` to have the same type but found {type(tensors)} and {type(new_tensors)}."
     84     if isinstance(tensors, (list, tuple)):
---> 85         return type(tensors)(nested_concat(t, n, padding_index=padding_index) for t, n in zip(tensors, new_tensors))
     86     elif isinstance(tensors, torch.Tensor):
     87         return torch_pad_and_concatenate(tensors, new_tensors, padding_index=padding_index)

/usr/local/bin/miniconda3/envs/tfhub/lib/python3.8/site-packages/transformers/trainer_pt_utils.py in <genexpr>(.0)
     83     ), f"Expected `tensors` and `new_tensors` to have the same type but found {type(tensors)} and {type(new_tensors)}."
     84     if isinstance(tensors, (list, tuple)):
---> 85         return type(tensors)(nested_concat(t, n, padding_index=padding_index) for t, n in zip(tensors, new_tensors))
     86     elif isinstance(tensors, torch.Tensor):
     87         return torch_pad_and_concatenate(tensors, new_tensors, padding_index=padding_index)

/usr/local/bin/miniconda3/envs/tfhub/lib/python3.8/site-packages/transformers/trainer_pt_utils.py in nested_concat(tensors, new_tensors, padding_index)
     85         return type(tensors)(nested_concat(t, n, padding_index=padding_index) for t, n in zip(tensors, new_tensors))
     86     elif isinstance(tensors, torch.Tensor):
---> 87         return torch_pad_and_concatenate(tensors, new_tensors, padding_index=padding_index)
     88     elif isinstance(tensors, np.ndarray):
     89         return numpy_pad_and_concatenate(tensors, new_tensors, padding_index=padding_index)

/usr/local/bin/miniconda3/envs/tfhub/lib/python3.8/site-packages/transformers/trainer_pt_utils.py in torch_pad_and_concatenate(tensor1, tensor2, padding_index)
     46 def torch_pad_and_concatenate(tensor1, tensor2, padding_index=-100):
     47     """Concatenates `tensor1` and `tensor2` on first axis, applying padding on the second if necessary."""
---> 48     if len(tensor1.shape) == 1 or tensor1.shape[1] == tensor2.shape[1]:
     49         return torch.cat((tensor1, tensor2), dim=0)
     50 

IndexError: tuple index out of rangeThanks in advance!
2021-02-01T20:36:09Z
Hi@AlanFeder, judging by the stack trace my first guess is that the problem comes from a conflict between padding in thedataset.mapoperation vs padding on-the-fly in theTrainer.As described in theTrainerdocs, when you pass the tokenizer to theTrainerit will be used as follows:The tokenizer used to preprocess the data. If provided, will be used to automatically pad the inputs the maximum length when batching inputs, and it will be saved along the model to make it easier to rerun an interrupted training or reuse the fine-tuned model.So it seems that in your code, you’re doing padding twice: once indataset.mapand then again during training.Can you remove thepadding=Trueargument from your tokenization step and see if that works?
2021-02-01T20:55:07Z
Thanks for the response@lewtun!  Unfortunately, it did not work, and I received the same error with the same stack trace.One thing I forgot to mention – the model seems to have run one training epoch successfully, and then crashed when doing the next step…image1296×274 16 KBNot sure if that makes a difference, or adds any useful context.Thanks again!
2021-02-01T21:40:17Z
Yes, you’re right that the problem is happening on thetrainer.evaluate()step. It might be coming from thelabel_namesargument in yourTrainingArguments. From the docs we have:The list of keys in your dictionary of inputs that correspond to the labels.Will eventually default to["labels"]except if the model used is one of theXxxForQuestionAnsweringin which case it will default to["start_positions", "end_positions"].So it seems you need to provide alistlike['label']instead of the string. If that doesn’t work, you could try renaming the “label” column in your CSV files to “labels” and then dropping thelabel_namesargument fromTrainingArguments.You can then check if it works by just runningtrainer.evaluate()which is faster than waiting for one epoch of trainingAs a tip, I would also specify all the implicit arguments of yourTrainingArgumentsandTrainerexplicitly, e.g. useouput_dir="test_20210201_1200"inTrainingArgumentsand similarly formodelandargsinTrainer.PS. one thing that looks a bit odd is the way you load the metric:metric = load_metric('f1', 'accuracy')I don’t think you can load multiple metrics this way since the second argument refers to the “configuration” of the metric (e.g. GLUE has a config for each task). Nevertheless, this is probably not the source of the problem.
2021-02-02T01:12:47Z
Yes!This worked!  Specifically, I changed the column name (just wrapping label in “[” & “]” did not work).I also made the other changed you suggested.Last question – do you happen to know if there is a way to load multiple accuracy metrics (e.g. both f1 and accuracy)?Thank you so much for all your assistance – my first fine tuning was successful!
2021-02-02T07:47:36Z
Great!I think the simplest way to track both accuracy and F1 score would be to first load them independently:accuracy_score = load_metric('accuracy')
f1_score = load_metric('f1')Then you can include them in thecompute_metricsfunction by simply returning adictof entries:def compute_metrics(pred):
    predictions, labels = eval_pred
    predictions = np.argmax(predictions, axis=1)
    # returns a dict like {'f1':0.54221}
    f1 = f1_score.compute(predictions=predictions, references=labels)
    # returns a dict like {'accuracy': 0.3241}
    acc = accuracy_score.compute(predictions=predictions, references=labels)
    # merge the two dictionaries
    return {**f1, **acc}
2021-02-03T12:57:40Z
Thanks so much for all your help!
2022-02-16T15:53:15Z
I have a similar problem!I am working on a summarization model with Bart.so far I can train the model and it is fine, but when I want to do trainer.evaluate() it returns this warning:“”"VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify ‘dtype=object’ when creating the ndarray.result = getattr(asarray(obj), method)(*args, **kwds)“”"and this error:“”"ValueError: could not broadcast input array from shape (50,32,50265) into shape (50,)“”""50 is the number of rows in my evaluation dataset32 is the max_evaluation_length50265 is BartForConditionalGeneration((model): BartModel((shared): Embedding(50265, 768, padding_idx=1)here is the code:from transformers import AutoTokenizermodel_checkpoint = “facebook/bart-base”tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)max_input_length = 64max_target_length = 32def preprocess_function(example):model_inputs = tokenizer(example['text'],max_length = max_input_length, padding=True , truncation= True) 

with tokenizer.as_target_tokenizer():

    labels = tokenizer(example['summary'], max_length =max_target_length ,padding=True , truncation = True ) 

model_inputs['labels'] = labels['input_ids']

return model_inputstokenized_datasets = dataset.map(preprocess_function, batched= True, remove_columns = [‘text’,‘summary’])from transformers import AutoModelForSeq2SeqLMmodel = AutoModelForSeq2SeqLM.from_pretrained(“facebook/bart-base”)from transformers import TrainingArgumentstraining_args = TrainingArguments(“test_trainer”)from transformers import Trainerimport numpy as npfrom datasets import load_metric#metric= load_metric(“accuracy”)metric = load_metric(“glue”, “mrpc”)def compute_metrics(eval_pred):logits, labels = eval_pred

predictions = np.argmax(logits, axis=-1)

return metric.compute(predictions=predictions, references=labels)trainer = Trainer(model=model,args=training_args,

        train_dataset=full_train_dataset,

        eval_dataset=full_eval_dataset,

        compute_metrics=compute_metrics,

        tokenizer = tokenizer)thenI train the model and it works fine but trainer.evaluate() returns that errorbut it works at the start like this:***** Running Evaluation *****Num examples = 50Batch size = 86%|▋         | 12/189 [13:07<3:13:36, 65.63s/it]33%|███▎      | 62/189 [08:23<17:12,  8.13s/it]then error!
2022-02-21T17:32:55Z
I believe you. can doload_metricas the OP. see reference documentation, you are not limited to 1 metric at a timehuggingface.coFine-tuning a model with the Trainer API - Hugging Face CourseWe’re on a journey to advance and democratize artificial intelligence through open source and open science.