The size of tensor error while fine tuning whisper
Iam using my custom dataset . Now , while running [trainer.train()] , i am getting error as [RuntimeError: The size of tensor a (517) must match the size of tensor b (448) at non-singleton dimension 1]Please note I am using whisper medium model as seen below :from transformers import Seq2SeqTrainingArgumentstraining_args = Seq2SeqTrainingArguments(output_dir=“./whisper-medium-hi”, # change to a repo name of your choiceper_device_train_batch_size=2,gradient_accumulation_steps=16, # increase by 2x for every 2x decrease in batch sizelearning_rate=1e-5,warmup_steps=500,max_steps=4000,gradient_checkpointing=True,fp16=True,evaluation_strategy=“steps”,per_device_eval_batch_size=8,predict_with_generate=True,generation_max_length=225,save_steps=1000,eval_steps=1000,logging_steps=25,report_to=[“tensorboard”],load_best_model_at_end=True,metric_for_best_model=“wer”,greater_is_better=False,push_to_hub=True,)Below is the error stacktrace :use_cache = Trueis incompatible with gradient checkpointing. Settinguse_cache = False…[1001/4000 1:31:27 < 4:34:34, 0.18 it/s, Epoch 1.76/8]Step Training Loss Validation Loss[151/228 11:43 < 06:01, 0.21 it/s]RuntimeError Traceback (most recent call last)in <cell line: 1>()----> 1 trainer.train()15 frames/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)1578 # Disable progress bars when uploading models during checkpoints to avoid polluting stdout1579 REDACTED_TOKEN_utils.disable_progress_bars()→ 1580 return inner_training_loop(1581 args=args,1582 resume_from_checkpoint=resume_from_checkpoint,/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in _inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)1980 self.control = self.callback_handler.on_step_end(args, self.state, self.control)1981→ 1982 self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)1983 else:1984 self.control = self.callback_handler.on_substep_end(args, self.state, self.control)/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in _maybe_log_save_evaluate(self, tr_loss, model, trial, epoch, ignore_keys_for_eval)2324 metrics.update(dataset_metrics)2325 else:→ 2326 metrics = self.evaluate(ignore_keys=ignore_keys_for_eval)2327 self._report_to_hp_search(trial, self.state.global_step, metrics)2328/usr/local/lib/python3.10/dist-packages/transformers/trainer_seq2seq.py in evaluate(self, eval_dataset, ignore_keys, metric_key_prefix, **gen_kwargs)163 self._gen_kwargs = gen_kwargs164→ 165 return super().evaluate(eval_dataset, ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix)166167 def predict(/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in evaluate(self, eval_dataset, ignore_keys, metric_key_prefix)30623063 eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop→ 3064 output = eval_loop(3065 eval_dataloader,3066 description=“Evaluation”,/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in evaluation_loop(self, dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix)32513252 # Prediction step→ 3253 loss, logits, labels = self.prediction_step(model, inputs, prediction_loss_only, ignore_keys=ignore_keys)3254 main_input_name = getattr(self.model, “main_input_name”, “input_ids”)3255 inputs_decode = self._prepare_input(inputs[main_input_name]) if args.include_inputs_for_metrics else None/usr/local/lib/python3.10/dist-packages/transformers/trainer_seq2seq.py in prediction_step(self, model, inputs, prediction_loss_only, ignore_keys, **gen_kwargs)310 if has_labels:311 with self.compute_loss_context_manager():→ 312 outputs = model(**inputs)313 if self.label_smoother is not None:314 loss = self.label_smoother(outputs, inputs[“labels”]).mean().detach()/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)1499 or _global_backward_pre_hooks or _global_backward_hooks1500 or _global_forward_hooks or _global_forward_pre_hooks):→ 1501 return forward_call(*args, **kwargs)1502 # Do not call functions when jit is used1503 full_backward_hooks, non_full_backward_hooks = ,/usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py in forward(*args, **kwargs)634635 def forward(*args, **kwargs):→ 636 return model_forward(*args, **kwargs)637638 # To act like a decorator so that it can be popped when doingextract_model_from_parallel/usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py incall(self, *args, **kwargs)622623 defcall(self, *args, **kwargs):→ 624 return convert_to_fp32(self.model_forward(*args, **kwargs))625626 defgetstate(self):/usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py in decorate_autocast(*args, **kwargs)12 def decorate_autocast(*args, **kwargs):13 with autocast_instance:—> 14 return func(*args, **kwargs)15 decorate_autocast.__script_unsupported = ‘@autocast() decorator is not supported in script mode’ # type: ignore[attr-defined]16 return decorate_autocast/usr/local/lib/python3.10/dist-packages/transformers/models/whisper/modeling_whisper.py in forward(self, input_features, attention_mask, decoder_input_ids, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, decoder_inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)1484 )1485→ 1486 outputs = self.model(1487 input_features,1488 attention_mask=attention_mask,/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)1499 or _global_backward_pre_hooks or _global_backward_hooks1500 or _global_forward_hooks or _global_forward_pre_hooks):→ 1501 return forward_call(*args, **kwargs)1502 # Do not call functions when jit is used1503 full_backward_hooks, non_full_backward_hooks = ,/usr/local/lib/python3.10/dist-packages/transformers/models/whisper/modeling_whisper.py in forward(self, input_features, attention_mask, decoder_input_ids, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, decoder_inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)13601361 # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn)→ 1362 decoder_outputs = self.decoder(1363 input_ids=decoder_input_ids,1364 attention_mask=decoder_attention_mask,/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in _call_impl(self, *args, **kwargs)1499 or _global_backward_pre_hooks or _global_backward_hooks1500 or _global_forward_hooks or _global_forward_pre_hooks):→ 1501 return forward_call(*args, **kwargs)1502 # Do not call functions when jit is used1503 full_backward_hooks, non_full_backward_hooks = ,/usr/local/lib/python3.10/dist-packages/transformers/models/whisper/modeling_whisper.py in forward(self, input_ids, attention_mask, encoder_hidden_states, head_mask, cross_attn_head_mask, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)1124 positions = self.embed_positions(inputs_embeds, past_key_values_length=past_key_values_length)1125→ 1126 hidden_states = inputs_embeds + positions1127 hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)1128RuntimeError: The size of tensor a (517) must match the size of tensor b (448) at non-singleton dimension 1
2024-02-13T08:01:13Z
did you solve it?