Finetuning Donut Transformer on DocParsing
Hi everyone,I am trying to finetune the donut transformer on docparsing for a batch commercial invoice documents that are used in the banking sector. However; I am constantly receiving the below error, even after a significant amount of research. If you can direct me about how to solve the error, I will be more than thankful.I tried to change the arguments of the trainer, data collator, and training args objects. I also tried to change the DataCollatorWithPadding() object to DefaultDataCollator() after receiving an error about padding. Moreover, for constructing the tokenizer object, I also tried to use AutoTokenizer.fromPretrained() instead of processor.tokenizer(). Lastly, I checked the HuggingFace documentation for Donut Transformer. However, nothing changed. I continue to receive the below error.Best regards,BarışThe code that I have written:import reimport sysfrom PIL import Imageimport torchimport osimport sslimport globfrom datasets import load_datasetfrom transformers import DonutProcessor, VisionEncoderDecoderModel, AutoTokenizer, AutoFeatureExtractor, AutoFeatureExtractorfrom transformers import TrainingArguments, Trainer, DefaultDataCollatorDisabling SSL verification and configuring environment variablesssl._create_default_https_context = ssl._create_unverified_contextos.environ[“REQUESTS_CA_BUNDLE”] = “/etc/ssl/certs/ca-bundle.trust.crt”os.environ[“TF_ENABLE_ONEDNN_OPTS”] = “0”def fine_tune_donut_model():# Load the pre-trained Donut processor and model as a base for fine-tuningprocessor = DonutProcessor.from_pretrained(“naver-clova-ix/donut-base”)model = VisionEncoderDecoderModel.from_pretrained(“naver-clova-ix/donut-base”)# Inside the `fine_tune_donut_model` function, after loading the dataset
dataset = load_dataset("/home/t098317/donut_directory/donut/commercial_invoice")

# Load a pretrained tokenizer for text data
# tokenizer = AutoTokenizer.from_pretrained("naver-clova-ix/donut-base")

task_prompt = "<s_cord-v2>"
tokenizer = processor.tokenizer(task_prompt, add_special_tokens=False, return_tensors="pt")
print('the tokenizer is: '+str(tokenizer)+'')

# Check if a CUDA-enabled GPU is available, otherwise use CPU
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)

# Define your fine-tuning hyperparameters and training arguments
training_args = TrainingArguments(
    output_dir="/home/t098317/donut_directory/donut/finetuned_donut",
    per_device_train_batch_size=8,
    num_train_epochs=3,
    evaluation_strategy="steps",
    save_steps=10_000,
    remove_unused_columns=False,
    learning_rate=0.0001
)

print('The training args are: '+str(training_args)+'')
print()
print()

# Create a data collator for your training data
data_collator = DefaultDataCollator(
    return_tensors="pt"
)

print('The data collator is: '+str(data_collator)+'')
print()
print()

trainer = Trainer(
    tokenizer=tokenizer,
    model=model,
    args=training_args,
    data_collator=data_collator,  
    train_dataset=dataset["train"]
)

print('The trainer is: '+str(trainer)+'')
print()
print()

# Start the fine-tuning process
try:
# Start the fine-tuning process
    trainer.train()
except Exception as e:
    print(f"Error during training: {str(e)}")

print('abc')

# Save the fine-tuned model in the specified directory
model.save_pretrained("/home/t098317/donut_directory/donut/finetuned_donut")Function to extract information from all images in a directorydef extract_info_from_images_in_directory(directory_path, task_name=“cord-v2”):# Load the fine-tuned Donut processor and modelprocessor = DonutProcessor.from_pretrained(“/home/t098317/donut_directory/donut/finetuned_donut”)model = VisionEncoderDecoderModel.from_pretrained(“/home/t098317/donut_directory/donut/finetuned_donut”)# Check if a CUDA-enabled GPU is available, otherwise use CPU
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)

# Get a list of image files in the specified directory
image_extensions = ['*.jpg', '*.png']

# Use glob to find all image files with the specified extensions
image_files = []
for ext in image_extensions:
    image_files.extend(glob.glob(os.path.join(directory_path, ext)))

for image_path in image_files:
    print(image_path)
    # Load the image using PIL (Python Imaging Library)
    input_img = Image.open(image_path)

    # Prepare the image input for Donut
    pixel_values = processor(input_img, return_tensors="pt")

    # Prepare decoder inputs
    task_prompt = f"<s_{task_name}>"
    decoder_input_ids = processor.tokenizer(task_prompt, add_special_tokens=False, return_tensors="pt").input_ids

    # Generate text from the image and decoder inputs
    outputs = model.generate(
        pixel_values.to(device),
        input_ids=decoder_input_ids.to(device),  # Use input_ids instead of decoder_input_ids
        pad_token_id=processor.tokenizer.pad_token_id,
        eos_token_id=processor.tokenizer.eos_token_id,
        use_cache=True,
        max_new_tokens=700,
        bad_words_ids=[[processor.tokenizer.unk_token_id]],
        return_dict_in_generate=True,
        decoder_start_token_id=model.config.decoder_start_token_id,  # Add this line if necessary
    )
    
    print('output: '+str(outputs)+'')


    # Decode the generated sequence
    sequence = processor.batch_decode(outputs.sequences)[0]
    sequence = sequence.replace(processor.tokenizer.eos_token, "").replace(processor.tokenizer.pad_token, "")
    sequence = re.sub(r"<.*?>", "", sequence, count=1).strip()  # Remove the first task start token

    # Print the extracted information
    print(f"Information from {image_path}:")
    print(processor.token2json(sequence))
    print()ifname== “main”:# Execute the fine-tuning functionfine_tune_donut_model()# Specify the directory containing the images you want to extract information from
image_directory_path = "/home/t098317/donut_directory/donut/commercial_invoice"

# Execute the information extraction function for all images in the directory with task name "cord-v2"
extract_info_from_images_in_directory(image_directory_path, task_name="cord-v2")The error I am receiving:Could not find image processor class in the image processor config or the model config. Loading based on pattern matching with the model’s feature extractor configuration.the tokenizer is: {‘input_ids’: tensor([[41040, 46192, 41403, 36697, 52165, 37093, 35934, 34791]]), ‘attention_mask’: tensor([[1, 1, 1, 1, 1, 1, 1, 1]])}The training args are: TrainingArguments(_n_gpu=3,adafactor=False,adam_beta1=0.9,adam_beta2=0.999,adam_epsilon=1e-08,auto_find_batch_size=False,bf16=False,bf16_full_eval=False,data_seed=None,dataloader_drop_last=False,dataloader_num_workers=0,dataloader_pin_memory=True,ddp_backend=None,ddp_broadcast_buffers=None,ddp_bucket_cap_mb=None,ddp_find_unused_parameters=None,ddp_timeout=1800,debug=,deepspeed=None,disable_tqdm=False,dispatch_batches=None,do_eval=True,do_predict=False,do_train=False,eval_accumulation_steps=None,eval_delay=0,eval_steps=500,evaluation_strategy=steps,fp16=False,fp16_backend=auto,fp16_full_eval=False,fp16_opt_level=O1,fsdp=,fsdp_config={‘min_num_params’: 0, ‘xla’: False, ‘xla_fsdp_grad_ckpt’: False},fsdp_min_num_params=0,fsdp_transformer_layer_cls_to_wrap=None,full_determinism=False,gradient_accumulation_steps=1,gradient_checkpointing=False,greater_is_better=None,group_by_length=False,half_precision_backend=auto,hub_always_push=False,hub_model_id=None,hub_private_repo=False,hub_strategy=every_save,hub_token=<HUB_TOKEN>,ignore_data_skip=False,include_inputs_for_metrics=False,include_tokens_per_second=False,jit_mode_eval=False,label_names=None,label_smoothing_factor=0.0,learning_rate=0.0001,length_column_name=length,load_best_model_at_end=False,local_rank=0,log_level=passive,log_level_replica=warning,log_on_each_node=True,logging_dir=/home/t098317/donut_directory/donut/finetuned_donut/runs/Oct23_14-07-30_ovrargegpudev1,logging_first_step=False,logging_nan_inf_filter=True,logging_steps=500,logging_strategy=steps,lr_scheduler_type=linear,max_grad_norm=1.0,max_steps=-1,metric_for_best_model=None,mp_parameters=,no_cuda=False,num_train_epochs=3,optim=adamw_torch,optim_args=None,output_dir=/home/t098317/donut_directory/donut/finetuned_donut,overwrite_output_dir=False,past_index=-1,per_device_eval_batch_size=8,per_device_train_batch_size=8,prediction_loss_only=False,push_to_hub=False,push_to_hub_model_id=None,push_to_hub_organization=None,push_to_hub_token=<PUSH_TO_HUB_TOKEN>,ray_scope=last,remove_unused_columns=False,report_to=[‘tensorboard’],resume_from_checkpoint=None,run_name=/home/t098317/donut_directory/donut/finetuned_donut,save_on_each_node=False,save_safetensors=False,save_steps=10000,save_strategy=steps,save_total_limit=None,seed=42,sharded_ddp=,skip_memory_metrics=True,tf32=None,torch_compile=False,torch_compile_backend=None,torch_compile_mode=None,torchdynamo=None,tpu_metrics_debug=False,tpu_num_cores=None,use_cpu=False,use_ipex=False,use_legacy_prediction_loop=False,use_mps_device=False,warmup_ratio=0.0,warmup_steps=0,weight_decay=0.0,)The data collator is: DefaultDataCollator(return_tensors=‘pt’)The trainer is: <transformers.trainer.Trainer object at 0x7f4d26040ac0>0%|                                                                                                          | 0/3 [00:00<?, ?it/s]Error during training: Could not infer dtype of JpegImageFileabc0%|                                                                                                          | 0/3 [00:03<?, ?it/s]Traceback (most recent call last):File “/home/t098317/donut_directory/donut/finetune_donut_dp.py”, line 149, inextract_info_from_images_in_directory(image_directory_path, task_name=“cord-v2”)File “/home/t098317/donut_directory/donut/finetune_donut_dp.py”, line 88, in extract_info_from_images_in_directoryprocessor = DonutProcessor.from_pretrained(“/home/t098317/donut_directory/donut/finetuned_donut”)File “/storage/miniconda/envs/yagmur_donut/lib/python3.9/site-packages/transformers/processing_utils.py”, line 226, in from_pretrainedargs = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs)File “/storage/miniconda/envs/yagmur_donut/lib/python3.9/site-packages/transformers/processing_utils.py”, line 270, in _get_arguments_from_pretrainedargs.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs))File “/storage/miniconda/envs/yagmur_donut/lib/python3.9/site-packages/transformers/models/auto/image_processing_auto.py”, line 346, in from_pretrainedconfig_dict, _ = ImageProcessingMixin.get_image_processor_dict(pretrained_model_name_or_path, **kwargs)File “/storage/miniconda/envs/yagmur_donut/lib/python3.9/site-packages/transformers/image_processing_utils.py”, line 329, in get_image_processor_dictresolved_image_processor_file = cached_file(File “/storage/miniconda/envs/yagmur_donut/lib/python3.9/site-packages/transformers/utils/hub.py”, line 400, in cached_fileraise EnvironmentError(OSError: /home/t098317/donut_directory/donut/finetuned_donut does not appear to have a file named preprocessor_config.json. Checkout ‘https://huggingface.co//home/t098317/donut_directory/donut/finetuned_donut/main’ for available files.