Qwen

/

Qwen2.5-VL-72B-Instruct

like 316

Follow

Qwen 16.3k

Hi, when I used batch inference for annotating, I encountered this error:

Traceback (most recent call last):
  File "/open_sourced_model/code/qwen-vl-2.5/annotation_code/batch_annotation.py", line 152, in <module>
    main()
  File "/open_sourced_model/code/qwen-vl-2.5/annotation_code/batch_annotation.py", line 148, in main
    batch_annotate_with_vlm(args.input_file, args.output_file, args.model_name, args.batch_size, args.max_new_tokens)
  File "/open_sourced_model/code/qwen-vl-2.5/annotation_code/batch_annotation.py", line 115, in batch_annotate_with_vlm
    batch_results = inference_batch(batch_image_paths, batch_prompts, model, processor, max_new_tokens)
  File "/open_sourced_model/code/qwen-vl-2.5/annotation_code/batch_annotation.py", line 72, in inference_batch
    generated_ids = model.generate(**inputs, max_new_tokens=max_new_tokens)
  File "/home/ma-user/anaconda3/envs/PyTorch-2.0.0/lib/python3.9/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "/open_sourced_model/code/transformers-main/src/transformers/generation/utils.py", line 2227, in generate
    result = self._sample(
  File "/open_sourced_model/code/transformers-main/src/transformers/generation/utils.py", line 3215, in _sample
    outputs = self(**model_inputs, return_dict=True)
  File "/home/ma-user/anaconda3/envs/PyTorch-2.0.0/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
  File "/home/ma-user/anaconda3/envs/PyTorch-2.0.0/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/ma-user/anaconda3/envs/PyTorch-2.0.0/lib/python3.9/site-packages/accelerate/hooks.py", line 166, in new_forward
    output = module._old_forward(*args, **kwargs)
  File "/open_sourced_model/code/transformers-main/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py", line 1799, in forward
    raise ValueError(
ValueError: Image features and image tokens do not match: tokens: 0, features 5040

The code is like:

import os
import json
import time
import torch
from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor
from qwen_vl_utils import process_vision_info
import warnings
import argparse
import logging

warnings.filterwarnings("ignore")
# Set up logging configuration
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)


def get_model_and_processor(model_dir):
    """
    Load the VLM model and processor.
    """
    processor = AutoProcessor.from_pretrained(model_dir)
    model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
        model_dir, torch_dtype="auto", device_map="auto"
    )
    return model, processor


def inference_batch(image_paths, prompts, model, processor, max_new_tokens):
    """
    Perform batch inference for a list of image paths and corresponding prompts.

    Args:
      image_paths: List of image file paths.
      prompts: List of text prompts corresponding to each image.
      model: Loaded VLM model.
      processor: Loaded processor.
      max_new_tokens:

    Returns:
      List of model-generated output texts for each image-text pair in the batch.
    """
    # Prepare the messages in the required format for batch inference
    messages_batch = [
        {
            "role": "user",
            "content": [
                {"type": "image", "image": image_path},
                {"type": "text", "text": prompt},
            ],
        }
        for image_path, prompt in zip(image_paths, prompts)
    ]

    # Prepare the input for the processor
    texts = [
        processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        for messages in messages_batch
    ]

    image_inputs, video_inputs = process_vision_info(messages_batch)
    inputs = processor(
        text=texts,
        images=image_inputs,
        videos=video_inputs,
        padding=True,
        return_tensors="pt",
    )

    inputs = inputs.to("cuda")

    # Perform batch inference for all images
    generated_ids = model.generate(**inputs, max_new_tokens=max_new_tokens)
    generated_ids_trimmed = [
        out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
    ]
    output_texts = processor.batch_decode(
        generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
    )

    return output_texts

How can I fix this bug?