Visualbert config.json
I am doing visualBERT VQA evaluation but the target tensor is always 0. Could share the pytorch dataset for local files or update the config. json? Thank youHere is my code:===========================================================================================Read questionsimport jsonChecking path and Opening JSON file,f_read_questions = open(‘./vqa2/v2_OpenEnded_mscoco_val2014_questions.json’)Return JSON object as dictionarydata_questions = json.load(f_read_questions)print(data_questions.keys())questions = data_questions[‘questions’]print(“Number of questions:”, len(questions))from os import listdirfrom os.path import isfile, joinroot at which all images are storedroot = ‘./multimodal_data/vqa2/val2014’file_names = [f for f in listdir(root) if isfile(join(root, f))]import refrom typing import Optionalfilename_re = re.compile(r".*(\d{12}).((jpg)|(png))")source:https://github.com/allenai/allennlp-models/blob/a36aed540e605c4293c25f73d6674071ca9edfc3/allennlp_models/vision/dataset_readers/vqav2.py#L141def id_from_filename(filename: str) → Optional[int]:match = filename_re.fullmatch(filename)if match is None:return Nonereturn int(match.group(1))filename_to_id = {root + “/” + file: id_from_filename(file) for file in file_names}id_to_filename = {v: k for k, v in filename_to_id.items()}from os import listdirfrom os.path import isfile, joinroot at which all images are storedroot = ‘./multimodal_data/vqa2/val2014’file_names = [f for f in listdir(root) if isfile(join(root, f))]Read annotationsf_read_annotations = open(“./vqa2/v2_mscoco_val2014_annotations.json”)Return JSON object as dictionarydata_annotations = json.load(f_read_annotations)print(data_annotations.keys())show answwersannotations = data_annotations[‘annotations’]print(“Number of annotations:”, len(annotations))from transformers import VisualBertConfig, VisualBertModelconfig = VisualBertConfig.from_pretrained(“./pretrained/visualBERT/visualbert-vqa”)from tqdm.notebook import tqdmdef get_score(count: int) → float:return min(1.0, count / 3)for annotation in tqdm(annotations):answers = annotation[‘answers’]answer_count = {}for answer in answers:answer_ = answer[“answer”]answer_count[answer_] = answer_count.get(answer_, 0) + 1labels =scores =for answer in answer_count:if answer not in list(config.label2id.keys()):continuelabels.append(config.label2id[answer])score = get_score(answer_count[answer])scores.append(score)annotation[‘labels’] = labelsannotation[‘scores’] = scoresclass VQADataset(torch.utils.data.Dataset):“”“VQA (v2) dataset.”“”def __init__(self, questions, annotations, image_preprocess):
    self.questions = questions
    self.annotations = annotations
    self.image_preprocess = image_preprocess

def __len__(self):
    return len(self.annotations)

def __getitem__(self, idx):
    # get image + text
    annotation = self.annotations[idx]
    questions = self.questions[idx]
    image = id_to_filename[annotation['image_id']]
    text = questions['question']

    inputs = tokenizer(
                text,
                padding="max_length",
                max_length=40,
                truncation=True,
                return_token_type_ids=True,
                return_attention_mask=True,
                add_special_tokens=True,
                return_tensors="pt",
            )

     #faster-rcnn
    images, sizes, scales_yx = self.image_preprocess(image)
    output_dict = frcnn(
        images,
        sizes,
        scales_yx=scales_yx,
        padding="max_detections",
        max_detections=frcnn_cfg.max_detections,
        return_tensors="pt",
    )

    features = output_dict.get("roi_features")

    inputs.update(
        {"input_ids": inputs.input_ids,
         "attention_mask": inputs.attention_mask,
         "token_type_ids": inputs.token_type_ids,
         "visual_embeds": features,
         "visual_attention_mask": torch.ones(features.shape[:-1], dtype=torch.float),
         "visual_token_type_ids": torch.ones(features .shape[:-1], dtype=torch.long),
         # "output_attentions": False
         }
    )

  # remove batch dimension
    for k, v in inputs.items():
        inputs[k] = v.squeeze()

   # add labels
    labels = annotation['labels']
    scores = torch.tensor(annotation['scores'])

    targets = torch.zeros(len(config.id2label), dtype=torch.float)
    for label, score in zip(labels, scores):
        targets[label] = score
    inputs["labels"] = targets # get the index of the highest score as target

    # based on: https://github.com/dandelin/ViLT/blob/762fd3975c180db6fc88f577cf39549983fa373a/vilt/modules/objectives.py#L301
    return inputsifname== ‘main’:from visual_bert.processing_image import Preprocessfrom visual_bert.visualizing_image import SingleImageVizfrom visual_bert.modeling_frcnn import GeneralizedRCNNfrom visual_bert.utils import Configfrcnn_cfg = Config.from_pretrained("unc-nlp/frcnn-vg-finetuned")
frcnn = GeneralizedRCNN.from_pretrained("unc-nlp/frcnn-vg-finetuned", config=frcnn_cfg)
image_preprocess = Preprocess(frcnn_cfg)

from transformers import VisualBertForQuestionAnswering, BertTokenizerFast
tokenizer = BertTokenizerFast.from_pretrained("./pretrained/bert-base-uncased")
model = VisualBertForQuestionAnswering.from_pretrained("./pretrained/visualBERT/visualbert-vqa")

dataset = VQADataset(questions=questions[:100],
                     annotations=annotations[:100],
                     image_preprocess=image_preprocess)

test_dataloader = DataLoader(dataset, batch_size=1, shuffle=False)

correct = 0.0
total = 0

loss_function = nn.CrossEntropyLoss()
# optimizer = optim.Adam(model.classifier.parameters(), lr=0.001)

for batch in tqdm(test_dataloader):
    # with torch.no_grad():
   
        batch = {k: v.to(device) for k, v in batch.items()}
        # here is the probelm
        print(batch["labels"])

        # forward pass
        outputs = model(**batch)
        logits= outputs.logits
        _, pre = torch.max(logits.data, 1)
        _, target = torch.max(batch["labels"].data, 1)
        print("prediction:", pre)
        print("target:", target)

        correct += (pre == target).sum().item()
        total = total + 1
        print(total)
        print("==============================================================")

final_acc = correct / float(len(test_dataloader.dataset))
print('Accuracy of test: %f %%' % (100 * float(final_acc)))