TypeError: new_full(): argument ‘fill_value’ (position 2) must be Number, not NoneType
Hello huggers! I am trying to pretrain a GPT2 model from scratch for casual language modeling (given a sequence of words, whats the next most likely word) and I am new to both Pytorch Lightning and the amazing Hugging Face libraries!When I try to train aGPT2LMHeadModelfrom scratch using the code provided below, I get an errorTypeError: new_full(): argument ‘fill_value’ (position 2) must be Number, not NoneTypeI have 2 questions here:Whats causing this error and how can we fix it?When training a GPT2 model for casual LM (i.e. predicting the next word in a sentence), should a label be provided by theDatasetin addition to the input ids? TheEsperantoDatasetin Huggingface’s tutorialHow to train a new language model from scratch using Transformers and Tokenizersdoes not, so myMyDatasetdataset for my GPT2 model does not return the label as well. Is this correct? Can aDataCollatorForLanguageModelingwithmlm=Falsebe useful here for GPT2?Thank you so much!import os
from pathlib import Path
import torch
import pytorch_lightning as pl
from transformers import (
    GPT2Config,
    GPT2LMHeadModel,
    GPT2Tokenizer,
    DataCollatorForLanguageModeling,
)
from transformers.optimization import AdamW
from tokenizers import ByteLevelBPETokenizer
from torch.utils.data import (
    DataLoader,
    Dataset,
)

TOKENIZER_DIRPATH = os.path.join("..", "data")


def tokenize_data():
    tokenizer = ByteLevelBPETokenizer()
    tokenizer.train(
        files=os.path.join(TOKENIZER_DIRPATH, "words.txt"),
        vocab_size=50000,
        min_frequency=2,
        special_tokens=["<s>", "</s>", "<unk>", "<mask>", "<pad>",],
    )
    tokenizer.save_model("../data")


class MyDataset(Dataset):
    def __init__(self):
        tokenizer = GPT2Tokenizer(
            os.path.join(TOKENIZER_DIRPATH, "vocab.json"),
            os.path.join(TOKENIZER_DIRPATH, "merges.txt"),
        )
        tokenizer.pad_token = tokenizer.eos_token

        src_file = Path(os.path.join(TOKENIZER_DIRPATH, "words.txt"))
        lines = src_file.read_text(encoding="utf-8").splitlines()
        self.examples = [tokenizer.encode(line) for line in lines]

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, i):
        return torch.tensor(self.examples[i])


class MyDataModule(pl.LightningDataModule):
    def __init__(self):
        super().__init__()
        tokenizer = GPT2Tokenizer(
            os.path.join(TOKENIZER_DIRPATH, "vocab.json"),
            os.path.join(TOKENIZER_DIRPATH, "merges.txt"),
        )
        tokenizer.pad_token = tokenizer.eos_token

        self.tokenizer = tokenizer

    def setup(self, stage):
        self.train_dataset = MyDataset()

    def train_dataloader(self):
        data_collator = DataCollatorForLanguageModeling(
            tokenizer=self.tokenizer, mlm=False
        )
        train_dataloader = DataLoader(
            self.train_dataset, collate_fn=data_collator, batch_size=32, shuffle=True,
        )
        return train_dataloader


class MyModel(pl.LightningModule):
    def __init__(self, learning_rate, adam_beta1, adam_beta2, adam_epsilon):
        super().__init__()
        self.save_hyperparameters()
        config = GPT2Config()
        self.model = GPT2LMHeadModel(config)

    def forward(self, x):
        return self.model(x).logits

    def training_step(self, batch, batch_idx):
        input_ids, labels = batch
        loss = self.model(input_ids, labels=labels).loss
        self.log("train_loss", loss, on_epoch=True)
        return loss

    def configure_optimizers(self):
        optimizer = AdamW(
            self.parameters(),
            self.hparams.learning_rate,
            betas=(self.hparams.adam_beta1, self.hparams.adam_beta2),
            eps=self.hparams.adam_epsilon,
        )
        return optimizer


tokenize_data()
dm = MyDataModule()
model = MyModel(
    learning_rate=5e-5, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-8,
)

trainer = pl.Trainer()
trainer.fit(model, dm)Error TracebackTraceback (most recent call last):
  File "test_gpt.py", line 122, in <module>
    trainer.fit(model, dm)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 499, in fit
    self.dispatch()
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 546, in dispatch
    self.accelerator.start_training(self)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py", line 73, in start_training
    self.training_type_plugin.start_training(trainer)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py", line 114, in start_training
    self._results = trainer.run_train()
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 637, in run_train
    self.train_loop.run_training_epoch()
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 485, in run_training_epoch
    for batch_idx, (batch, is_last_batch) in train_dataloader:
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/profiler/profilers.py", line 82, in profile_iterable
    value = next(iterator)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py", line 47, in _with_is_last
    last = next(it)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/supporters.py", line 470, in __next__
    return self.request_next_batch(self.loader_iters)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/trainer/supporters.py", line 484, in request_next_batch
    return apply_to_collection(loader_iters, Iterator, next)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/pytorch_lightning/utilities/apply_func.py", line 84, in apply_to_collection
    return function(data, *args, **kwargs)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 435, in __next__
    data = self._next_data()
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1085, in _next_data
    return self._process_data(data)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1111, in _process_data
    data.reraise()
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/torch/_utils.py", line 428, in reraise
    raise self.exc_type(msg)
TypeError: Caught TypeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 198, in _worker_loop
    data = fetcher.fetch(index)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 47, in fetch
    return self.collate_fn(data)
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/transformers/data/data_collator.py", line 341, in __call__
    batch = {"input_ids": _collate_batch(examples, self.tokenizer)}
  File "/opt/anaconda3/envs/myhuggingface/lib/python3.8/site-packages/transformers/data/data_collator.py", line 216, in _collate_batch
    result = examples[0].new_full([len(examples), max_length], tokenizer.pad_token_id)
TypeError: new_full(): argument 'fill_value' (position 2) must be Number, not NoneType