Finetuning 4bit model
Hi, im triying to finetune an llama-7B repo, a quantized 4bit version, but when the training starts suddenly drops to 0, the model im triying que finetune is this: “LinkSoul/Chinese-Llama-2-7b-4bit” This its my training scipt:import pandas as pdimport torchimport wandbfrom torch.utils.data import Datasetfrom transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments, DataCollatorForLanguageModelingwandb.init(project=“mi-proyecto-alpaca”)class ParquetDataset(Dataset):definit(self, tokenizer, file_paths, block_size):self.tokenizer = tokenizerself.block_size = block_sizeself.inputs =self.targets =for file_path in file_paths:
        try:
            df = pd.read_parquet(file_path)
        except Exception as e:
            print(f"Error al leer el archivo {file_path}: {e}")
            continue

        self.inputs.extend(df['instruction'].tolist())
        self.targets.extend(df['output'].tolist())

    # Tokenizar las entradas y las salidas
    self.tokenized_inputs = tokenizer(self.inputs, truncation=True, padding='max_length', max_length=block_size)
    self.tokenized_targets = tokenizer(self.targets, truncation=True, padding='max_length', max_length=block_size)

def __len__(self):
    return len(self.inputs)

def __getitem__(self, i):
    return {'input_ids': torch.tensor(self.tokenized_inputs['input_ids'][i]).long(),
            'attention_mask': torch.tensor(self.tokenized_inputs['attention_mask'][i]).long(),
            'labels': torch.tensor(self.tokenized_targets['input_ids'][i]).long()}from transformers import GPTQConfig  # Importing GPTQConfig for quantizationdef main():model_name_or_path = “./alpacashort”tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)# Setting up GPTQConfig for 4-bit quantization
gptq_config = GPTQConfig(bits=4, disable_exllama=True)  # Add other parameters as needed

model = AutoModelForCausalLM.from_pretrained(
    model_name_or_path, 
    load_in_4bit=True,
    device_map="auto",  # Setting device_map to "auto"
    quantization_config=gptq_config  # Adding quantization config
)

train_file_paths = ["./alpacadataset/train-00000-of-00001-6ef3991c06080e14.parquet"]
train_dataset = ParquetDataset(tokenizer, train_file_paths, block_size=128)

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)


training_args = TrainingArguments(
    output_dir="./output",
    overwrite_output_dir=True,
    num_train_epochs=1,
    per_device_train_batch_size=32,  # Specify batch size
    save_steps=10_000,
    save_total_limit=2,
    pad_token_id=tokenizer.pad_token_id  # Specify pad_token_id
)

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=train_dataset,
)

trainer.train()ifname== “main”:main()And this is the training output:You are using the default legacy behaviour of the <class ‘transformers.models.llama.tokenization_llama.LlamaTokenizer’>. If you see this, DO NOT PANIC! This is expected, and simply means that thelegacy(previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, setlegacy=True.This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in⚠️⚠️[`T5Tokenize`] Fix T5 family tokenizers⚠️⚠️ by ArthurZucker · Pull Request #24565 · huggingface/transformers · GitHubLoading checkpoint shards: 100%|██████████████████| 2/2[00:45<00:00, 22.84s/it]0%|                                                  | 0/4334 [00:00<?, ?it/s]You’re using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the__call__method is faster than using a method to encode the text followed by a call to thepadmethod to get a padded encoding./home/gus/.local/lib/python3.8/site-packages/bitsandbytes/nn/modules.py:224: UserWarning: Input type into Linear4bit is torch.float16, but bnb_4bit_compute_type=torch.float32 (default). This will lead to slow inference or training speed.warnings.warn(f’Input type into Linear4bit is torch.float16, but bnb_4bit_compute_type=torch.float32 (default). This will lead to slow inference or training speed.'){‘loss’: 1.7572, ‘learning_rate’: 2.0000000000000003e-06, ‘epoch’: 0.0}{‘loss’: 0.0, ‘learning_rate’: 4.000000000000001e-06, ‘epoch’: 0.0}{‘loss’: 0.0, ‘learning_rate’: 6e-06, ‘epoch’: 0.01}{‘loss’: 0.0, ‘learning_rate’: 8.000000000000001e-06, ‘epoch’: 0.01}{‘loss’: 0.0, ‘learning_rate’: 1e-05, ‘epoch’: 0.01}{‘loss’: 0.0, ‘learning_rate’: 1.2e-05, ‘epoch’: 0.01}{‘loss’: 0.0, ‘learning_rate’: 1.4000000000000001e-05, ‘epoch’: 0.02}Any clue about what im doing wrong?
2023-08-29T18:02:52Z
Hey do you mind sharing your notebook?