Binary model either predicts all 0s or all 1s
My model ran without errors before but only predicted the most frequent class (binary classification) in every instance (0). So I followedthis adviceand added a custom loss function - but now it only predicts the other class (1).It doesn’t matter how I adjust the weights, it’s either all 0s or all 1s.By adding the CustomTrainer I got the following warning at the beginning although I declare in the model callignore_mismatched_sizes=Trueand so I don’t know if it matters:Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at ishan/distilbert-base-uncased-mnli and are newly initialized because the shapes did not match:classifier.weight: found shape torch.Size([3, 768]) in the checkpoint and torch.Size([2, 768]) in the model instantiatedclassifier.bias: found shape torch.Size([3]) in the checkpoint and torch.Size([2]) in the model instantiatedYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.I’m not even sure the CustomTrainer takes the proper model and data.The code only predicting 1s for my dev set looks roughly like this:from transformers import DefaultDataCollator, EarlyStoppingCallback, IntervalStrategy, AutoTokenizer, \
    AutoModelForSequenceClassification, TrainingArguments, Trainer
from models.general import format_dataset
from db_connector.db_access import get_all
import torch
from torch import nn
import evaluate
from torch.utils.data import Dataset
from torch.nn import functional as F

num_train_epochs = 3
model_name = "ishan/distilbert-base-uncased-mnli"

# use GPU instead of CPU
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")

tokenizer = AutoTokenizer.from_pretrained(model_name)  # PyTorch model


class CustomDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['label'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)


class CustomTrainer(Trainer):
    def compute_loss(self, some_model, inputs, return_outputs=False):
        labels = inputs.get("labels")
        outputs = some_model(**inputs)
        logits = outputs.get("logits")
        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([1.0, 100.0])).cuda()
        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))
        return (loss, outputs) if return_outputs else loss


# read users from database
train_users = get_all("train_users")
dev_users = get_all("dev_users")
train_label_list, train_tweet_list = format_dataset(train_users)
dev_label_list, dev_tweet_list = format_dataset(dev_users)
train_encodings = tokenizer(train_tweet_list, truncation=True, padding=True)
dev_encodings = tokenizer(dev_tweet_list, truncation=True, padding=True)

train_dataset = CustomDataset(train_encodings, train_label_list)
dev_dataset = CustomDataset(dev_encodings, dev_label_list)

model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2,
                                                           ignore_mismatched_sizes=True)
model.to(device)

data_collator = DefaultDataCollator(return_tensors="pt")

training_args = TrainingArguments(
    output_dir=result_path,  # output directory
    evaluation_strategy=IntervalStrategy.EPOCH,  # evaluation strategy to adopt during training
    num_train_epochs=num_train_epochs,  # total number of training epochs (default: 8)
    per_device_train_batch_size=16,  # batch size per device during training (default: 8)
    per_device_eval_batch_size=64,  # batch size for evaluation
    logging_dir=log_path,  # directory for storing logs
    logging_first_step=True,
    save_strategy=IntervalStrategy.EPOCH,
    save_total_limit=1,  # save only the last model (and the best one, if save_best_model is True)
    logging_strategy=IntervalStrategy.EPOCH,
    load_best_model_at_end=True,  # load best model at the end of training
)

trainer = CustomTrainer(
    model=model,  # the instantiated 🤗 Transformers model to be trained
    args=training_args,  # training arguments, defined above
    train_dataset=train_dataset,  # training dataset
    eval_dataset=dev_dataset,  # evaluation dataset
    callbacks=[EarlyStoppingCallback(early_stopping_patience=5)]
)

trainer.train()

# Evaluation
clf_metrics = evaluate.combine(["accuracy", "f1", "precision", "recall"])

# predictions and metrics for the dev set
dev_pred = trainer.predict(dev_dataset)  # returns: predictions, label_ids, metrics
torch_logits = torch.from_numpy(dev_pred.predictions)  # convert numpy ndarray to torch tensor
probabilities_scores = F.softmax(torch_logits, dim=-1).numpy()  # returns a list of numpy.ndarrays
torch_probs = torch.from_numpy(probabilities_scores)
# Get the max probability and the index of the highest prediction for each output.
max_probabilities, max_indices = torch.max(torch_probs, dim=1)  # returns two tensors
# print(type(max_probabilities))
predicted_labels = dev_pred.predictions.argmax(-1)  # returns a numpy ndarray

unseen_labels = set(predicted_labels) - set(dev_dataset.labels)
unpredicted_labels = set(dev_dataset.labels) - set(predicted_labels)
if unseen_labels:
    print("WARNING: Labels that are not part of the training set were predicted: {}".format(unseen_labels))
if unpredicted_labels:
    print("WARNING: Some labels were never predicted: {}".format(unpredicted_labels))

dev_res = clf_metrics.compute(predictions=predicted_labels, references=dev_dataset.labels)
print("DEV RESULTS: ", dev_res)Please help, I already spent ages on this.
2023-05-22T01:55:12Z
Hi@HarpiyeCan you try setting thelearning_rateto a smaller value say1e-7and increase thenum_train_epochsto say10It should work outTo do this, yourtraining_argswill be defined like -TrainingArguments( learning_rate = 1e-7, num_train_epochs = 10 ....)