Kfolds leaking into subsequent folds
I am attempting to use Kfolds cross-validation to train my model but the F1 score keeps on increasing between folds, which tells me that there is some serious data leaking.Screenshot 2024-04-18 at 12.13.09 PM1576×546 26 KBHere is my training codefor fold, (train_idxs, val_idxs) in enumerate(zip(train_list, val_list), start=1): # K-fold loop
        
        output_dir = f"{model_name}/output_dir_{weight_decay}_fold_{fold}"

        resume_bool = False
        
        #Skip folds already completed
        if os.path.exists(f"{output_dir}"):
            if os.path.exists(f"{output_dir}/finished.txt"):
                print('\033[93m'+ f"Skipping {output_dir} as it is indicated as finished" + '\033[0m')
                continue
            else:
                print('\033[93m'+ f"Starting from last checkpoint {output_dir}"+ '\033[0m')
                resume_bool = True # resume from the last checkpoint if there is an output folder but it is not finished.


        print(f"------Fold {fold}/{len(train_list)}--------\n")
        train_ds = tokenized_Hraf["train"].select(train_idxs)
        val_ds = tokenized_Hraf["train"].select(val_idxs)


        training_args = TrainingArguments(
            output_dir=output_dir,
            learning_rate=learning_rate,
            per_device_train_batch_size=8,
            per_device_eval_batch_size=8,
            num_train_epochs=3,
            weight_decay=weight_decay,
            evaluation_strategy="epoch",
            save_strategy="epoch",
            metric_for_best_model='f1',
            push_to_hub=False,
            logging_dir=f"{model_name}/logs_{weight_decay}_fold_{fold}",
            logging_steps=100,
        )


        trainer = Trainer(
            model=model,
            args=training_args,
            train_dataset=train_ds,
            eval_dataset=val_ds,
            tokenizer=tokenizer,
            data_collator=data_collator,
            # callbacks=[best_checkpoint_callback], 
            compute_metrics=compute_metrics,
            
        )
        try:
            trainer.train() 
        except:
            print('\033[91m'+ f"A crash occurred, restarting fold from checkpoint"+ '\033[0m')
            trainer.train(resume_from_checkpoint=True) #This is the same thing above but often restarting can make all the difference so let's try it