LLaMa 2 Error when merging adapters to base model
Traceback (most recent call last):
  File "/content/merge.py", line 45, in <module>
    main()
  File "/content/merge.py", line 34, in main
    model = PeftModel.from_pretrained(base_model, args.peft_model_path, **device_arg)
  File "/usr/local/lib/python3.10/dist-packages/peft/peft_model.py", line 278, in from_pretrained
    model.load_adapter(model_id, adapter_name, is_trainable=is_trainable, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/peft/peft_model.py", line 587, in load_adapter
    dispatch_model(
  File "/usr/local/lib/python3.10/dist-packages/accelerate/big_modeling.py", line 363, in dispatch_model
    raise ValueError(
ValueError: We need an `offload_dir` to dispatch this model according to this `device_map`, the following submodules need to be offloaded: base_model.model.model.layers.16, base_model.model.model.layers.17, base_model.model.model.layers.18, base_model.model.model.layers.19, base_model.model.model.layers.20, base_model.model.model.layers.21, base_model.model.model.layers.22, base_model.model.model.layers.23, base_model.model.model.layers.24, base_model.model.model.layers.25, base_model.model.model.layers.26, base_model.model.model.layers.27, base_model.model.model.layers.28, base_model.model.model.layers.29, base_model.model.model.layers.30, base_model.model.model.layers.31, base_model.model.model.layers.32, base_model.model.model.layers.33, base_model.model.model.layers.34, base_model.model.model.layers.35, base_model.model.model.layers.36, base_model.model.model.layers.37, base_model.model.model.layers.38, base_model.model.model.layers.39, base_model.model.model.norm, base_model.model.lm_head.I am merging the LLaMa 2 13b model to this adapter:baebee/Porpoise-13b · Hugging FaceI am using the platypus way of merging, here is the script:from transformers import AutoModelForCausalLM, AutoTokenizerfrom peft import PeftModelimport torchimport osimport argparsedef get_args():parser = argparse.ArgumentParser()parser.add_argument(“–base_model_name_or_path”, type=str)parser.add_argument(“–peft_model_path”, type=str)parser.add_argument(“–output_dir”, type=str)parser.add_argument(“–device”, type=str, default=“auto”)return parser.parse_args()def main():args = get_args()if args.device == 'auto':
    device_arg = { 'device_map': 'auto' }
else:
    device_arg = { 'device_map': { "": args.device} }

print(f"Loading base model: {args.base_model_name_or_path}")
base_model = AutoModelForCausalLM.from_pretrained(
    args.base_model_name_or_path,
    return_dict=True,
    torch_dtype=torch.float16,
    **device_arg
)

print(f"Loading PEFT: {args.peft_model_path}")
model = PeftModel.from_pretrained(base_model, args.peft_model_path, **device_arg)
print(f"Running merge_and_unload")
model = model.merge_and_unload()

tokenizer = AutoTokenizer.from_pretrained(args.base_model_name_or_path)

model.save_pretrained(f"{args.output_dir}")
tokenizer.save_pretrained(f"{args.output_dir}")
print(f"Model saved to {args.output_dir}")ifname== “main” :main()