Create README.md

README.md

@@ -0,0 +1,18 @@
+Please use the following code sample to load these AWQ models:
+```
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+from auto_gptq import AutoGPTQForCausalLM
+import torch
+
+model_path = 'efficient-llm/vicuna-13b-v1.3-awq'
+config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+enc = AutoTokenizer.from_pretrained('meta-llama/Llama-2-7b-hf', trust_remote_code=True)
+kwargs = {"torch_dtype": torch.float16, "low_cpu_mem_usage": True}
+model = AutoModelForCausalLM.from_pretrained(
+    model_path, config=config, trust_remote_code=True, device_map='auto', revision='3bit_128g', **kwargs)
+
+model.eval()
+input_ids = enc('How are you today?', return_tensors='pt').input_ids.to('cuda')
+outputs = model.generate(input_ids=input_ids, max_length=128)
+print(enc.decode(outputs[0]))
+```