MohamedRashad
/

AceGPT-7B-chat-AWQ

Text Generation

text-generation-inference

4-bit precision

Model card Files Files and versions Community

MohamedRashad commited on Nov 18, 2023

Commit

c3fcc0e

·

1 Parent(s): d1bae6d

Update README.md

Files changed (1) hide show

README.md +3 -1

README.md CHANGED Viewed

@@ -93,9 +93,11 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
 model_name_or_path = "MohamedRashad/AceGPT-7B-chat-AWQ"
-tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
 model = AutoModelForCausalLM.from_pretrained(
     model_name_or_path,
     low_cpu_mem_usage=True,
     device_map="auto"
 )

 model_name_or_path = "MohamedRashad/AceGPT-7B-chat-AWQ"
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side="right")
 model = AutoModelForCausalLM.from_pretrained(
     model_name_or_path,
+    use_flash_attention_2=True,
+    torch_dtype=torch.float16,
     low_cpu_mem_usage=True,
     device_map="auto"
 )