Create README.md

README.md

@@ -0,0 +1,26 @@
+## Inference calling Way:
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    HqqConfig
+)
+from peft import PeftModel
+import torch
+from awq import AutoAWQForCausalLM
+import torch
+
+device = torch.device("cuda")
+
+model_id = "huzaifa1117/tinyllama_AWQ_4bit"
+# All linear layers will use the same quantization config
+quant_config = HqqConfig(nbits=1, group_size=64, quant_zero=False, quant_scale=False, axis=1)
+
+#Load the tokenizer to save it along with the model
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+
+model = AutoAWQForCausalLM.from_pretrained(model_id, low_cpu_mem_usage=True, use_cache=False, device_map='cuda')
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+# model = AutoModelForCausalLM.from_pretrained(model_id)
+
+model.to(device)