Update README.md

README.md

@@ -1,3 +1,23 @@
 ---
 license: cc-by-4.0
 ---
+
+Experimental quantization. 
+
+Working inference code (regular inference with autogptq does not work without return_token_type_ids=False, didn't get it to work with textgen-webui): 
+
+from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig 
+
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir, use_fast=True) 
+
+model = AutoGPTQForCausalLM.from_quantized(quantized_model_dir, device="cuda:0", use_triton=False)  
+
+input_ids = tokenizer("Question: What is the purpose of life?\n\nAnswer:", return_tensors="pt").input_ids.to("cuda:0")
+
+out = model.generate(input_ids=input_ids,max_length=300)
+
+print(tokenizer.decode(out[0]))
+
+or 
+
+print(tokenizer.decode(model.generate(**tokenizer("test is", return_tensors="pt", return_token_type_ids=False).to("cuda:0"))[0]))