quantization_config

app.py

@@ -1,9 +1,18 @@
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
 
-tokenizer = AutoTokenizer.from_pretrained("TheBloke/SOLAR-10.7B-Instruct-v1.0-uncensored-GPTQ")
+tokenizer = AutoTokenizer.from_pretrained("TheBloke/SOLAR-10.7B-Instruct-v1.0-uncensored-GPTQ", )
 model = AutoModelForCausalLM.from_pretrained("TheBloke/SOLAR-10.7B-Instruct-v1.0-uncensored-GPTQ")
 
+# Add the quantization config with disable_exllama=True
+quantization_config = {
+    'disable_exllama': True,
+}
+
+model = torch.quantization.quantize_dynamic(
+    model, quantization_config=quantization_config,
+)
+
 def generate_response(prompt):
     conversation = [{'role': 'user', 'content': prompt}]
     prompt = tokenizer.apply_chat_template(conversation, tokenizer=False, add_generation_prompt=True)