Update app_v2.py

app_v2.py

@@ -13,11 +13,25 @@ os.makedirs(quantized_model_dir, exist_ok=True)
 # Quantization configuration
 quantize_config = BaseQuantizeConfig(bits=4, group_size=128, desc_act=False)
 
+# Load the model using from_quantized
+model = AutoGPTQForCausalLM.from_quantized(
+    pretrained_model_dir,
+    use_safetensors=True,
+    strict=False,
+    #model_basename=quantized_model_dir,
+    device="cuda:0",
+    trust_remote_code=True,
+    use_triton=False,
+    quantize_config=quantize_config
+)
+
+model.save_quantized(quantized_model_dir)
+
+
 # Load the tokenizer
-tokenizer = AutoTokenizer.from_pretrained(pretrained_model_dir, use_fast=True)
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir, use_fast=True)
 
-# Load the model using Option 1
-model = AutoGPTQForCausalLM.from_pretrained(pretrained_model_dir, quantize_config)
+model_for_inference = AutoGPTQForCausalLM.from_pretrained(quantized_model_dir)
 
 # Starting Streamlit app
 st.title("AutoGPTQ Streamlit App")