Upload 2 files

app.py

@@ -1,23 +1,17 @@
 import gradio as gr
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
 from einops import einsum
 from tqdm import tqdm
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 model_name = 'microsoft/Phi-3-mini-4k-instruct'
 
-quantization_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_compute_dtype=torch.float16
-)
-
 model = AutoModelForCausalLM.from_pretrained(
     model_name, 
     device_map=device, 
     torch_dtype="auto", 
     trust_remote_code=True, 
-    quantization_config=quantization_config,
 )
 
 tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -179,4 +173,3 @@ def launch_app():
 
 if __name__ == "__main__":
     launch_app()
-

requirements.txt

@@ -10,7 +10,7 @@ appnope==0.1.4
 asttokens==2.4.1
 astunparse==1.6.3
 attrs==24.2.0
-bitsandbytes==0.44.1
+bitsandbytes==0.42.0
 certifi==2024.8.30
 charset-normalizer==3.3.2
 click==8.1.7