Fix app

app.py

@@ -1,4 +1,5 @@
 # Import modules
+import torch
 import gradio as gr
 from langchain_community.llms import HuggingFacePipeline
 from langchain_community.embeddings.sentence_transformer import SentenceTransformerEmbeddings
@@ -6,7 +7,7 @@ from langchain_community.vectorstores import Chroma
 from langchain_core.runnables import RunnablePassthrough
 from langchain_core.prompts import PromptTemplate
 from peft import PeftModel
-from transformers import AutoModelForCausalLM, GenerationConfig, pipeline
+from transformers import AutoModelForCausalLM, BitsAndBytesConfig, GenerationConfig, pipeline
 
 base_model = "microsoft/phi-2"
 
@@ -17,6 +18,13 @@ embedding_function = SentenceTransformerEmbeddings(
     model_kwargs={"device": "cuda"},  # Use the GPU
 )
 
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.float16,
+    bnb_4bit_use_double_quant=False,
+)
+
 # Load the fine-tuned model by merging the base model and the adapter
 # (checkpointed at 1 epoch = 77 steps)
 adapter = "./results/checkpoint-77"