Update app.py

app.py

@@ -1,48 +1,46 @@
 import gradio as gr
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, pipeline
+import os
+from langchain.llms import CTransformers
+from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
 
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_use_double_quant=True,
-)
+
+MODEL_PATH = 'TheBloke/Mistral-7B-Instruct-v0.1-GGUF'
+
+# Some basic configurations for the model
+config = {
+    "max_new_tokens": 1000,
+    "context_length": 1000,
+    "repetition_penalty": 1.1,
+    "temperature": 0.5,
+    "top_k": 50,
+    "top_p": 0.9,
+    "stream": True,
+    "threads": int(os.cpu_count() / 2)
+}
 
 model_name = "mistralai/Mistral-7B-Instruct-v0.1"
 
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        load_in_4bit=True,
-        quantization_config=bnb_config,
-        torch_dtype=torch.bfloat16,
-        device_map="auto",
-        trust_remote_code=True,
-    )
 
-device = "cuda"
+# We use Langchain's CTransformers llm class to load our quantized model
+llm = CTransformers(model=MODEL_PATH,
+                    config=config)
+
+# Tokenizer for Mistral-7B-Instruct from HuggingFace
+tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1")
 
 def greet(input_text):
   question = input_text
 
   prompt = f"""<s>[INST] Le contexte est l'assurance maladie en France[/INST]
-    {question}</s>
-    [INST] Rédige un email courtois de réponse en français à la question [/INST]"""
-
-  messages = [
-    {"role": "user", "content": question},
-    {"role": "assistant", "content": "le contexte est l'assurance maladie en France"},
-    {"role": "user", "content": "Rédige un email courtois de réponse en français à la question"}
-]
-
-  encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt")
+  {question}</s>
+  [INST] Rédige un email courtois de réponse en français à la question [/INST]"""
 
-  model_inputs = encodeds.to(device)
+  answer = llm(prompt)
 
-  generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
-  decoded = tokenizer.batch_decode(generated_ids)
+  answer = answer.replace("</s>", "").replace("[Votre nom]", "").replace("[nom]", "")
 
-  answer = decoded[0].split("[/INST]")[2].replace("</s>", "").replace("[Votre nom]", "").replace("[nom]", "")
+  
 
   return answer