Update app.py

app.py

@@ -18,6 +18,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.vectorstores import FAISS
 from langchain.chains import ConversationalRetrievalChain
+from huggingface_hub import InferenceClient
 
 # Login to Hugging Face using a token
 # huggingface_hub.login(HF_TOKEN)
@@ -38,12 +39,37 @@ device = f'cuda:{cuda.current_device()}' if cuda.is_available() else 'cpu'
 #     bnb_4bit_compute_dtype=bfloat16
 # )
 
-tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct",token=HF_TOKEN)
-model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct", device_map="auto",token=HF_TOKEN)  # to("cuda:0") 
-terminators = [
-    tokenizer.eos_token_id,
-    tokenizer.convert_tokens_to_ids("<|eot_id|>")
-]
+# tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct",token=HF_TOKEN)
+# model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct", device_map="auto",token=HF_TOKEN)  # to("cuda:0") 
+# terminators = [
+#     tokenizer.eos_token_id,
+#     tokenizer.convert_tokens_to_ids("<|eot_id|>")
+# ]
+
+
+model_config = transformers.AutoConfig.from_pretrained(
+            self.model_id,
+            # use_auth_token=hf_auth
+        )
+model = transformers.AutoModelForCausalLM.from_pretrained(
+            self.model_id,
+            trust_remote_code=True,
+            config=model_config,
+            quantization_config=bnb_config,
+            # use_auth_token=hf_auth
+        )
+model.eval()
+tokenizer = transformers.AutoTokenizer.from_pretrained(
+            self.model_id,
+            # use_auth_token=hf_auth
+        )
+generate_text = transformers.pipeline(
+            model=self.model, tokenizer=self.tokenizer,
+            return_full_text=True,
+            task='text-generation',
+            temperature=0.01,
+            max_new_tokens=512
+        )
 
 """
 Setting up the stop list to define stopping criteria.