update

app.py

@@ -1,5 +1,7 @@
 import os
 import gradio as gr
+
+import torch
 from transformers import pipeline, StoppingCriteriaList, MaxTimeCriteria
 from elasticsearch import Elasticsearch
 
@@ -38,7 +40,7 @@ def rag_pipeline(prompt, index="pubmed", num_docs=3, model_name="HuggingFaceH4/z
     A simple RAG pipeline that retrieves documents and uses them to enrich the context for the LLM.
     """
     # Load your language model from HuggingFace Transformers
-    generator = pipeline("text-generation", model=model_name)
+    generator = pipeline("text-generation", model=model_name, torch_dtype=torch.bfloat16)
 
     num_docs = int(num_docs)