import os import json import pandas as pd import time import phoenix as px from phoenix.trace.langchain import OpenInferenceTracer, LangChainInstrumentor #from hallucinator import HallucinatonEvaluater from langchain.embeddings import HuggingFaceEmbeddings #for using HugginFace models from langchain.chains.question_answering import load_qa_chain from langchain import HuggingFaceHub from langchain.prompts import PromptTemplate from langchain.chains import RetrievalQA from langchain.callbacks import StdOutCallbackHandler #from langchain.retrievers import KNNRetriever from langchain.storage import LocalFileStore from langchain.embeddings import CacheBackedEmbeddings from langchain.vectorstores import FAISS from langchain.document_loaders import WebBaseLoader from langchain.text_splitter import RecursiveCharacterTextSplitter import numpy as np import streamlit as st import pandas as pd # from sklearn import datasets # from sklearn.ensemble import RandomForestClassifier from PIL import Image global trace_df # Page config st.set_page_config(page_title="RAG PoC", layout="wide") st.sidebar.image(Image.open("./test-logo.png"), use_column_width=True) @st.cache_resource def tracer_config(): #phoenix setup session = px.launch_app() # If no exporter is specified, the tracer will export to the locally running Phoenix server tracer = OpenInferenceTracer() # If no tracer is specified, a tracer is constructed for you LangChainInstrumentor(tracer).instrument() time.sleep(3) print(session.url) tracer_config() tab1, tab2 = st.tabs(["📈 **RAG**", "🗃 FactVsHallucinate" ]) os.environ["HUGGINGFACEHUB_API_TOKEN"] = "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" # embedding cache #store = LocalFileStore("./cache/") # define embedder embedder = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") #embedder=HuggingFaceHub(repo_id="sentence-transformers/all-mpnet-base-v2") #embedder = CacheBackedEmbeddings.from_bytes_store(core_embeddings_model, store) # define llm llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000}) #llm=HuggingFaceHub(repo_id="gpt2", model_kwargs={"temperature":1, "max_length":1000000}) handler = StdOutCallbackHandler() # set global variable # vectorstore = None # retriever = None class HallucinatePromptContext: def __init__(self): self.variables_list = ["query","answer","context"] self.base_template = """In this task, you will be presented with a query, a reference text and an answer. The answer is generated to the question based on the reference text. The answer may contain false information, you must use the reference text to determine if the answer to the question contains false information, if the answer is a hallucination of facts. Your objective is to determine whether the reference text contains factual information and is not a hallucination. A 'hallucination' in this context refers to an answer that is not based on the reference text or assumes information that is not available in the reference text. Your response should be a single word: either "factual" or "hallucinated", and it should not include any other text or characters. "hallucinated" indicates that the answer provides factually inaccurate information to the query based on the reference text. "factual" indicates that the answer to the question is correct relative to the reference text, and does not contain made up information. Please read the query and reference text carefully before determining your response. # Query: {query} # Reference text: {context} # Answer: {answer} Is the answer above factual or hallucinated based on the query and reference text?""" class HallucinatonEvaluater: def __init__(self, item): self.question = item["question"] self.answer = item["answer"] #self.domain = item["domain"] self.context = item["context"] self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000}) def get_prompt_template(self): prompt = HallucinatePromptContext() template = prompt.base_template varialbles = prompt.variables_list eval_template = PromptTemplate(input_variables=varialbles, template=template) return eval_template def evaluate(self): prompt = self.get_prompt_template().format(query = self.question, answer = self.answer, context = self.context) score = self.llm(prompt) return score @st.cache_resource def initialize_vectorstore(): webpage_loader = WebBaseLoader("https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us").load() webpage_chunks = _text_splitter(webpage_loader) global vectorstore global retriever # store embeddings in vector store vectorstore = FAISS.from_documents(webpage_chunks, embedder) print("vector store initialized with sample doc") # instantiate a retriever retriever = vectorstore.as_retriever() return retriever def _text_splitter(doc): text_splitter = RecursiveCharacterTextSplitter( chunk_size=600, chunk_overlap=50, length_function=len, ) return text_splitter.transform_documents(doc) def _load_docs(path: str): load_doc = WebBaseLoader(path).load() doc = _text_splitter(load_doc) return doc def rag_response(response): #st.markdown("""