Upload 11 files

.streamlit/config.toml

@@ -0,0 +1,18 @@
+[theme] # You have to add this line
+
+#primaryColor = '#FF8C02' # Bright Orange
+
+#secondaryColor = '#FF8C02' # Bright Orange
+
+#backgroundColor = '#00325B' # Dark Blue
+
+#secondaryBackgroundColor = '#55B2FF' # Lighter Blue
+
+
+primaryColor="#ff4b4b"
+backgroundColor="#00325B"
+secondaryBackgroundColor="#262730"
+textColor="#fafafa"
+font="monospace"
+
+

Dockerfile

@@ -0,0 +1,31 @@
+# Use the official Python 3.9 image
+FROM python:3.9
+
+# Set the working directory to /code
+WORKDIR /code
+
+# Copy the current directory contents into the container at /code
+COPY ./requirements.txt /code/requirements.txt
+
+
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+# Install requirements.txt 
+RUN pip install --no-cache-dir --upgrade --user -r /code/requirements.txt
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+EXPOSE 6060
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+CMD ["streamlit", "run", "demo_app/main.py", "--server.port", "7860"]

Index.py

@@ -0,0 +1,326 @@
+from fastapi import FastAPI
+import os
+import json
+import pandas as pd
+import time
+
+import phoenix as px
+from phoenix.trace.langchain import OpenInferenceTracer, LangChainInstrumentor
+
+
+from langchain.embeddings import HuggingFaceEmbeddings #for using HugginFace models
+from langchain.chains.question_answering import load_qa_chain
+from langchain import HuggingFaceHub
+from langchain.prompts import PromptTemplate
+
+from langchain.chains import RetrievalQA
+from langchain.callbacks import StdOutCallbackHandler
+
+#from langchain.retrievers import KNNRetriever
+from langchain.storage import LocalFileStore
+from langchain.embeddings import CacheBackedEmbeddings
+from langchain.vectorstores import FAISS
+
+
+from langchain.document_loaders import WebBaseLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+# from langchain import HuggingFaceHub
+# from langchain.prompts import PromptTemplate
+# from langchain.chains import LLMChain
+# from txtai.embeddings import Embeddings
+# from txtai.pipeline import Extractor
+
+# import pandas as pd
+# import sqlite3
+# import os
+
+# NOTE - we configure docs_url to serve the interactive Docs at the root path
+# of the app. This way, we can use the docs as a landing page for the app on Spaces.
+app = FastAPI(docs_url="/")
+
+#phoenix setup
+session = px.launch_app()
+# If no exporter is specified, the tracer will export to the locally running Phoenix server
+tracer = OpenInferenceTracer()
+# If no tracer is specified, a tracer is constructed for you
+LangChainInstrumentor(tracer).instrument()
+print(session.url)
+
+
+os.environ["HUGGINGFACEHUB_API_TOKEN"] = "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"
+
+# embedding cache
+store = LocalFileStore("./cache/")
+
+# define embedder
+core_embeddings_model  = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+embedder = CacheBackedEmbeddings.from_bytes_store(core_embeddings_model, store)
+
+# define llm
+llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
+#llm=HuggingFaceHub(repo_id="gpt2", model_kwargs={"temperature":1, "max_length":1000000})
+handler =  StdOutCallbackHandler()
+
+# set global variable
+vectorstore = None
+retriever = None
+
+
+def initialize_vectorstore():
+
+    webpage_loader = WebBaseLoader("https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us").load()
+    webpage_chunks = _text_splitter(webpage_loader)
+
+    global vectorstore
+    global retriever
+
+    # store embeddings in vector store
+    vectorstore = FAISS.from_documents(webpage_chunks, embedder)
+    print("vector store initialized with sample doc")
+
+    # instantiate a retriever
+    retriever = vectorstore.as_retriever()
+
+
+def _text_splitter(doc):
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=600,
+        chunk_overlap=50,
+        length_function=len,
+    )
+    return text_splitter.transform_documents(doc)
+
+def _load_docs(path: str):
+    load_doc = WebBaseLoader(path).load()
+    doc = _text_splitter(load_doc)
+    return doc
+
+
+@app.get("/index/")
+def get_domain_file_path(file_path: str):
+    print("file_path  " ,file_path)
+     
+    webpage_loader = _load_docs(file_path)
+
+    webpage_chunks = _text_splitter(webpage_loader)
+
+    # store embeddings in vector store
+    vectorstore.add_documents(webpage_chunks)
+
+    return "document loaded to vector store successfully!!"
+
+
+def _prompt(question):
+    return f"""Answer following question using only the context below. Say 'Could not find answer with provided context' when question can't be answered.
+            Question: {question}
+            Context: """
+
+
+
+class BasePromptContext:
+    def __init__(self):
+        self.variables_list = ["question","answer","context"]
+        self.base_template = """Please act as an impartial judge and evaluate the quality of the provided answer which attempts to answer the provided question based on a provided context.
+And you'll need to submit your grading for the correctness, comprehensiveness and readability of the answer, using JSON format with the 2 items in parenthesis:
+("score": [your score number for the correctness of the answer], "reasoning": [your one line step by step reasoning about the correctness of the answer])
+  Below is your grading rubric:
+- Correctness: If the answer correctly answer the question, below are the details for different scores:
+  - Score 0: the answer is completely incorrect, doesn’t mention anything about the question or is completely contrary to the correct answer.
+      - For example, when asked “How to terminate a databricks cluster”, the answer is empty string, or content that’s completely irrelevant, or sorry I don’t know the answer.
+  - Score 4: the answer provides some relevance to the question and answer one aspect of the question correctly.
+      - Example:
+          - Question: How to terminate a databricks cluster
+          - Answer: Databricks cluster is a cloud-based computing environment that allows users to process big data and run distributed data processing tasks efficiently.
+          - Or answer:  In the Databricks workspace, navigate to the "Clusters" tab. And then this is a hard question that I need to think more about it
+  - Score 7: the answer mostly answer the question but is missing or hallucinating on one critical aspect.
+      - Example:
+          - Question: How to terminate a databricks cluster”
+          - Answer: “In the Databricks workspace, navigate to the "Clusters" tab.
+          Find the cluster you want to terminate from the list of active clusters.
+          And then you’ll find a button to terminate all clusters at once”
+  - Score 10: the answer correctly answer the question and not missing any major aspect
+      - Example:
+          - Question: How to terminate a databricks cluster
+          - Answer: In the Databricks workspace, navigate to the "Clusters" tab.
+          Find the cluster you want to terminate from the list of active clusters.
+          Click on the down-arrow next to the cluster name to open the cluster details.
+          Click on the "Terminate" button. A confirmation dialog will appear. Click "Terminate" again to confirm the action.”
+Provided question:
+{question}
+Provided answer:
+{answer}
+Provided context:
+{context}
+Please provide your grading for the correctness and explain you gave the particular grading"""
+        
+        
+class Evaluater:
+    def __init__(self, item):
+        self.question = item["question"]
+        self.answer = item["answer"]
+        #self.domain = item["domain"]
+        self.context = item["context"]
+        self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
+
+    def get_prompt_template(self):
+        prompt = BasePromptContext()
+        template = prompt.base_template
+        varialbles = prompt.variables_list
+        eval_template = PromptTemplate(input_variables=varialbles, template=template)
+        return eval_template
+
+    def evaluate(self):
+        prompt = self.get_prompt_template().format(question = self.question, answer = self.answer, context = self.context)
+        score = self.llm(prompt)
+        return score
+
+# Create extractor instance
+def _create_evaluation_scenario(item):
+    score = Evaluater(item).evaluate()
+    return score
+
+
+class HallucinatePromptContext:
+    def __init__(self):
+        self.variables_list = ["query","answer","context"]
+        self.base_template = """In this task, you will be presented with a query, a reference text and an answer. The answer is
+
+generated to the question based on the reference text. The answer may contain false information, you
+
+must use the reference text to determine if the answer to the question contains false information,
+
+if the answer is a hallucination of facts. Your objective is to determine whether the reference text
+
+contains factual information and is not a hallucination. A 'hallucination' in this context refers to
+
+an answer that is not based on the reference text or assumes information that is not available in
+
+the reference text. Your response should be a single word: either "factual" or "hallucinated", and
+
+it should not include any other text or characters. "hallucinated" indicates that the answer
+
+provides factually inaccurate information to the query based on the reference text. "factual"
+
+indicates that the answer to the question is correct relative to the reference text, and does not
+
+contain made up information. Please read the query and reference text carefully before determining
+
+your response.
+
+ 
+
+    # Query: {query}
+
+    # Reference text: {context}
+
+    # Answer: {answer}
+
+    Is the answer above factual or hallucinated based on the query and reference text?"""
+        
+        
+class HallucinatonEvaluater:
+    def __init__(self, item):
+        self.question = item["question"]
+        self.answer = item["answer"]
+        #self.domain = item["domain"]
+        self.context = item["context"]
+        self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
+
+    def get_prompt_template(self):
+        prompt = HallucinatePromptContext()
+        template = prompt.base_template
+        varialbles = prompt.variables_list
+        eval_template = PromptTemplate(input_variables=varialbles, template=template)
+        return eval_template
+
+    def evaluate(self):
+        prompt = self.get_prompt_template().format(query = self.question, answer = self.answer, context = self.context)
+        score = self.llm(prompt)
+        return score
+
+# Create extractor instance
+def _create_evaluation_scenario(item):
+    score = Evaluater(item).evaluate()
+    return score
+
+# Create extractor instance
+def _create_hallucination_scenario(item):
+    score = HallucinatonEvaluater(item).evaluate()
+    return score
+
+
+@app.get("/rag")
+def rag( question: str, evaluate: bool):   
+
+    chain = RetrievalQA.from_chain_type(
+    llm=llm,
+    retriever=retriever,
+    callbacks=[handler],
+    return_source_documents=True
+    )
+
+    #response = chain("how tredence brought good insight?")
+    response = chain(question)
+    print(response["result"])
+
+    time.sleep(4)
+    
+    df = px.active_session().get_spans_dataframe()
+    print(px.active_session())
+    print(px.active_session().get_spans_dataframe())
+    print(df.count())        
+    df_sorted = df.sort_values(by='end_time',ascending=False) 
+
+    model_input = json.loads(df_sorted[df_sorted["name"] == "LLMChain"]["attributes.input.value"][0])
+    context = model_input["context"]
+
+    print(context)
+
+    if evaluate:
+        score = _create_evaluation_scenario({
+            "question": question,
+            "answer": response['result'],
+            "context": context            
+        })
+    else:
+        score = "Evaluation is Turned OFF"
+ 
+    return {"question": question, "answer": response['result'], "context": context, "score": score}
+
+
+initialize_vectorstore()
+
+
+@app.get("/trace")
+def trace():   
+    df = px.active_session().get_spans_dataframe().fillna('')
+    return df
+
+
+@app.get("/hallucinate")
+def trace(question: str, answer: str, context: str):
+    hallucination_score = _create_hallucination_scenario({
+            "question": question,
+            "answer": answer,
+            "context": context            
+        }        
+    )
+
+    return {"hallucination_score": hallucination_score}
+    
+
+
+
+'''
+#import getpass
+from pyngrok import ngrok, conf
+#print("Enter your authtoken, which can be copied from https://dashboard.ngrok.com/auth")
+conf.get_default().auth_token="2WJNWULs5bCOyJnV24WQYJEKod3_YQUbM5EGCp8sgE4aQvzi"
+port = 37689
+# Open a ngrok tunnel to the HTTP server
+conf.get_default().monitor_thread = False
+public_url = ngrok.connect(port).public_url
+print(" * ngrok tunnel \"{}\" -> \"http://127.0.0.1:{}\"".format(public_url, port))
+'''
+

README.md

@@ -1,10 +1,11 @@
 ---
-title: Streamlit
-emoji: 🏃
-colorFrom: indigo
-colorTo: pink
+title: Text Generation
+emoji: 🌍
+colorFrom: green
+colorTo: yellow
 sdk: docker
 pinned: false
+license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,20 @@
+from fastapi import FastAPI
+from transformers import pipeline
+
+
+# NOTE - we configure docs_url to serve the interactive Docs at the root path
+# of the app. This way, we can use the docs as a landing page for the app on Spaces.
+app = FastAPI(docs_url="/")
+
+pipe = pipeline("text2text-generation", model="google/flan-t5-small")
+
+
+@app.get("/generate")
+def generate(text: str):
+    """
+    Using the text2text-generation pipeline from `transformers`, generate text
+    from the given input text. The model used is `google/flan-t5-small`, which
+    can be found [here](https://huggingface.co/google/flan-t5-small).
+    """
+    output = pipe(text)
+    return {"output": output[0]["generated_text"]}

extractor.py

@@ -0,0 +1,94 @@
+from fastapi import FastAPI
+
+# from transformers import pipeline
+from txtai.embeddings import Embeddings
+from txtai.pipeline import Extractor
+from langchain.document_loaders import WebBaseLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+# NOTE - we configure docs_url to serve the interactive Docs at the root path
+# of the app. This way, we can use the docs as a landing page for the app on Spaces.
+app = FastAPI(docs_url="/")
+
+# Create embeddings model with content support
+embeddings = Embeddings(
+    {"path": "sentence-transformers/all-MiniLM-L6-v2", "content": True}
+)
+
+
+# Create extractor instance
+# extractor = Extractor(embeddings, "google/flan-t5-base")
+
+
+def _stream(dataset, limit, index: int = 0):
+    for row in dataset:
+        yield (index, row.page_content, None)
+        index += 1
+
+        if index >= limit:
+            break
+
+
+def _max_index_id(path):
+    db = sqlite3.connect(path)
+
+    table = "sections"
+    df = pd.read_sql_query(f"select * from {table}", db)
+    return {"max_index": df["indexid"].max()}
+
+
+def _prompt(question):
+    return f"""Answer the following question using only the context below. Say 'no answer' when the question can't be answered.
+            Question: {question}
+            Context: """
+
+
+async def _search(query, extractor, question=None):
+    # Default question to query if empty
+    if not question:
+        question = query
+
+    return extractor([("answer", query, _prompt(question), False)])[0][1]
+
+
+def _text_splitter(doc):
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=500,
+        chunk_overlap=50,
+        length_function=len,
+    )
+    return text_splitter.transform_documents(doc)
+
+
+def _load_docs(path: str):
+    load_doc = WebBaseLoader(path).load()
+    doc = _text_splitter(load_doc)
+    return doc
+
+
+async def _upsert_docs(doc):
+    max_index = _max_index_id("index/documents")
+    embeddings.upsert(_stream(doc, 500, max_index["max_index"]))
+    embeddings.save("index")
+
+    return embeddings
+
+
+@app.put("/rag/{path}")
+async def get_doc_path(path: str):
+    return path
+
+
+@app.get("/rag")
+async def rag(question: str):
+    # question = "what is the document about?"
+    embeddings.load("index")
+    path = await get_doc_path(path)
+    doc = _load_docs(path)
+    embeddings = _upsert_docs(doc)
+
+    # Create extractor instance
+    extractor = Extractor(embeddings, "google/flan-t5-base")
+    answer = await _search(question, extractor)
+    # print(question, answer)
+    return {answer}

hallucinator.py

@@ -0,0 +1,19 @@
+class HallucinatonEvaluater:
+    def __init__(self, item):
+        self.question = item["question"]
+        self.answer = item["answer"]
+        #self.domain = item["domain"]
+        self.context = item["context"]
+        self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
+
+    def get_prompt_template(self):
+        prompt = HallucinatePromptContext()
+        template = prompt.base_template
+        varialbles = prompt.variables_list
+        eval_template = PromptTemplate(input_variables=varialbles, template=template)
+        return eval_template
+
+    def evaluate(self):
+        prompt = self.get_prompt_template().format(query = self.question, answer = self.answer, context = self.context)
+        score = self.llm(prompt)
+        return score

main.py

@@ -0,0 +1,85 @@
+from fastapi import FastAPI
+from txtai.embeddings import Embeddings
+from txtai.pipeline import Extractor
+import os
+from langchain import HuggingFaceHub
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+
+# from transformers import pipeline
+
+# NOTE - we configure docs_url to serve the interactive Docs at the root path
+# of the app. This way, we can use the docs as a landing page for the app on Spaces.
+app = FastAPI(docs_url="/")
+
+# @app.get("/generate")
+# def generate(text: str):
+#     """
+#     Using the text2text-generation pipeline from `transformers`, generate text
+#     from the given input text. The model used is `google/flan-t5-small`, which
+#     can be found [here](https://huggingface.co/google/flan-t5-small).
+#     """
+#     output = pipe(text)
+#     return {"output": output[0]["generated_text"]}
+
+
+def _check_if_db_exists(db_path: str) -> bool:
+    return os.path.exists(db_path)
+
+
+def _load_embeddings_from_db(
+    db_present: bool,
+    domain: str,
+    path: str = "sentence-transformers/all-MiniLM-L6-v2",
+):
+    # Create embeddings model with content support
+    embeddings = Embeddings({"path": path, "content": True})
+    # if Vector DB is not present
+    if not db_present:
+        return embeddings
+    else:
+        if domain == "":
+            embeddings.load("index")  # change this later
+        else:
+            print(3)
+            embeddings.load(f"index/{domain}")
+        return embeddings
+
+
+def _prompt(question):
+    return f"""Answer the following question using only the context below. Say 'no answer' when the question can't be answered.
+            Question: {question}
+            Context: """
+
+
+def _search(query, extractor, question=None):
+    # Default question to query if empty
+    if not question:
+        question = query
+
+    # template = f"""Answer the following question using only the context below. Say 'no answer' when the question can't be answered.
+    #         Question: {question}
+    #         Context: """
+
+    # prompt = PromptTemplate(template=template, input_variables=["question"])
+    # llm_chain = LLMChain(prompt=prompt, llm=extractor)
+
+    # return {"question": question, "answer": llm_chain.run(question)}
+    return extractor([("answer", query, _prompt(question), False)])[0][1]
+
+
+@app.get("/rag")
+def rag(domain: str, question: str):
+    db_exists = _check_if_db_exists(db_path=f"{os.getcwd()}\index\{domain}\documents")
+    print(db_exists)
+    # if db_exists:
+    embeddings = _load_embeddings_from_db(db_exists, domain)
+    # Create extractor instance
+    extractor = Extractor(embeddings, "google/flan-t5-base")
+    # llm = HuggingFaceHub(
+    #     repo_id="google/flan-t5-xxl",
+    #     model_kwargs={"temperature": 1, "max_length": 1000000},
+    # )
+    # else:
+    answer = _search(question, extractor)
+    return {"question": question, "answer": answer}

requirements.txt

@@ -0,0 +1,12 @@
+fastapi==0.74.*
+requests==2.27.*
+uvicorn[standard]==0.17.*
+sentencepiece==0.1.*
+torch==1.12.*
+transformers==4.*
+langchain==0.0.301
+arize-phoenix
+huggingface_hub
+faiss-cpu
+bs4==0.0.1
+streamlit

streamapp.py

@@ -0,0 +1,317 @@
+import os
+import json
+import pandas as pd
+import time
+
+#from hallucinator import HallucinatonEvaluater
+
+from langchain.embeddings import HuggingFaceEmbeddings #for using HugginFace models
+from langchain.chains.question_answering import load_qa_chain
+from langchain import HuggingFaceHub
+from langchain.prompts import PromptTemplate
+
+from langchain.chains import RetrievalQA
+from langchain.callbacks import StdOutCallbackHandler
+
+#from langchain.retrievers import KNNRetriever
+from langchain.storage import LocalFileStore
+from langchain.embeddings import CacheBackedEmbeddings
+from langchain.vectorstores import FAISS
+
+
+from langchain.document_loaders import WebBaseLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+import numpy as np
+import streamlit as st
+import pandas as pd
+# from sklearn import datasets
+# from sklearn.ensemble import RandomForestClassifier
+
+
+tab1, tab2 = st.tabs(["📈 RAG", "🗃 FactVsHallucinate"])
+
+
+
+
+os.environ["HUGGINGFACEHUB_API_TOKEN"] = "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"
+
+# embedding cache
+#store = LocalFileStore("./cache/")
+
+# define embedder
+embedder  = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+#embedder=HuggingFaceHub(repo_id="sentence-transformers/all-mpnet-base-v2")
+#embedder = CacheBackedEmbeddings.from_bytes_store(core_embeddings_model, store)
+
+# define llm
+llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
+#llm=HuggingFaceHub(repo_id="gpt2", model_kwargs={"temperature":1, "max_length":1000000})
+handler =  StdOutCallbackHandler()
+
+# set global variable
+# vectorstore = None
+# retriever = None
+
+
+class HallucinatePromptContext:
+    def __init__(self):
+        self.variables_list = ["query","answer","context"]
+        self.base_template = """In this task, you will be presented with a query, a reference text and an answer. The answer is
+generated to the question based on the reference text. The answer may contain false information, you
+must use the reference text to determine if the answer to the question contains false information,
+if the answer is a hallucination of facts. Your objective is to determine whether the reference text
+contains factual information and is not a hallucination. A 'hallucination' in this context refers to
+an answer that is not based on the reference text or assumes information that is not available in
+the reference text. Your response should be a single word: either "factual" or "hallucinated", and
+it should not include any other text or characters. "hallucinated" indicates that the answer
+provides factually inaccurate information to the query based on the reference text. "factual"
+indicates that the answer to the question is correct relative to the reference text, and does not
+contain made up information. Please read the query and reference text carefully before determining
+your response.
+ 
+    # Query: {query}
+    # Reference text: {context}
+    # Answer: {answer}
+    Is the answer above factual or hallucinated based on the query and reference text?"""
+        
+        
+
+class HallucinatonEvaluater:
+    def __init__(self, item):
+        self.question = item["question"]
+        self.answer = item["answer"]
+        #self.domain = item["domain"]
+        self.context = item["context"]
+        self.llm=HuggingFaceHub(repo_id="google/flan-t5-xxl", model_kwargs={"temperature":1, "max_length":1000000})
+
+    def get_prompt_template(self):
+        prompt = HallucinatePromptContext()
+        template = prompt.base_template
+        varialbles = prompt.variables_list
+        eval_template = PromptTemplate(input_variables=varialbles, template=template)
+        return eval_template
+
+    def evaluate(self):
+        prompt = self.get_prompt_template().format(query = self.question, answer = self.answer, context = self.context)
+        score = self.llm(prompt)
+        return score
+    
+
+
+
+@st.cache_resource
+def initialize_vectorstore():
+
+    webpage_loader = WebBaseLoader("https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us").load()    
+    webpage_chunks = _text_splitter(webpage_loader)
+
+    global vectorstore
+    global retriever
+
+    # store embeddings in vector store
+    vectorstore = FAISS.from_documents(webpage_chunks, embedder)
+    print("vector store initialized with sample doc")
+
+    # instantiate a retriever
+    retriever = vectorstore.as_retriever()
+
+    return retriever
+
+
+def _text_splitter(doc):
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=600,
+        chunk_overlap=50,
+        length_function=len,
+    )
+    return text_splitter.transform_documents(doc)
+
+def _load_docs(path: str):
+    load_doc = WebBaseLoader(path).load()
+    doc = _text_splitter(load_doc)
+    return doc
+
+
+
+
+
+
+def rag_response(response):
+    st.markdown("""<hr style="height:10px;border:none;color:#333;background-color:#333;" /> """, unsafe_allow_html=True)
+    st.subheader('RAG response')  
+    st.text_area(label="user query", value=response["query"], height=30)
+    st.text_area(label="RAG output", value=response["result"])
+    st.text_area(label="Augmented knowledge", value=response["source_documents"])
+    
+    #st.button("Check Hallucination")
+
+
+
+
+
+# Create extractor instance
+def _create_hallucination_scenario(item):
+    score = HallucinatonEvaluater(item).evaluate()
+    return score
+
+def hallu_eval(question: str, answer: str, context: str):
+    print("in hallu eval")
+    hallucination_score = _create_hallucination_scenario({
+            "question": question,
+            "answer": answer,
+            "context": context            
+        }        
+    )
+    print("got hallu score")
+    st.text_area(label="Hallucinated?", value=hallucination_score, height=30)
+    #return {"hallucination_score": hallucination_score}    
+    #time.sleep(10)
+
+
+
+# if 'clicked' not in st.session_state:
+#     print("set state to False")
+#     st.session_state.clicked = False
+
+
+def click_button(response):
+    # print("set state to True")
+    # st.session_state.clicked = True
+
+    hallu_eval(response["query"], response["result"], "blah blah")
+    
+    
+
+
+
+
+
+#st.write(''' # RAG App''')
+
+with tab1:
+
+  with st.form(" RAG with evaluation - scoring & hallucination "):
+    #tab1.subheader(''' # RAG App''')
+    retriever = initialize_vectorstore()
+    
+    #print("lenght in tab1,  ", len(vectorstore.serialize_to_bytes()))
+    options = ["true", "false"]
+    question = st.text_input(label="user question", value="", label_visibility="visible", disabled=False)
+    evaluate = st.selectbox(label="Evaluation",options=options, index=0,  placeholder="Choose an option", disabled=False, label_visibility="visible")
+
+
+
+    if st.form_submit_button("RAG with evaluation"):
+      print("retrie  ,", retriever)
+      chain = RetrievalQA.from_chain_type(
+      llm=llm,
+      retriever=retriever,
+      callbacks=[handler],
+      return_source_documents=True
+      )
+
+      #response = chain("how tredence brought good insight?")
+      response = chain(question)
+      print(response["result"])
+
+    
+      rag_response(response)
+
+      click_button(response)
+
+
+
+
+
+# if st.session_state.clicked:
+  
+#   # The message and nested widget will remain on the page
+#   hallu_eval(response["query"], response["result"], "blah blah")
+  
+
+#   print("in if for hallu")
+      
+
+
+with tab2:
+  
+  
+  
+  with st.form(" LLM-aasisted evaluation of Hallucination"):
+      
+    
+    #print("lenght in tab2,  ", len(vectorstore.serialize_to_bytes()))    
+    question = st.text_input(label="question", value="", label_visibility="visible", disabled=False)
+    answer = st.text_input(label="answer", value="", label_visibility="visible", disabled=False)
+    context = st.text_input(label="context", value="", label_visibility="visible", disabled=False)
+
+
+    if st.form_submit_button("Evaluate"):
+        hallu_eval(question, answer, context)
+      
+  
+
+
+
+
+
+def rag():       
+  print("in rag")
+  options = ["true", "false"]
+  question = st.text_input(label="user question", value="", label_visibility="visible", disabled=False)
+  evaluate = st.selectbox(label="select evaluation",options=options, index=0,  placeholder="Choose an option", disabled=False, label_visibility="visible")
+
+
+
+  if st.button("do RAG"):
+    chain = RetrievalQA.from_chain_type(
+    llm=llm,
+    retriever=retriever,
+    callbacks=[handler],
+    return_source_documents=True
+    )
+
+    #response = chain("how tredence brought good insight?")
+    response = chain(question)
+    print(response["result"])
+
+    # time.sleep(4)
+    
+    # df = px.active_session().get_spans_dataframe()
+    # print(px.active_session())
+    # print(px.active_session().get_spans_dataframe())
+    # print(df.count())        
+    # df_sorted = df.sort_values(by='end_time',ascending=False) 
+
+    # model_input = json.loads(df_sorted[df_sorted["name"] == "LLMChain"]["attributes.input.value"][0])
+    # context = model_input["context"]
+
+    # print(context)
+
+    # if evaluate:
+    #     score = _create_evaluation_scenario({
+    #         "question": question,
+    #         "answer": response['result'],
+    #         "context": context            
+    #     })
+    # else:
+    #     score = "Evaluation is Turned OFF"
+
+    # return {"question": question, "answer": response['result'], "context": context, "score": score}
+    rag_response(response)
+
+    # if st.button("click me"):
+    #   click_button(response)
+
+    click = st.button("Do you want to see more?")
+    if click:
+        st.session_state.more_stuff = True
+
+    if st.session_state.more_stuff:
+        click_button(response)
+        #st.write("Doing more optional stuff")
+
+
+    return(response) 

test.py

@@ -0,0 +1,12 @@
+
+
+a= [
+  "Document(page_content='Key Benefits\\n\\nThe forecasting can help the clients in estimating the shortfalls in installs and activity of the new app, also aid in budget allocation to new app paid marketing\\n\\n\\n\\nResults\\n\\n\\n\\n\\n\\n\\n\\n\\n                        Enhanced the marketing efforts by 30% for successful migration of all existing app users to the new app portal\\n                     \\n\\n\\n\\n\\n\\n\\n\\n\\nTalk To Us\\n\\n\\n\\n\\n\\n\\nFull name *\\n\\n\\n\\nCompany\\n\\n\\n\\nEmail address *\\n\\n\\n\\nPhone number\\n\\n\\n\\nMessage\\n\\n\\n\\nConnect with\\n                  us', metadata={'source': 'https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us', 'title': '\\n        Forecasting App Installs for a Large Retailer in the US - Tredence\\n    ', 'description': 'Forecasting App Installs for a Large Retailer in the US - Tredence', 'language': 'en'})",
+  "Document(page_content='Supply Chain\\n\\n\\nCustomer Analytics\\n\\n\\nCX Management\\n\\n\\nIndustry X.0\\n\\n\\nSupport Solutions\\n\\n\\n\\nSolutions\\n\\nSupply Chain Control Tower\\n\\n\\nTest and Learn Platform (TALP)\\n\\n\\nOn-shelf Availability (OSA)\\n\\n\\nCustomer Cosmos\\n\\n\\nRevenue Growth Management (RGM)\\n\\n\\nMLWorks\\n\\n\\nSustainability Analytics\\n\\n\\n\\n\\nTredence Studio\\n\\n\\n\\nBlogs\\n\\n\\n\\nClient Success\\n\\n\\n\\nLife at Tredence\\n\\n\\n\\nCareers\\n\\n\\n\\nContact us\\n\\n\\n\\n\\n\\nFollow us on\\n\\u2003\\n\\u2003\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nTerms of\\n                            Use\\n\\nPrivacy Policy\\n\\nCookies Policy\\n\\n\\n© Copyright . Tredence.Inc. All rights reserved\\n\\n\\n\\n \\n\\n\\n\\n\\n\\ntzCx9f4MNdCKSWQx6bxOR31y_2csI0A1Kb1rgsiOfdQ=.html', metadata={'source': 'https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us', 'title': '\\n        Forecasting App Installs for a Large Retailer in the US - Tredence\\n    ', 'description': 'Forecasting App Installs for a Large Retailer in the US - Tredence', 'language': 'en'})",
+  "Document(page_content='Connect with\\n                  us\\n\\n\\n\\n\\n\\n\\n\\n\\nWe use cookies for the best user experience on our website, including to personalize content & offerings. By clicking “Accept Cookies” you agree to our use of cookies. For further details please visit our Privacy Policy and Cookies Policy\\n\\n\\n                        Accept cookies\\n                    \\n\\n                        Decline\\n                    \\n\\n\\n \\n\\n\\n\\nIndustries\\n\\nRetail\\n\\n\\nCPG\\n\\n\\nIndustrials\\n\\n\\nHealthcare\\n\\n\\nTMT\\n\\n\\nFinancial Services\\n\\n\\nTravel and Hospitality\\n\\n\\n\\nServices\\n\\nAI Consulting\\n\\n\\nData Engineering\\n\\n\\nSupply Chain\\n\\n\\nCustomer Analytics', metadata={'source': 'https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us', 'title': '\\n        Forecasting App Installs for a Large Retailer in the US - Tredence\\n    ', 'description': 'Forecasting App Installs for a Large Retailer in the US - Tredence', 'language': 'en'})",
+  "Document(page_content='Summary\\n\\nClient was planning to bring in a new stand-alone app in the market which merges the different channels of the business. However, the marketing managers were not sure of the number of optimum marketing efforts that will be required to push the existing app users across BUs to install the new app. Tredence team aimed at forecasting the installs of the new app to help the client in planning the marketing budget and setting goals for the upcoming app migration in FY21\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nApproach', metadata={'source': 'https://www.tredence.com/case-studies/forecasting-app-installs-for-a-large-retailer-in-the-us', 'title': '\\n        Forecasting App Installs for a Large Retailer in the US - Tredence\\n    ', 'description': 'Forecasting App Installs for a Large Retailer in the US - Tredence', 'language': 'en'})"
+]
+
+
+print(type(a[0]))
+print(type(a[0]))