Update utils.py

utils.py

@@ -15,6 +15,9 @@ import sys
 import gc
 from pygments.lexers import guess_lexer, ClassNotFound
 import time
+import json
+import operator
+from typing import Annotated, Sequence, TypedDict
 
 import gradio as gr
 from pypinyin import lazy_pinyin
@@ -37,9 +40,22 @@ from langchain.llms import HuggingFaceTextGenInference
 from langchain.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings, HuggingFaceBgeEmbeddings, HuggingFaceInferenceAPIEmbeddings
 from langchain.tools import DuckDuckGoSearchRun
 from langchain.retrievers.tavily_search_api import TavilySearchAPIRetriever
-
+from typing import Dict, TypedDict
+from langchain_core.messages import BaseMessage
 from langchain.embeddings.openai import OpenAIEmbeddings
 from langchain.prompts import PromptTemplate
+
+from langchain import hub
+from langchain.output_parsers.openai_tools import PydanticToolsParser
+from langchain.prompts import PromptTemplate
+from langchain.schema import Document
+from langchain_community.tools.tavily_search import TavilySearchResults
+from langchain_community.vectorstores import Chroma
+from langchain_core.messages import BaseMessage, FunctionMessage
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.pydantic_v1 import BaseModel, Field
+from langchain_core.runnables import RunnablePassthrough
+from langchain_core.utils.function_calling import convert_to_openai_tool
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import Chroma
 from chromadb.errors import InvalidDimensionException
@@ -121,6 +137,13 @@ WEB_URL       = "https://openai.com/research/gpt-4"
 YOUTUBE_URL_1 = "https://www.youtube.com/watch?v=--khbXchTeE"
 YOUTUBE_URL_2 = "https://www.youtube.com/watch?v=hdhZwyf24mE"
 #YOUTUBE_URL_3 = "https://www.youtube.com/watch?v=vw-KWfKwvTQ"
+#spezielle Webseiten als Datenbasis laden
+urls = [
+    "https://kkg.hamburg.de/unser-leitbild/"
+    "https://kkg.hamburg.de/unsere-schulcharta/",
+    "https://kkg.hamburg.de/koordination-unterrichtsentwicklung/",
+    "https://kkg.hamburg.de/konzept-medien-und-it-am-kkg/",
+]
 
 
 #################################################
@@ -202,14 +225,24 @@ def document_loading_splitting():
     pdf_loader = create_directory_loader('.pdf', CHROMA_PDF)
     word_loader = create_directory_loader('.word', CHROMA_WORD)
     
-    # Load the files
+    # Load the files - pdf und word
     pdf_documents = pdf_loader.load()
     word_documents = word_loader.load()
 
-    #alle zusammen in docs...
-    docs.extend(pdf_documents)
-    docs.extend(word_documents)
+    #urls -zum Thema passend
+    docs_web = [WebBaseLoader(url).load() for url in urls]
+    docs_list = [item for sublist in docs_web for item in sublist]
 
+    #alle zusammen in docs...
+    #pdf_docs als Liste umschreiben, um es mit den anderen Materialien in der docs_list zusammenzubringen
+    pdf_list = [pdf_documents]
+    word_list = [word_documents]
+    #die neuen Dokeumente der Gesamt-Liste von material hinzufügen
+    #alle zusammen in docs...
+    for doc in pdf_list:
+        docs_list.extend(doc)
+    for doc in word_list:
+        docs_list.extend(doc)
 
     #andere loader...
     # Load PDF
@@ -223,20 +256,29 @@ def document_loading_splitting():
     #docs.extend(loader.load())
     ################################
     # Document splitting
-    text_splitter = RecursiveCharacterTextSplitter(chunk_overlap = 150, chunk_size = 1500)
-    splits = text_splitter.split_documents(docs)
+    text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=1500, chunk_overlap=250)
+    doc_splits = text_splitter.split_documents(docs_list)
+    #text_splitter = RecursiveCharacterTextSplitter(chunk_overlap = 150, chunk_size = 1500)
+    #splits = text_splitter.split_documents(docs)
     
     return splits
 
 ###########################################
 #Chroma DB die splits ablegen - vektorisiert...
 def document_storage_chroma(splits):
+    # Add to vectorDB
+    vectorstore = Chroma.from_documents(documents=splits,collection_name="rag-chroma",embedding=OpenAIEmbeddings(disallowed_special = ()),  persist_directory = PATH_WORK + CHROMA_DIR)
+    retriever = vectorstore.as_retriever(search_kwargs = {"k": 4})
+
     #OpenAi embeddings----------------------------------
-    Chroma.from_documents(documents = splits, embedding = OpenAIEmbeddings(disallowed_special = ()),  persist_directory = PATH_WORK + CHROMA_DIR)  
+    #Chroma.from_documents(documents = splits, embedding = OpenAIEmbeddings(disallowed_special = ()),  persist_directory = PATH_WORK + CHROMA_DIR)  
 
     #HF embeddings--------------------------------------
     #Chroma.from_documents(documents = splits, embedding = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", model_kwargs={"device": "cpu"}, encode_kwargs={'normalize_embeddings': False}),  persist_directory = PATH_WORK + CHROMA_DIR)  
-    
+    return vectorstore, retriever
+
+
+"""
 #Mongo DB die splits ablegen - vektorisiert...
 def document_storage_mongodb(splits):
     MongoDBAtlasVectorSearch.from_documents(documents = splits,
@@ -288,6 +330,9 @@ def document_retrieval_mongodb(llm, prompt):
                                                          OpenAIEmbeddings(disallowed_special = ()),
                                                          index_name = MONGODB_INDEX_NAME)
     return db
+"""
+
+
 
 ###############################################
 #Langchain anlegen
@@ -1005,4 +1050,315 @@ class CustomDocTemplate(SimpleDocTemplate):
         current_date = datetime.now().strftime("%Y-%m-%d")
         # Passen Sie hier die Positionierung an Ihre Bedürfnisse an
         self.canv.drawRightString(550, 800, current_date)  # Position anpassen
-        self.canv.restoreState()
+        self.canv.restoreState()
+
+
+######################################################################
+#Zustandsgraph für Langgraph, um RAG zu implementieren mit verschiedenen Zuständen
+#die durchlaufen werden:
+#1. Dokumente aus vektorstore bekommen
+#2. die Relevanz ddr Dokuemnte einschätzen
+#3. wenn zu wenig relevante infos: Frage neu formulieren
+#4. nochmal 1. und 2.
+#5. wenn nun genug relevante Dokumente: Anfrage an Modell mit den Doks
+#6. wenn nicht gneug Dokumente relevant: Anfrage an Modell ohne Doks
+#####################################################################
+
+# Zustandsgraph als Datenstruktur zum Umsetzen
+class GraphState(TypedDict):
+    """
+    Represents the state of our graph.
+
+    Attributes:
+        keys: A dictionary where each key is a string.
+    """
+    keys: Dict[str, any]
+
+
+#Methoden, um den Graph und die Zustände umzusetzen
+### Nodes ###
+# die Knoten des Graphen definieren, die der Reihe noch (bzw. je nach Outcome des Vorgänger Knotens) durchlaufen werden
+def retrieve(state):
+    """
+    Retrieve documents
+    Args:
+        state (dict): The current graph state
+    Returns:
+        state (dict): New keys added to state: documents, that contains retrieved documents, der wievielte Versuch gemacht wird
+    """
+    print("---RETRIEVE ---")
+    state_dict = state["keys"]
+    question = state_dict["question"]
+    documents = retriever.get_relevant_documents(question)
+    second_trial="ja"
+    if 'second_trial' in state_dict:
+        print("second time")
+        second_trail = "ja"
+    else:
+        print("first time")
+        second_trial="nein"
+    return {"keys": {"documents": documents, "second_trial":second_trial, "question": question, }}
+
+
+def retrieve_redirect(state):
+    """
+    Retrieve redirect (wenn nach transform:question neues retrieven gemacht werden soll)
+    Args:
+        state (dict): The current graph state
+    Returns:
+        state (dict): New key added to state: second_trial 
+    """
+    print("---RETRIEVE REDIRECT---")
+    second_trial="ja"
+    state_dict = state["keys"]
+    question= state_dict["question"]
+    documents= state_dict["documents"]
+    return {"keys": {"documents": documents, "second_trial":second_trial, "question": question, }}
+
+
+
+def generate(state):
+    """
+    Generate answer
+    Args:
+        state (dict): The current graph state
+    Returns:
+        state (dict): New key added to state, generation, that contains LLM generation
+    """
+    print("---GENERATE---")
+    state_dict = state["keys"]
+    question = state_dict["question"]
+    documents = state_dict["documents"]
+
+    # Prompt
+    prompt = hub.pull("rlm/rag-prompt")
+
+    # LLM
+    llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.3, streaming=True)
+
+    # Post-processing
+    #def format_docs(docs):
+        #return "\n\n".join(doc.page_content for doc in docs)
+
+    # Chain
+    rag_chain = prompt | llm | StrOutputParser()
+
+    # Run
+    generation = rag_chain.invoke({"context": documents, "question": question})
+    return {
+        "keys": {"documents": documents, "question": question, "generation": generation}
+    }
+
+def generate_ohne(state):
+    """
+    Generate answer
+    Args:
+        state (dict): The current graph state
+    Returns:
+        state (dict): New key added to state, generation, that contains LLM generation
+    """
+    print("---GENERATE OHNE---")
+    state_dict = state["keys"]
+    question = state_dict["question"]
+    #documents = state_dict["documents"]
+
+    # Prompt
+    prompt = PromptTemplate(
+        template="""\Antworte in deutsch, wenn es nicht explizit anders gefordert wird. Wenn du die Antwort nicht kennst, antworte direkt, dass du es nicht weißt.
+        Versuche nicht es zu umschreiben. Versuche nicht, die Antwort zu erfinden oder aufzumocken. Halte die Antwort kurz aber ausführlich genug und exakt. \n\n
+        Hier ist die Useranfrage: {question} """,
+        input_variables=["question"])
+
+    # LLM
+    llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.3, streaming=True)
+
+    # Post-processing
+    #def format_docs(docs):
+        #return "\n\n".join(doc.page_content for doc in docs)
+
+    # Chain
+    llm_chain = prompt | llm | StrOutputParser()
+
+    # Run
+    generation = llm_chain.invoke({ "question": question})
+    return {
+        "keys": {"question": question, "generation": generation}
+    }
+
+
+def grade_documents(state):
+    """
+    Determines whether the retrieved documents are relevant to the question.
+    Args:
+        state (dict): The current graph state
+    Returns:
+        state (dict): Updates documents key with relevant documents
+    """
+
+    print("---CHECK RELEVANCE---")
+    state_dict = state["keys"]
+    question = state_dict["question"]
+    documents = state_dict["documents"]
+    second_trial =state_dict["second_trial"]
+
+    # Data model
+    class grade(BaseModel):
+        """Binary score for relevance check."""
+        binary_score: str = Field(description="Relevanz Bewertung 'ja' oder 'nein'")
+
+    # LLM
+    model = ChatOpenAI(temperature=0.3, model="gpt-4-0125-preview", streaming=True)
+
+    # Tool
+    grade_tool_oai = convert_to_openai_tool(grade)
+
+    # LLM with tool and enforce invocation
+    llm_with_tool = model.bind(
+        tools=[convert_to_openai_tool(grade_tool_oai)],
+        tool_choice={"type": "function", "function": {"name": "grade"}},
+    )
+
+    # Parser
+    parser_tool = PydanticToolsParser(tools=[grade])
+
+    # Prompt
+    prompt = PromptTemplate(
+        template="""Du bist ein Bewerter, der die Relevanz von einem erhaltenen Dokument zu einer Nutzeranfrage bewerten soll. \n
+        Hier ist das erhaltene Dokument: \n\n {context} \n\n
+        Hier ist die Nutzeranfrage: {question} \n
+        Wenn das erhaltene Dokument Keywörter oder semantische Bedeutung in Bezug auf die Nutzeranfrage hat, bewerte es als relevant. \n
+        Gib eine binäre Bewertung von 'ja' oder 'nein' Bewertung, um anzuzeigen ob das Dokuemnt relevant ist zur Nutzeranfrage oder nicht.""",
+        input_variables=["context", "question"],
+    )
+
+    # Chain
+    chain = prompt | llm_with_tool | parser_tool
+
+    # Score
+    filtered_docs = []
+    anzahl_relevant = 0
+    search = "nein"  # Default do not opt for re-questioning to supplement retrieval
+    for d in documents:
+        score = chain.invoke({"question": question, "context": d.page_content})
+        grade = score[0].binary_score
+        if grade == "ja":
+            #search = "nein"  # mind. ein relevantes Dokument -> keine Websuche nötig
+            print("---Bewertung: Dokument ist relevant---")
+            anzahl_relevant = anzahl_relevant +1
+            filtered_docs.append(d)
+        else:
+            print("---Bewertung: Dokument irrelevant---")
+            search = "ja"  # mind ein Dokument irrelevant -> Frage umformulieren
+            continue
+    #wenn mehrheit der Dokumente relevant -> generieren starten damit
+    if (anzahl_relevant>= len(documents)/2):
+        search = "nein"
+    print("second trial grade_docs:.....................")
+    print(second_trial)
+    return {
+        "keys": {
+            "documents": filtered_docs,
+            "question": question,
+            "search_again": search,
+            "second_trial": second_trial
+        }
+    }
+
+
+def transform_query(state):
+    """
+    Transform the query to produce a better question.
+    Args:
+        state (dict): The current graph state
+    Returns:
+        state (dict): Updates question key with a re-phrased question
+    """
+
+    print("---TRANSFORM QUERY---")
+    state_dict = state["keys"]
+    question = state_dict["question"]
+    documents = state_dict["documents"]
+
+    # Create a prompt template with format instructions and the query
+    prompt = PromptTemplate(
+        template="""Du generierst Fragen, die optimiert sind für das Retrieval von Dokumenten. \n
+        Schaue auf den input und versuche die zugrundeliegende Absicht / Bedeutung zu bewerten. \n
+        Hier ist die ursprüngliche Frage:
+        \n ------- \n
+        {question} 
+        \n ------- \n
+        Formuliere eine verbesserte Frage: """,
+        input_variables=["question"],
+    )
+
+    # Grader
+    model = ChatOpenAI(temperature=0, model="gpt-4-0125-preview", streaming=True)
+
+    # Prompt
+    chain = prompt | model | StrOutputParser()
+    better_question = chain.invoke({"question": question})
+    second_trial="ja"
+
+    return {"keys": {"documents": documents, "question": better_question, "second_trial" : second_trial}}
+
+#websuche zur Zeit nicht in gebrauch
+def web_search(state):
+    """
+    Web search based on the re-phrased question using Tavily API.
+    Args:
+        state (dict): The current graph state
+    Returns:
+        state (dict): Updates documents key with appended web results
+    """
+
+    print("---WEB Suche---")
+    state_dict = state["keys"]
+    question = state_dict["question"]
+    documents = state_dict["documents"]
+
+    tool = TavilySearchResults()
+    docs = tool.invoke({"query": question})
+    web_results = "\n".join([d["content"] for d in docs])
+    web_results = Document(page_content=web_results)
+    documents.append(web_results)
+
+    return {"keys": {"documents": documents, "question": question}}
+
+
+### Edges
+
+
+def decide_to_generate(state):
+    """
+    Determines whether to generate an answer or re-generate a question for a new retriever question or generate without documents attached
+    Args:
+        state (dict): The current state of the agent, including all keys.
+    Returns:
+        str: Next node to call
+    """
+
+    print("---ENTSCHEIDE ZU GENERIEREN---")
+    print("current state")
+    print(state["keys"])
+    print("-------------------------------")
+    state_dict = state["keys"]
+    question = state_dict["question"]
+    filtered_documents = state_dict["documents"]
+    search_again = state_dict["search_again"]
+    second_trial=state_dict["second_trial"]
+
+
+    if search_again == "ja" :
+        if (not second_trial == "ja"):
+            # All documents have been filtered check_relevance
+            # We will re-generate a new query
+            print("---ENTSCHEIDUNG: VERÄNDERE DIE FRAGE ---")
+            return "transform_query"
+        else:
+            # keine neue frage, sondern generieren - ohne Dokumente anzuhängen
+            print("---ENTSCHEIDUNG: Generiere ohne Dokumente---")
+            return "generate_ohne" 
+    else:
+        # We have relevant documents, so generate answer
+        print("---ENTSCHEIDUNG: GENERIERE---")
+        return "generate"