Duplicate from Rams901/Cicero-QA-eval

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+db_full/index.faiss filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Cicero Interactive QA Dev
+emoji: 🏃
+colorFrom: green
+colorTo: gray
+sdk: gradio
+sdk_version: 3.23.0
+app_file: app.py
+pinned: false
+duplicated_from: Rams901/Cicero-QA-eval
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,182 @@
+import gradio as gr
+import numpy as np
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.chains import LLMChain
+from langchain import PromptTemplate
+import re
+import pandas as pd
+from langchain.vectorstores import FAISS
+import requests
+from typing import List
+from langchain.schema import (
+    SystemMessage,
+    HumanMessage,
+    AIMessage
+)
+import os
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.chat_models import ChatOpenAI
+
+from langchain.llms.base import LLM
+from typing import Optional, List, Mapping, Any
+
+import ast
+from utils import ClaudeLLM, extract_website_name, remove_numbers
+
+embeddings = HuggingFaceEmbeddings()
+db = FAISS.load_local('db_full', embeddings)
+
+mp_docs = {}
+llm = ClaudeLLM()
+# ChatOpenAI(
+#             temperature=0,
+#             model='gpt-3.5-turbo-16k'
+#         )
+
+
+def add_text(history, text):
+
+    print(history)
+    history = history + [(text, None)]
+
+    return history, ""
+
+pipeline = {'claude': (ClaudeLLM(), 0), 'gpt-3.5': (ChatOpenAI(temperature=0,model='gpt-3.5-turbo-16k'), 65), 'gpt-4': (ChatOpenAI(temperature=0, model='gpt-4'), 30)}
+
+def retrieve_thoughts_test(query):
+
+    # print(db.similarity_search_with_score(query = query, k = k, fetch_k = k*10))
+    docs_with_score = db.similarity_search_with_score(query = query, k = len(db.index_to_docstore_id.values()), fetch_k = len(db.index_to_docstore_id.values()))
+    df = pd.DataFrame([dict(doc[0])['metadata'] for doc in docs_with_score], )
+    df = pd.concat((df, pd.DataFrame([dict(doc[0])['page_content'] for doc in docs_with_score], columns = ['page_content'])), axis = 1)
+    df = pd.concat((df, pd.DataFrame([doc[1] for doc in docs_with_score], columns = ['score'])), axis = 1)
+
+  # TO-DO: What if user query doesn't match what we provide as documents
+
+    tier_1 = df[df['score'] < 0.7]
+    tier_2 = df[(df['score'] < 0.95) * (df["score"] > 0.7)]
+
+    chunks_1 = tier_1.groupby(['title', 'url', '_id']).apply(lambda x: "\n...\n".join(x.sort_values('id')['page_content'].values)).values
+    tier_1_adjusted = tier_1.groupby(['title', 'url', '_id']).first().reset_index()[['_id', 'title', 'url']]
+    tier_1_adjusted['ref'] = range(1, len(tier_1_adjusted) + 1 )
+    tier_1_adjusted['content'] = chunks_1
+
+    # chunks_2 = tier_2.groupby(['title', 'url', '_id']).apply(lambda x: "\n...\n".join(x.sort_values('id')['page_content'].values)).values
+    # tier_2_adjusted = tier_2.groupby(['title', 'url', '_id']).first().reset_index()[['_id', 'title', 'url']]
+    # tier_2_adjusted['content'] = chunks_2
+
+    print(len(tier_1_adjusted))
+  # tier_1 = [doc[0]  for doc in docs if ((doc[1] < 1))][:5]
+  # tier_2 = [doc[0]  for doc in docs if ((doc[1] > 0.7)*(doc[1] < 1.5))][10:15]
+
+    reference = tier_1_adjusted[['ref', 'url', 'title']].to_dict('records')
+
+    return {'tier 1':tier_1_adjusted, }, reference
+
+def qa_retrieve_test(query, llm, thoughts):
+
+    llm, n = pipeline[llm]
+
+    docs = ""
+
+    # global db
+    # print(db)
+
+    # global mp_docs
+    # thoughts = retrieve_thoughts(query, n)
+
+    # if not(thoughts):
+
+    #     if mp_docs:
+    #         thoughts = mp_docs
+
+    # else:
+    #     mp_docs = thoughts
+
+    tier_1 = thoughts['tier 1']
+    # tier_2 = thoughts['tier 2']
+
+    # reference = tier_1[['ref', 'url', 'title']].to_dict('records')
+
+    if n:
+      tier_1 = tier_1[:min(len(tier_1), n)]
+
+    tier_1 = list(tier_1.apply(lambda x: f"[{int(x['ref'])}] title: {x['title']}\n Content: {x.content}", axis = 1).values)
+    print(len(tier_1))
+    # tier_2 = list(tier_2.apply(lambda x: f"title: {x['title']}\n Content: {x.content}", axis = 1).values)
+
+    # print(f"QUERY: {query}\nTIER 1: {tier_1}\nTIER2: {tier_2}")
+    # print(f"DOCS RETRIEVED: {mp_docs.values}")
+
+    # Cynthesis Generation
+    session_prompt = """ A bot that is open to discussions about different cultural, philosophical and political exchanges. You will use do different analysis to the articles provided to me. Stay truthful and if you weren't provided any resources give your oppinion only."""
+    task = """Create a coherent synthesis in which you use references to the id of articles provided and relevant to the query.
+
+    Follow the example structure:
+
+    The best wine to pair with steak depends on the cut of steak and the preparation. Here are some general guidelines for pairing wine with steak:
+    - Choose a dry red wine. The rule of thumb is to choose dry red wines
+    -  leaner cuts of meat pair with lighter wines, while richer, fattier cuts pair up with high tannin wines that can cut through the fat [1].
+    - Consider the cut of steak. Lighter red wines tend to go best with the leaner cuts of steak such as filet mignon, while more marbled, higher fat cuts of meat like a rib eye do well when accompanied by more robust red wines [3].
+    - Take into account the preparation. For a spiced steak, go for a wine with lots of fruit to balance out the heat, like an Old Vine Zinfandel. And if you're drowning your steak in a decadent sauce, find a wine with enough body to stand up to it, like a Cabernet Sauvignon [5].
+    - Popular wine choices include Cabernet Sauvignon, Pinot Noir, Zinfandel, Malbec, Syrah, and Merlot [2].
+    Remember, the goal is to choose a wine that complements the cut of steak and not overwhelm or take away from the flavor of the meat [3].
+    "
+    """
+
+    prompt = PromptTemplate(
+            input_variables=["query", "task", "session_prompt", "articles"],
+            template="""
+            You are a {session_prompt}
+            {task}
+
+            query: {query}
+
+            Articles:
+            {articles}
+
+            Make sure to quote the article used if the argument corresponds to the query.
+            Use careful reasoning to explain your answer and give your conclusion about this.
+            """,
+        )
+
+    # llm = BardLLM()
+    chain = LLMChain(llm=llm, prompt = prompt)
+
+    response = chain.run(query=query, articles="\n".join(tier_1), session_prompt = session_prompt, task = task)
+
+    return response
+
+def greet(query):
+
+    llms = ['claude', 'gpt-3.5', 'gpt-4']
+    thoughts, reference = retrieve_thoughts_test(query)
+    
+    print(reference)
+    # get docs here then adjust inside the qa retrieve test because we're always having the same query so same docs to get.
+    
+    result = [qa_retrieve_test(query, llm, thoughts) for llm in llms]
+    result.append({'reference': reference,})
+    
+    print(len(result), result[-1], result[-2])
+
+    _1, _2, _3, _4 = result
+    
+    
+    return _1, _2, _3, _4
+
+
+examples = [
+    ["How to be happy"],
+    ]
+
+demo = gr.Interface(fn=greet, title="cicero-cynthesis-eval", inputs="text",
+                    outputs=[gr.components.Textbox(lines=3, label="Claude"),
+                             gr.components.Textbox(lines=3, label="GPT3.5"),
+                             gr.components.Textbox(lines=3, label="GPT-4"),
+                             gr.components.JSON( label="Reference")],
+                   examples=examples)
+
+demo.queue(concurrency_count = 4)
+demo.launch()
+

db_full/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9051c0122a839f58dc047ba2145fd887b64a33ecd746bd17aec950ca044f0653
+size 354250797

db_full/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ccf07d4b39015b8e101152d341883a99d311f22ab7dfc5edba88277041b9179
+size 102244751

requirements.txt

@@ -0,0 +1,9 @@
+pandas
+langchain
+openai
+FAISS-gpu
+tiktoken
+transformers
+sentence_transformers
+bson
+anthropic==0.2.10

utils.py

@@ -0,0 +1,49 @@
+from langchain.llms.base import LLM
+from typing import Optional, List, Mapping, Any
+import anthropic
+from urllib.parse import urlparse
+import os
+class ClaudeLLM(LLM):
+
+    @property
+    def _llm_type(self) -> str:
+
+        return "custom"
+
+    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
+
+
+        client = anthropic.Client(os.environ['ANTHROPIC_KEY'])
+
+
+        # How about the formatted prompt?
+        prompt_formatted = (
+        f"{anthropic.HUMAN_PROMPT}{prompt}\n{anthropic.AI_PROMPT}"
+        )
+
+
+        response = client.completion(
+        prompt=prompt_formatted,
+        stop_sequences=[anthropic.HUMAN_PROMPT],
+        model="claude-instant-v1-100k",
+        max_tokens_to_sample=100000,
+        temperature=0.3,
+         )
+
+        return response["completion"]
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        """Get the identifying parameters."""
+        return {
+
+        }
+        
+def remove_numbers(question):
+    return question.translate(str.maketrans('', '', '0123456789'))
+
+def extract_website_name(url):
+    parsed_url = urlparse(url)
+    if parsed_url.netloc.startswith("www."):
+        return parsed_url.netloc.split("www.")[1].split(".")[0]
+    return parsed_url.netloc.split(".")[0]