Update app.py

app.py

@@ -23,11 +23,48 @@ from typing import Optional, List, Mapping, Any
 import ast
 from utils import ClaudeLLM
 
-embeddings = HuggingFaceEmbeddings()
-db_art = FAISS.load_local('db_art', embeddings)
-db_yt = FAISS.load_local('db_yt', embeddings)
+from qdrant_client import models, QdrantClient
+from sentence_transformers import SentenceTransformer
+
+# embeddings = HuggingFaceEmbeddings()
+# db_art = FAISS.load_local('db_art', embeddings)
+# db_yt = FAISS.load_local('db_yt', embeddings)
 mp_docs = {}
 
+qdrant = QdrantClient(
+    "https://0a1b865d-8291-41ef-8c29-ca6c35e26391.us-east4-0.gcp.cloud.qdrant.io:6333",
+    prefer_grpc=True,
+    api_key=os.env['Qdrant_Api_Key']
+)
+encoder = SentenceTransformer('BAAI/bge-large-en-v1.5')
+def q_retrieve_thoughts(query, n, db = "articles"):
+    qdrant.search(
+    collection_name="articles",
+    query_vector=encoder.encode("Will Russia win the war in Ukraine?").tolist(),
+    limit=4000 # TO-DO: know the right number of thoughts existing maybe using get_collection
+)
+    df = pd.DataFrame.from_records([dict(hit) for hit in hits] )
+    payload = pd.DataFrame(list(df['payload'].values[:]))
+    
+    # payload.rename(columns = ['id': 'order_id'])
+    # payload['id'] = df['id']
+    
+    payload['score'] = df['score']
+    del df
+    payload.sort_values('score', ascending = False, inplace = True)
+    
+    tier_1 = payload
+    
+    chunks_1 = tier_1.groupby(['_id', ]).apply(lambda x: "\n...\n".join(x.sort_values('id')['page_content'].values)).values
+    score = tier_1.groupby(['_id', ]).apply(lambda x: x['score'].mean()).values
+    
+    tier_1_adjusted = tier_1.groupby(['_id', ]).first().reset_index()[[ 'title', 'url', 'author']]
+    tier_1_adjusted['content'] = list(chunks_1)
+    tier_1_adjusted['score'] = score
+    tier_1_adjusted = tier_1_adjusted[tier_1_adjusted['score']>0.5]
+    tier_1_adjusted.sort_values('score', ascending = False, inplace = True)
+    return {'tier 1':tier_1_adjusted, }
+
 def retrieve_thoughts(query, n, db):
 
     # print(db.similarity_search_with_score(query = query, k = k, fetch_k = k*10))
@@ -66,7 +103,7 @@ def qa_retrieve_art(query,):
     global db_art
 
     global mp_docs
-    thoughts = retrieve_thoughts(query, 0, db_art)
+    thoughts = q_retrieve_thoughts(query, 0)
     if not(thoughts):
 
         if mp_docs:
@@ -116,6 +153,6 @@ ref_art = gr.Interface(fn=qa_retrieve_art, label="Articles",
 ref_yt = gr.Interface(fn=qa_retrieve_yt, label="Youtube",
                      inputs=gr.inputs.Textbox(lines=5, label="what would you like to learn about?"),
                      outputs = gr.components.JSON(label="youtube"),title = "youtube", examples=examples)
-demo = gr.Parallel( ref_art, ref_yt)
+demo = gr.Parallel( ref_art,)
 
 demo.launch()