Upload 6 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_index/index.faiss filter=lfs diff=lfs merge=lfs -text

analysis.py

@@ -0,0 +1,84 @@
+from typing import List
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+import streamlit as st
+import matplotlib.pyplot as plt
+
+def calculate_word_overlaps(documents: List[str], query: str):
+    """
+    Calculate the average word overlaps between documents and the query.
+    """
+    query_words = set(query.lower().split())
+    word_overlaps = []
+
+    for doc in documents:
+        doc_words = set(doc.lower().split())
+        overlap = len(query_words.intersection(doc_words))
+        word_overlaps.append(overlap)
+
+    if len(word_overlaps) > 0:
+        average_word_overlap = np.mean(word_overlaps)
+    else:
+        average_word_overlap = 0.0
+
+    return average_word_overlap
+
+def calculate_duplication_rate(documents: List[str]):
+    """
+    Calculate the duplication rate among a list of documents.
+    """
+    total_words_set = set()
+    total_words = 0
+
+    for doc in documents:
+        doc_words = doc.lower().split()
+        total_words_set.update(doc_words)
+        total_words += len(doc_words)
+
+    if total_words > 0:
+        duplication_rate = (total_words - len(total_words_set)) / total_words
+    else:
+        duplication_rate = 0.0
+
+    return duplication_rate
+
+
+def cosine_similarity_score(documents: List[str], query: str):
+    """
+    Calculate cosine similarity between the query and each document.
+    """
+    tfidf_vectorizer = TfidfVectorizer()
+    tfidf_matrix = tfidf_vectorizer.fit_transform([query] + documents)
+    cosine_similarities = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:])
+    return cosine_similarities[0]
+
+def jaccard_similarity_score(documents: List[str], query: str):
+    """
+    Calculate Jaccard similarity between the query and each document.
+    """
+    query_words = set(query.lower().split())
+    jaccard_similarities = []
+
+    for doc in documents:
+        doc_words = set(doc.lower().split())
+        intersection_size = len(query_words.intersection(doc_words))
+        union_size = len(query_words.union(doc_words))
+        jaccard_similarity = intersection_size / union_size if union_size > 0 else 0
+        jaccard_similarities.append(jaccard_similarity)
+
+    return jaccard_similarities
+
+
+def display_similarity_results(cosine_scores, jaccard_scores, title):
+    st.subheader(f"{title} - Cosine Similarity to Query")
+    plt.bar(range(len(cosine_scores)), cosine_scores)
+    plt.xlabel("Documents")
+    plt.ylabel("Cosine Similarity")
+    st.pyplot(plt)
+
+    st.subheader(f"{title} - Jaccard Similarity to Query")
+    plt.bar(range(len(jaccard_scores)), jaccard_scores, color='orange')
+    plt.xlabel("Documents")
+    plt.ylabel("Jaccard Similarity")
+    st.pyplot(plt)

app.py

@@ -0,0 +1,125 @@
+import streamlit as st
+import pickle
+import os
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.retrievers import EnsembleRetriever, BM25Retriever, ContextualCompressionRetriever
+from langchain.document_transformers import EmbeddingsRedundantFilter
+from langchain.retrievers.document_compressors import EmbeddingsFilter, DocumentCompressorPipeline
+from langchain.text_splitter import CharacterTextSplitter
+
+
+from analysis import calculate_word_overlaps, calculate_duplication_rate, cosine_similarity_score, jaccard_similarity_score, display_similarity_results
+
+
+
+with open("docs_data.pkl", "rb") as file:
+    docs = pickle.load(file)
+
+metadata_list = []
+unique_metadata_list = []
+seen = set()
+
+embeddings = HuggingFaceEmbeddings(model_name = "thenlper/gte-large")
+vectorstore = FAISS.load_local("faiss_index", embeddings)
+retriever = vectorstore.as_retriever(search_type="similarity")
+
+splitter = CharacterTextSplitter(chunk_size=300, chunk_overlap=0, separator=". ")
+redundant_filter = EmbeddingsRedundantFilter(embeddings=embeddings)
+relevant_filter = EmbeddingsFilter(embeddings=embeddings, similarity_threshold=0.76)
+pipeline_compressor = DocumentCompressorPipeline(
+    transformers=[splitter, redundant_filter, relevant_filter]
+)
+
+bm25_retriever = BM25Retriever.from_texts(docs)
+
+st.title("Document Retrieval App")
+
+vecotstore_k = st.number_input("Set k value for Dense Retriever:", value=5, min_value=1, step=1)
+bm25_k = st.number_input("Set k value for sparse Retriever:", value=2, min_value=1, step=1)
+
+retriever.search_kwargs["k"] = vecotstore_k
+bm25_retriever.k = bm25_k
+
+compressed_retriever = ContextualCompressionRetriever(base_compressor=pipeline_compressor, base_retriever=retriever)
+bm25_compression_retriever = ContextualCompressionRetriever(base_compressor=pipeline_compressor, base_retriever=bm25_retriever)
+
+query = st.text_input("Enter a query:", "what is a horizontal conflict")
+
+if st.button("Retrieve Documents"):
+
+    compressed_ensemble_retriever = EnsembleRetriever(retrievers=[compressed_retriever, bm25_compression_retriever], weights=[0.5, 0.5])
+    ensemble_retriever = EnsembleRetriever(retrievers=[retriever, bm25_retriever], weights=[0.5, 0.5])
+
+    with st.expander("Retrieved Documents"):
+        col1, col2 = st.columns(2)
+
+        with col1:
+            st.header("Without Compression")
+            normal_results = ensemble_retriever.get_relevant_documents(query)
+            for doc in normal_results:
+                st.write(doc.page_content)
+                st.write("---")
+
+        with col2:
+            st.header("With Compression")
+            compressed_results = compressed_ensemble_retriever.get_relevant_documents(query)
+            for doc in compressed_results:
+                st.write(doc.page_content)
+                st.write("---")
+
+                if hasattr(doc, 'metadata'):
+                    metadata = doc.metadata
+                    metadata_list.append(metadata)
+
+            for metadata in metadata_list:
+                metadata_tuple = tuple(metadata.items())
+                if metadata_tuple not in seen:
+                    unique_metadata_list.append(metadata)
+                    seen.add(metadata_tuple)
+
+            st.write(unique_metadata_list)
+
+    with st.expander("Analysis"):
+        st.write("Analysis of Retrieval Results")
+
+        total_words_normal = sum(len(doc.page_content.split()) for doc in normal_results)
+        total_words_compressed = sum(len(doc.page_content.split()) for doc in compressed_results)
+        reduction_percentage = ((total_words_normal - total_words_compressed) / total_words_normal) * 100
+
+        col1, col2 = st.columns(2)
+        
+
+        st.write(f"Total words in documents (Normal): {total_words_normal}")
+        st.write(f"Total words in documents (Compressed): {total_words_compressed}")
+        st.write(f"Reduction Percentage: {reduction_percentage:.2f}%")
+
+        average_word_overlap_normal = calculate_word_overlaps([doc.page_content for doc in normal_results], query)
+        average_word_overlap_compressed = calculate_word_overlaps([doc.page_content for doc in compressed_results], query)
+
+        duplication_rate_normal = calculate_duplication_rate([doc.page_content for doc in normal_results])
+        duplication_rate_compressed = calculate_duplication_rate([doc.page_content for doc in compressed_results])
+        
+        cosine_scores_normal = cosine_similarity_score([doc.page_content for doc in normal_results], query)
+        jaccard_scores_normal = jaccard_similarity_score([doc.page_content for doc in normal_results], query)
+
+        cosine_scores_compressed = cosine_similarity_score([doc.page_content for doc in compressed_results], query)
+        jaccard_scores_compressed = jaccard_similarity_score([doc.page_content for doc in compressed_results], query)
+
+        with col1:
+            st.subheader("Normal")
+
+            st.write(f"Average Word Overlap: {average_word_overlap_normal:.2f}")
+            st.write(f"Duplication Rate: {duplication_rate_normal:.2%}")
+
+            st.write("Results without Compression:")
+            display_similarity_results(cosine_scores_normal, jaccard_scores_normal, "")            
+
+        with col2:
+            st.subheader("Compressed")
+
+            st.write(f"Average Word Overlap: {average_word_overlap_compressed:.2f}")
+            st.write(f"Duplication Rate: {duplication_rate_compressed:.2%}")     
+
+            st.write("Results with Compression:")
+            display_similarity_results(cosine_scores_compressed, jaccard_scores_compressed, "")

docs_data.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58e85d1679c567117937df49e7294968f0851ec52c3e6858f82cea45f0034a9f
+size 7965757

faiss_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcc64cacbcff655b1220247180d2944d22175f8670f28a0d04520e49d2eaa793
+size 19587117

faiss_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c8ea8d70b85579b394443d0caddb4fd12601533b15a259f9d7a722a86537841
+size 8545256

requirements.txt

@@ -0,0 +1,9 @@
+langchain
+openai
+streamlit_option_menu
+pypdf
+rank_bm25
+faiss-cpu
+tiktoken
+scikit-learn
+sentence_transformers