changed similarity to sentence transformers

requirements.txt

@@ -20,4 +20,5 @@ spacy
 textstat
 plotly
 tqdm
-pymupdf
+pymupdf
+sentence-transformers

utils.py

@@ -9,10 +9,12 @@ from collections import Counter
 import numpy as np
 import asyncio
 import nltk
+from sentence_transformers import SentenceTransformer, util
 
 nltk.download('punkt')
 
 WORD = re.compile(r"\w+")
+model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
 
 
 # returns cosine similarity of two vectors
@@ -53,6 +55,13 @@ def cosineSim(text1, text2):
     cosine = get_cosine(vector1, vector2)
     return cosine
 
+def sentence_similarity(text1, text2):
+    embedding_1= model.encode(text1, convert_to_tensor=True)
+    embedding_2 = model.encode(text2, convert_to_tensor=True)
+
+    o = util.pytorch_cos_sim(embedding_1, embedding_2)
+    return round(o.item(), 2)
+
 def get_soup_requests(url):
     page = requests.get(url)
     if page.status_code == 200:
@@ -130,7 +139,7 @@ def googleSearch(
                     urlList.append(url)
                     scoreArray.append([0] * len(sentences))
                 urlCount[url] = urlCount[url] + 1 if url in urlCount else 1
-                scoreArray[urlList.index(url)][i] = cosineSim(
+                scoreArray[urlList.index(url)][i] = sentence_similarity(
                     sentence, snippet
                 )
         else: