Added dup_ques without word embeddings

.gitignore

@@ -6,4 +6,5 @@
 /src/face_analytics/__pycache__
 /src/movie_rec/__pycache__
 /src/movie_2022_rec/__pycache__
-/src/movie_reviews/__pycache__
+/src/movie_reviews/__pycache__
+/src/dup_ques/__pycache__

main.py

@@ -1,9 +1,10 @@
 from fastapi import FastAPI
 
 # Importing Models and Schemas
+from src.dup_ques.main import dup_ques, Schema as DupQuesSchema
 from src.movie_reviews.main import movie_reviews, Schema as MovieReviewsSchema
 from src.cat_and_dog.main import cat_and_dog, Schema as CatAndDogSchema
-from src.face_analytics.main import face_analytics, Schema as FaceAnalytics
+from src.face_analytics.main import face_analytics, Schema as FaceAnalyticsSchema
 from src.book_rec.main import book_rec, Schema as BookRecSchema
 from src.movie_rec.main import movie_rec, Schema as MovieRecSchema
 from src.movie_2022_rec.main import movie_2022_rec, Schema as Movie2022RecSchema
@@ -35,6 +36,10 @@ print(" ........... App Started ........... ")
 def index():
 	return "Welcome to the API of PyModelsAI"
 
+@app.post("/dup_ques")
+def endpoint_movie_reviews(req: DupQuesSchema):
+    return dup_ques(req)
+
 @app.post("/movie_reviews")
 def endpoint_movie_reviews(req: MovieReviewsSchema):
     return movie_reviews(req)
@@ -44,7 +49,7 @@ def endpoint_cat_and_dog(req: CatAndDogSchema):
     return cat_and_dog(req)
 
 @app.post("/face_analytics")
-def endpoint_face_analytics(req: CatAndDogSchema):
+def endpoint_face_analytics(req: FaceAnalyticsSchema):
     return face_analytics(req)
 
 @app.post("/book_rec")

requirements.txt

@@ -5,4 +5,6 @@ scikit-learn
 numpy
 tensorflow-cpu
 keras
-Pillow
+Pillow
+distance
+fuzzywuzzy

src/dup_ques/main.py

@@ -0,0 +1,24 @@
+import joblib
+from pydantic import BaseModel
+from src.dup_ques.preprocess import get_x
+
+# SCHEMA
+class Schema(BaseModel):
+	ques1: str
+	ques2: str
+
+# Request Handler
+def dup_ques(req):
+	ques1 = req.ques1
+	ques2 = req.ques2
+
+	X = get_x(ques1, ques2)
+	y = predict(X)
+
+	return y
+
+# PIPELINE
+pipeline = joblib.load("./src/dup_ques/pipeline.pkl")
+
+def predict(X):
+	return pipeline.predict_proba(X).round(3).tolist()

src/dup_ques/pipeline.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fc8c1363c54b2332c840db78a82bed513bc78a4dcc78b7483f892185abb64a7
+size 4090964

src/dup_ques/preprocess.py

@@ -0,0 +1,315 @@
+import json
+
+WORD_EMBEDDINGS_PATH = "./src/dup_ques/word_embeddings.json"
+with open(WORD_EMBEDDINGS_PATH, 'rb') as f:
+    WORD_EMBEDDINGS = json.load(f)
+
+import nltk
+
+nltk.download('wordnet')
+nltk.download('omw-1.4')
+nltk.download('stopwords')
+
+abbv = {
+    "AFAIK":"as far as I know", "IMO":	"in my opinion", "IMHO":	"in my humble opinion", "LGTM":	"look good to me", "AKA":	"also know as", "ASAP":	"as sone as possible", "BTW":	"by the way", "FAQ":	"frequently asked questions", "DIY":	"do it yourself", "DM":	"direct message", "FYI":	"for your information", "IC":	"i see", "IOW":	"in other words", "IIRC":	"If I Remember Correctly", "icymi":"In case you missed it", "CUZ":	"because", "COS":	"because", "nv":	"nevermind", "PLZ":	"please",
+}
+
+# https://en.wikipedia.org/wiki/Wikipedia%3aList_of_English_contractions
+# https://stackoverflow.com/a/19794953
+contractions = { 
+    "ain't": "am not", "aren't": "are not", "can't": "can not", "can't've": "can not have", "'cause": "because", "could've": "could have", "couldn't": "could not", "couldn't've": "could not have", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hadn't've": "had not have", "hasn't": "has not", "haven't": "have not", "he'd": "he would", "he'd've": "he would have", "he'll": "he will", "he'll've": "he will have", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have", "i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have", "it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have", "mightn't": "might not", "mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have", "o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have", "so's": "so as", "that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would", "y'all'd've": "you all would have", "y'all're": "you all are", "y'all've": "you all have", "you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", "'ve": " have", "n't": " not", "'re": " are", "'ll": " will",
+}
+
+import re
+html_pattern = re.compile('<.*?>')
+urls_pattern = re.compile(r'https?://\S+|www\.\S+')
+emoji_pattern = re.compile("["
+	u"\U0001F600-\U0001F64F"  # emoticons
+	u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+	u"\U0001F680-\U0001F6FF"  # transport & map symbols
+	u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
+"]+", flags=re.UNICODE)
+
+from nltk.stem.porter import PorterStemmer
+ps = PorterStemmer()
+
+from nltk.stem import WordNetLemmatizer
+lemmatizer = WordNetLemmatizer()
+
+import string
+punc = string.punctuation
+
+from nltk.corpus import stopwords
+stopwords = stopwords.words('english')
+
+def text_preprocess(q, allow_stopwords=True):
+    
+    q = str(q).lower().strip()
+
+    # HTML Tags
+    q = html_pattern.sub(r'', q)
+
+    # urls
+    q = urls_pattern.sub(r'', q)
+
+    # punctuations
+    q = q.translate(str.maketrans("", "", punc))
+
+    # Emojis
+    q = emoji_pattern.sub(r'', q)
+    
+    # Replace certain special characters with their string equivalents
+    q = q.replace('%', ' percent')
+    q = q.replace('$', ' dollar ')
+    q = q.replace('₹', ' rupee ')
+    q = q.replace('€', ' euro ')
+    q = q.replace('@', ' at ')
+    
+    # The pattern '[math]' appears around 900 times in the whole dataset.
+    q = q.replace('[math]', '')
+    
+    # Replacing some numbers with string equivalents (not perfect, can be done better to account for more cases)
+    q = q.replace(',000,000,000 ', 'b ')
+    q = q.replace(',000,000 ', 'm ')
+    q = q.replace(',000 ', 'k ')
+    q = re.sub(r'([0-9]+)000000000', r'\1b', q)
+    q = re.sub(r'([0-9]+)000000', r'\1m', q)
+    q = re.sub(r'([0-9]+)000', r'\1k', q)
+    
+    # Decontracting words
+    new_text = []
+
+    for word in q.split():
+
+        # Contractions
+        word = contractions.get(word.upper(), word)
+
+        # abbreviations
+        word = abbv.get(word.upper(), word)
+            
+        # Stemming
+        # word = ps.stem(word)
+
+        # Lemmatizing
+        word = lemmatizer.lemmatize(word)
+
+        if word not in stopwords or allow_stopwords:
+            new_text.append(word)
+
+    q = ' '.join(new_text)
+
+    return q
+
+import distance
+from fuzzywuzzy import fuzz
+import numpy as np
+from numpy.linalg import norm
+SAFE_DIV = 0.0001
+
+def cos_sim(q1, q2, allow_stopwords=True):
+    q1 = [i for i in q1.split() if i not in stopwords or allow_stopwords]
+    q2 = [i for i in q2.split() if i not in stopwords or allow_stopwords]
+
+    vocab = set(q1 + q2)
+
+    vocab1 = dict(zip(vocab, [0]*len(vocab)))
+    vocab2 = dict(zip(vocab, [0]*len(vocab)))
+
+    for w in q1:
+        vocab1[w] += 1
+    for w in q2:
+        vocab2[w] += 1
+
+    v1 = list(vocab1.values())
+    v2 = list(vocab2.values())
+
+    return (np.dot(v1,v2) + SAFE_DIV)/(norm(v1)*norm(v2) + SAFE_DIV)
+
+def cos_sim_vec(v1, v2):
+    return (np.dot(v1,v2) + SAFE_DIV)/(norm(v1)*norm(v2) + SAFE_DIV)
+
+def euler_distance(v1, v2):
+    return sum((v1 - v2)**2)
+
+def sentence_emb(sent):
+    embs = np.zeros(100)
+    counter = 0
+    for word in sent.split():
+        emb = WORD_EMBEDDINGS.get(word)
+        if emb != None:
+            embs += emb
+            counter += 1
+    if counter == 0:
+        counter = 1
+    return embs / counter
+
+def test_common_words(q1,q2):
+    w1 = set(map(lambda word: word.lower().strip(), q1.split(" ")))
+    w2 = set(map(lambda word: word.lower().strip(), q2.split(" ")))
+    return len(w1 & w2)
+
+def test_total_words(q1,q2):
+    w1 = set(map(lambda word: word.lower().strip(), q1.split(" ")))
+    w2 = set(map(lambda word: word.lower().strip(), q2.split(" ")))
+    return (len(w1) + len(w2))
+
+
+def test_fetch_token_features(q1, q2):
+    SAFE_DIV = 0.0001
+
+    # STOP_WORDS = pickle.load(open('stopwords.pkl','rb'))
+    STOP_WORDS = stopwords
+
+    token_features = [0.0] * 8
+
+    # Converting the Sentence into Tokens:
+    q1_tokens = q1.split()
+    q2_tokens = q2.split()
+
+    if len(q1_tokens) == 0 or len(q2_tokens) == 0:
+        return token_features
+
+    # Get the non-stopwords in Questions
+    q1_words = set([word for word in q1_tokens if word not in STOP_WORDS])
+    q2_words = set([word for word in q2_tokens if word not in STOP_WORDS])
+
+    # Get the stopwords in Questions
+    q1_stops = set([word for word in q1_tokens if word in STOP_WORDS])
+    q2_stops = set([word for word in q2_tokens if word in STOP_WORDS])
+
+    # Get the common non-stopwords from Question pair
+    common_word_count = len(q1_words.intersection(q2_words))
+
+    # Get the common stopwords from Question pair
+    common_stop_count = len(q1_stops.intersection(q2_stops))
+
+    # Get the common Tokens from Question pair
+    common_token_count = len(set(q1_tokens).intersection(set(q2_tokens)))
+
+    token_features[0] = common_word_count / (min(len(q1_words), len(q2_words)) + SAFE_DIV)
+    token_features[1] = common_word_count / (max(len(q1_words), len(q2_words)) + SAFE_DIV)
+    token_features[2] = common_stop_count / (min(len(q1_stops), len(q2_stops)) + SAFE_DIV)
+    token_features[3] = common_stop_count / (max(len(q1_stops), len(q2_stops)) + SAFE_DIV)
+    token_features[4] = common_token_count / (min(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)
+    token_features[5] = common_token_count / (max(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)
+
+    # Last word of both question is same or not
+    token_features[6] = int(q1_tokens[-1] == q2_tokens[-1])
+
+    # First word of both question is same or not
+    token_features[7] = int(q1_tokens[0] == q2_tokens[0])
+
+    return token_features
+
+
+def test_fetch_length_features(q1, q2):
+    length_features = [0.0] * 3
+
+    # Converting the Sentence into Tokens:
+    q1_tokens = q1.split()
+    q2_tokens = q2.split()
+
+    if len(q1_tokens) == 0 or len(q2_tokens) == 0:
+        return length_features
+
+    # Absolute length features
+    length_features[0] = abs(len(q1_tokens) - len(q2_tokens))
+
+    # Average Token Length of both Questions
+    length_features[1] = (len(q1_tokens) + len(q2_tokens)) / 2
+
+    strs = list(distance.lcsubstrings(q1, q2))
+    if len(strs) > 0:
+        length_features[2] = len(strs[0]) / (min(len(q1), len(q2)) + 1)
+
+    return length_features
+
+
+def test_fetch_fuzzy_features(q1, q2):
+    fuzzy_features = [0.0] * 4
+
+    # fuzz_ratio
+    fuzzy_features[0] = fuzz.QRatio(q1, q2)
+
+    # fuzz_partial_ratio
+    fuzzy_features[1] = fuzz.partial_ratio(q1, q2)
+
+    # token_sort_ratio
+    fuzzy_features[2] = fuzz.token_sort_ratio(q1, q2)
+
+    # token_set_ratio
+    fuzzy_features[3] = fuzz.token_set_ratio(q1, q2)
+
+    return fuzzy_features
+
+
+def query_point_creator(q1, q2, allow_stopwords):
+    input_query = []
+
+    # preprocess
+    q1 = text_preprocess(q1, allow_stopwords)
+    q2 = text_preprocess(q2, allow_stopwords)
+
+    # cosine similarity
+    input_query.append(cos_sim(q1, q2))
+
+    # fetch basic features
+    input_query.append(len(q1))
+    input_query.append(len(q2))
+
+    input_query.append(len(q1.split(" ")))
+    input_query.append(len(q2.split(" ")))
+
+    input_query.append(test_common_words(q1, q2))
+    input_query.append(test_total_words(q1, q2))
+    input_query.append(round(test_common_words(q1, q2) / test_total_words(q1, q2), 2))
+
+    # fetch token features
+    token_features = test_fetch_token_features(q1, q2)
+    input_query.extend(token_features)
+
+    # fetch length based features
+    length_features = test_fetch_length_features(q1, q2)
+    input_query.extend(length_features)
+
+    # fetch fuzzy features
+    fuzzy_features = test_fetch_fuzzy_features(q1, q2)
+    input_query.extend(fuzzy_features)
+
+    return input_query
+
+def sentence_emb(sent):
+    embs = np.zeros(100)
+    counter = 0
+    for word in sent.split():
+        emb = WORD_EMBEDDINGS.get(word)
+        if emb != None:
+            embs += emb
+            counter += 1
+    if counter == 0:
+        counter = 1
+    return embs / counter
+
+def get_x(q1, q2):
+    x = []
+        
+    x.extend(
+        query_point_creator(q1, q2, False)
+    )
+    x.extend(
+        query_point_creator(q1, q2, True)
+    )
+
+    q1 = text_preprocess(q1, allow_stopwords=True)
+    q2 = text_preprocess(q2, allow_stopwords=True)
+
+    emb1 = sentence_emb(q1)
+    emb2 = sentence_emb(q2)
+    
+    x.append(cos_sim_vec(emb1, emb2))
+    x.append(euler_distance(emb1, emb2))
+
+    x.extend(emb1)
+    x.extend(emb2)
+
+    return np.expand_dims(x, axis=0)

src/movie_reviews/main.py

@@ -50,11 +50,7 @@ pipeline = joblib.load("./src/movie_reviews/pipeline.pkl")
 
 def predict(text):
 	cleaned = preprocess(text)
-	pred = pipeline.predict([cleaned])[0]
-	output = [0, 0]
-	output[pred] = 0.8
-	output[1-pred] = 0.2
-	return [output]
+	return pipeline.predict_proba([cleaned]).round(3).tolist()
 
 def preprocess(text):
     text = text.lower()	# Lowercase