from firebase_admin import credentials from firebase_admin import firestore import firebase_admin from firebase_admin import db cred = credentials.Certificate( "text-to-emotions-firebase-adminsdk-8isbn-dffbdf01e8.json") firebase_admin.initialize_app(cred) # Import the Firebase Admin SDK import threading import firebase_admin from datetime import datetime import time import Cleaning from collections import Counter import models from datetime import datetime date_format = '%d-%m-%Y' def current_milli_time(): return round(time.time() * 1000) import Classes as Classes db = firestore.client() # Firebase ininlaziton # for doc in docs: # print(f"{doc.id} => {doc.to_dict()}") all_reviews = db.collection("complaints") # Create a query against the collection today_date = current_milli_time() documents_to_summarize = all_reviews.get() feedbacks = db.collection("feedbacks").where("feedbacks","!=",'').get() documents=[] documnetskey={} #get all documents for today that have no summary # def get_all_document(): # for doc in documents_to_summarize: # document =Classes.Shakwa.from_dict(source=doc.to_dict()) # documents.append(document) # return documents def get_num_of_words(): for doc in documents_to_summarize: print(len(doc.complaintbody)) def shakwa_common_words(): allcomplaints = [] most_common_words = {} global documents for document in documents: document = Classes.Shakwa.from_dict(source=doc.to_dict()) allcomplaints.append(document.complaintbody) documents.clear() # print(allcomplaints) words_in_docs = " ".join(allcomplaints) words_in_docs = Cleaning.txt_preprocess(words_in_docs) words_in_docs = words_in_docs.split(" ") for word in words_in_docs: if word in most_common_words: most_common_words[word] += 1 else: most_common_words[word] = 1 most_common_words = sorted(most_common_words.items(), key=lambda x: x[1]) db.collection("per_day_common_words").add({str(datetime.today().date()):most_common_words}) def feedback_common_words(): allfeedbacks= [] most_common_words = {} for doc in feedbacks: document = Classes.Feedback.from_dict(source=doc.to_dict()) allcomplaints.append(document.feedback ) # print(allcomplaints) words_in_docs = " ".join(allfeedbacks) words_in_docs = Cleaning.txt_preprocess(words_in_docs) words_in_docs = words_in_docs.split(" ") for word in words_in_docs: if word in most_common_words: most_common_words[word] += 1 else: most_common_words[word] = 1 most_common_words = sorted(most_common_words.items(), key=lambda x: x[1]) return dict(most_common_words) def get_most_common_places(): dic_place_count = {} for doc in all_reviews.get(): document = Classes.Shakwa.from_dict(source=doc.to_dict()) if document.governorate not in dic_place_count.keys(): dic_place_count[document.governorate] = 1 else: dic_place_count[document.governorate] += 1 return dic_place_count queuedUnSummurizedShawkas = [] semphoreShakwas=threading.Semaphore(0) def summrizedshakwas(): global queuedUnSummurizedShawkas global semphoreShakwas global db while True: semphoreShakwas.acquire() shawka=queuedUnSummurizedShawkas.pop(0) tmpdict= models.modelsummary(shawka.complaintbody) shawka.summary=tmpdict['summary'] db.collection("complaints").document(shawka.id).update({"summary":shawka.summary}) thread = threading.Thread(target=summrizedshakwas) thread.start() #lithening to changes of documnts callback_done = threading.Event() def on_snapshot(doc_snapshot, changes, read_time): global queuedUnSummurizedShawkas global semphoreShakwas global documnetskey global documents for doc in doc_snapshot: # print(doc.to_dict()) shakw = Classes.Shakwa.from_dict(source=doc.to_dict()) datetime_obj = datetime.strptime(shakw.date, date_format) if datetime_obj.date() == datetime.today().date() and shakw.id not in documnetskey: documents.append(shakw) documnetskey.add(document.id) if shakw.summary==None: queuedUnSummurizedShawkas.append(shakw) semphoreShakwas.release() threadofcommonwords = threading.Thread(target=summrizedshakwas) threadofcommonwords.start() callback_done.set() docwatch= db.collection("complaints").on_snapshot(on_snapshot,)