Create new file

app.py

@@ -0,0 +1,252 @@
+import nltk
+nltk.download('stopwords')
+nltk.download('punkt')
+import pickle
+from keybert import KeyBERT
+from nltk.util import everygrams
+from nltk.corpus import stopwords
+from nltk.tokenize import sent_tokenize
+from fincat_utils import extract_context_words
+from fincat_utils import bert_embedding_extract
+from sentence_transformers import SentenceTransformer, util
+import torch
+from transformers import BertTokenizer, BertForSequenceClassification, pipeline, AutoTokenizer, AutoModelForSequenceClassification, TextClassificationPipeline, AutoModelForSeq2SeqLM, AutoModel, RobertaModel, RobertaTokenizer
+import gradio as gr
+import pandas as pd
+from fin_readability_sustainability import BERTClass, do_predict
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+kw_model = KeyBERT(model='all-mpnet-base-v2')
+
+#ESG
+finbert_esg = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-esg',num_labels=4)
+tokenizer_esg = BertTokenizer.from_pretrained('yiyanghkust/finbert-esg')
+nlp_esg = pipeline("text-classification", model=finbert_esg, tokenizer=tokenizer_esg)
+
+#FLS
+finbert_fls = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-fls',num_labels=3)
+tokenizer_fls = BertTokenizer.from_pretrained('yiyanghkust/finbert-fls')
+nlp_fls = pipeline("text-classification", model=finbert_fls, tokenizer=tokenizer_fls)
+
+#FinCAT - Claim Detection
+lr_clf_claim = pickle.load(open("lr_clf_FiNCAT.pickle",'rb'))
+
+#Sustainability
+tokenizer_sus = RobertaTokenizer.from_pretrained('roberta-base')
+model_sustain = BERTClass(2, "sustanability")
+model_sustain.to(device)
+model_sustain.load_state_dict(torch.load('sustainability_model.bin', map_location=device)['model_state_dict'])
+
+#Readability
+tokenizer_read = BertTokenizer.from_pretrained('ProsusAI/finbert')
+model_read = BERTClass(2, "readability")
+model_read.to(device)
+model_read.load_state_dict(torch.load('readability_model.bin', map_location=device)['model_state_dict'])
+
+#Sentiment
+model_senti1 = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-tone',num_labels=3)
+tokenizer_senti1 = BertTokenizer.from_pretrained('yiyanghkust/finbert-tone')
+senti1 = pipeline("sentiment-analysis", model=model_senti1, tokenizer=tokenizer_senti1)
+model_senti2 = AutoModelForSequenceClassification.from_pretrained("mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis")
+tokenizer_senti2 = AutoTokenizer.from_pretrained("mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis")
+senti2 = TextClassificationPipeline(model=model_senti2, tokenizer=tokenizer_senti2)
+
+#Summarization
+model_finsum = AutoModelForSeq2SeqLM.from_pretrained("human-centered-summarization/financial-summarization-pegasus")
+tokenizer_finsum = AutoTokenizer.from_pretrained("human-centered-summarization/financial-summarization-pegasus")
+
+
+#Hypernym Detection
+model_finlipi = SentenceTransformer('sohomghosh/LIPI_FinSim3_Hypernym')
+hypernyms = ['Bonds','Forward','Funds', 'Future', 'MMIs','Option', 'Stocks', 'Swap', 'Equity Index', 'Credit Index', 'Securities restrictions', 'Parametric schedules', 'Debt pricing and yields', 'Credit Events','Stock Corporation', 'Central Securities Depository', 'Regulatory Agency']
+hyp_di = { v:k for v, k in enumerate(hypernyms)}
+hypernyms_embeddings = model_finlipi.encode(hypernyms)
+
+
+#ESG
+def esg(text):
+  sents = sent_tokenize(text)
+  results = nlp_esg(sents)
+  highlight = [(text,i['label']) for text,i in zip(sents,results)]
+  return highlight
+
+#FLS
+def fls(text):
+  sents = sent_tokenize(text)
+  results = nlp_fls(sents)
+  highlight = [(text,i['label']) for text,i in zip(sents,results)]
+  return highlight
+
+#Sentiment
+def getfinsenti(text):
+  highlight = []
+  for text in sent_tokenize(text):
+    senti1_output = senti1(text)[0]
+    senti2_output = senti2(text)[0]
+    if senti1_output['score'] >= senti2_output['score']:
+      label = senti1_output['label']
+      score = round(senti1_output['score'], 4)
+    else:
+      label = senti2_output['label']
+      score = round(senti2_output['score'], 4)
+    highlight.append((text, label.strip().lower()))
+  return highlight
+
+#Summarization
+def summarize_pega(text):
+  input_ids = tokenizer_finsum(text, return_tensors="pt").input_ids
+  output = model_finsum.generate(
+      input_ids, 
+      max_length=32, 
+      num_beams=5, 
+      early_stopping=True
+  )
+  summary =  str(tokenizer_finsum.decode(output[0], skip_special_tokens=True))
+  return summary
+
+#Hypernym Detection
+def get_hyp(words, th=0.85):
+  queries = [wd.strip() for wd in words.split(",")]
+  highlight = []
+  if len(queries)>0:
+    query_embeddings = model_finlipi.encode(queries)
+    cos_scores = util.pytorch_cos_sim(query_embeddings, hypernyms_embeddings)
+    ans = torch.max(cos_scores, dim=1)
+    
+    for sim,ind,query in zip(ans.values, ans.indices, queries):
+      if query.strip()!="":
+        if sim.item()>th:
+          highlight.append((query, hyp_di[ind.item()]))
+        else:
+          highlight.append((query, 'no hypernym found'))
+  
+  return highlight
+
+#FinCAT - Claim Detection
+def score_fincat(txt):
+  '''
+  Extracts numerals from financial texts and checks if they are in-claim or out-of claim
+
+    Parameters:
+      txt (str): Financial Text. This is to be given as input. Numerals present in this text will be evaluated.
+
+    Returns:
+      highlight (list): A list each element of which is a tuple. Each tuple has two elements i) word ii) whether the word is in-claim or out-of-claim.
+  '''
+  #li = []
+  highlight = []
+  txt = " " + txt + " "
+  k = ''
+  for word in txt.split():
+    if any(char.isdigit() for char in word):
+      if word[-1] in ['.', ',', ';', ":", "-", "!", "?", ")", '"', "'"]:
+        k = word[-1]
+        word = word[:-1]
+      st = txt.index(" " + word + k + " ")+1
+      k = ''
+      ed = st + len(word)
+      x = {'paragraph' : txt, 'offset_start':st, 'offset_end':ed}
+      context_text = extract_context_words(x)
+      features = bert_embedding_extract(context_text, word)
+      prediction = lr_clf_claim.predict(features.reshape(1, 768))
+      highlight.append((word, 'In-claim' if prediction==1 else 'Out-of-Claim'))
+    else:
+      highlight.append((word, ''))
+  headers = ['numeral', 'prediction', 'probability']
+  return highlight
+
+#Readability
+def get_readability(text):
+  df = pd.DataFrame({'sentence':sent_tokenize(text)})
+  actual_predictions_read = do_predict(model_read, tokenizer_read, df)
+  highlight = [(sent, 'readable') if i==1 else (sent, 'non-readable') for sent,i in zip(df['sentence'].values, actual_predictions_read[0])]
+  return highlight
+
+
+#Sustainability
+def get_sustainability(text):
+  df = pd.DataFrame({'sentence':sent_tokenize(text)})
+  actual_predictions_sustainability = do_predict(model_sustain, tokenizer_sus, df)
+  highlight = []
+  for sent, prob in zip(df['sentence'].values, actual_predictions_sustainability[1]):
+    if prob>=2.8:#4.384316:
+      highlight.append((sent, 'non-sustainable'))
+    elif prob<=1.423736:
+      highlight.append((sent, 'sustainable'))
+    else:
+      highlight.append((sent, '-'))
+  return highlight
+
+#keywords
+def get_keywords(text):
+  keywords = kw_model.extract_keywords(text, keyphrase_ngram_range=(1, 3), stop_words='english', highlight=False, top_n=3)
+  keywords_list= list(dict(keywords).keys())
+  return ",".join([i.strip() for i in keywords_list])
+
+#examples
+def set_example_text(example_text):
+    return gr.Textbox.update(value=example_text[0])
+
+
+demo = gr.Blocks()
+
+with demo:
+    gr.Markdown("# **Financial Language Understandability Enhancement Toolkit (FLUEnT)**")
+    with gr.Row():
+      with gr.Column():
+        text = gr.inputs.Textbox(label="Enter financial text here", lines=6, placeholder="Enter Financial Text here...")
+        
+        b_hyp_th = gr.inputs.Slider(minimum=0, maximum=1, step=0.01, label="Detect hypernyms with confidence of")
+        with gr.Row():
+          b1 = gr.Button("Get Keywords For Hypernym Detection")
+
+        with gr.Row():
+          jargons = gr.Textbox(label="Enter words for Hypernyms Detection separated by comma")
+          b1.click(get_keywords, inputs = text, outputs=jargons)
+
+        example_text = gr.Dataset(components=[text], samples=[["Markets are falling."], ["Exchanges the coupon on a bond for LIBOR plus a spread."], ["We follow a carbon neutrality strategy, seek to use resources efficiently and work to deliver sustainable value for society"], ["NGOs have been instrumental in shaping the economy"], ["We expect to boost our sales by 80% this year by using eco-friendly products."], ["We will continue to evaluate the need for an employee allowance as it hinders growth."],["As an example, in the calculation as of the end of 2020, carbon emissions of an issuer relate to 2019, whereas market capitalization is shown as of the end of 2020."], ["In addition to the impacts from the merger, insurance income increased $121 million due to strong production and acquisitions."],["In the year 2021, the markets were bullish. We expect to boost our sales by 80% this year by using eco-friendly products."], ["Noninterest income increased $1.7 billion due primarily to the Merger and higher residential mortgage income as a result of the lower rate environment driving mortgage production through refinance activity, partially offset by lower residential mortgage servicing income driven by higher prepayment and an MSR fair value adjustment in 2020. This year it will increase to $3M."]])
+        example_text.click(fn=set_example_text,
+                             inputs=example_text,
+                             outputs=example_text.components)  
+
+      with gr.Column():
+        with gr.Tabs():
+          with gr.TabItem("Hypernyms & Claims"):
+            with gr.Row():
+              b_hyp = gr.Button("Get Hypernyms")
+              b_hyp.click(get_hyp, inputs = [jargons, b_hyp_th], outputs = gr.HighlightedText())
+            with gr.Row():
+              b3 = gr.Button("Get Claims")
+              b3.click(score_fincat, inputs = text, outputs = gr.HighlightedText().style(color_map={"In-claim": "red", "Out-of-Claim": "green"}))
+          
+          with gr.TabItem("Summary & Sentiment"):                
+            with gr.Row():
+              b2 = gr.Button("Get Summary")
+              b2.click(summarize_pega, inputs = text, outputs = gr.Textbox(label="Summary"))
+
+            with gr.Row():
+              b4 = gr.Button("Get Sentiment")
+              b4.click(getfinsenti, inputs = text, outputs = gr.HighlightedText().style(color_map={"negative": "red", "neutral":"blue", "positive": "green"}))
+
+          with gr.TabItem("Readability & Sustainability"):
+            
+            with gr.Row():
+              b5 = gr.Button("Get Readability")
+              b5.click(get_readability, inputs = text, outputs = gr.HighlightedText().style(color_map={"non-readable": "red", "readable": "green"}))
+
+            with gr.Row():
+              b6 = gr.Button("Get Sustainability")
+              b6.click(get_sustainability, inputs = text, outputs = gr.HighlightedText().style(color_map={"non-sustainable": "red", "-":"blue", "sustainable": "green"}))
+
+          with gr.TabItem("ESG & FLS"):
+
+            with gr.Row():
+              b6 = gr.Button("Get Environmental, Social & Gov.(ESG)")
+              b6.click(esg, inputs = text, outputs = gr.HighlightedText().style(color_map={"Governance": "red", "Social":"blue", "Environmental": "green", "None":"yellow"}))
+            
+            with gr.Row():
+              b6 = gr.Button("Get Forward Looking Statements(FLS)")
+              b6.click(fls, inputs = text, outputs = gr.HighlightedText().style(color_map={"Non-specific FLS": "red", "Not-FLS":"blue", "Specific-FLS": "green"}))
+    gr.Markdown("How to use? [link](https://youtu.be/Sk3PiQdr9Og), Warning: User discretion is advised.")
+demo.launch()