# set path import glob, os, sys sys.path.append('../utils') #import needed libraries import seaborn as sns import matplotlib.pyplot as plt import numpy as np import pandas as pd import streamlit as st from utils.adapmit_classifier import load_adapmitClassifier,adapmit_classification # from utils.keyword_extraction import textrank import logging logger = logging.getLogger(__name__) from utils.config import get_classifier_params from utils.preprocessing import paraLengthCheck from io import BytesIO import xlsxwriter import plotly.express as px # Declare all the necessary variables classifier_identifier = 'adapmit' params = get_classifier_params(classifier_identifier) @st.cache_data def to_excel(df): len_df = len(df) output = BytesIO() writer = pd.ExcelWriter(output, engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') workbook = writer.book worksheet = writer.sheets['Sheet1'] worksheet.data_validation('E2:E{}'.format(len_df), {'validate': 'list', 'source': ['No', 'Yes', 'Discard']}) worksheet.data_validation('F2:F{}'.format(len_df), {'validate': 'list', 'source': ['No', 'Yes', 'Discard']}) worksheet.data_validation('G2:G{}'.format(len_df), {'validate': 'list', 'source': ['No', 'Yes', 'Discard']}) writer.save() processed_data = output.getvalue() return processed_data def app(): ### Main app code ### with st.container(): if 'key1' in st.session_state: df = st.session_state.key1 classifier = load_adapmitClassifier(classifier_name=params['model_name']) st.session_state['{}_classifier'.format(classifier_identifier)] = classifier if sum(df['Target Label'] == 'TARGET') > 100: warning_msg = ": This might take sometime, please sit back and relax." else: warning_msg = "" df = adapmit_classification(haystack_doc=df, threshold= params['threshold']) st.session_state.key1 = df # threshold= params['threshold'] # truth_df = df.drop(['text'],axis=1) # truth_df = truth_df.astype(float) >= threshold # truth_df = truth_df.astype(str) # categories = list(truth_df.columns) # placeholder = {} # for val in categories: # placeholder[val] = dict(truth_df[val].value_counts()) # count_df = pd.DataFrame.from_dict(placeholder) # count_df = count_df.T # count_df = count_df.reset_index() # # st.write(count_df) # placeholder = [] # for i in range(len(count_df)): # placeholder.append([count_df.iloc[i]['index'],count_df['True'][i],'Yes']) # placeholder.append([count_df.iloc[i]['index'],count_df['False'][i],'No']) # count_df = pd.DataFrame(placeholder, columns = ['category','count','truth_value']) # # st.write("Total Paragraphs: {}".format(len(df))) # fig = px.bar(count_df, y='category', x='count', # color='truth_value',orientation='h', height =200) # c1, c2 = st.columns([1,1]) # with c1: # st.plotly_chart(fig,use_container_width= True) # truth_df['labels'] = truth_df.apply(lambda x: {i if x[i]=='True' else None for i in categories}, axis=1) # truth_df['labels'] = truth_df.apply(lambda x: list(x['labels'] -{None}),axis=1) # # st.write(truth_df) # df = pd.concat([df,truth_df['labels']],axis=1) # st.markdown("###### Top few 'Mitigation' related paragraph/text ######") # df = df.sort_values(by = ['Mitigation'], ascending=False) # for i in range(3): # if df.iloc[i]['Mitigation'] >= 0.50: # st.write('**Result {}** (Relevancy Score: {:.2f})'.format(i+1,df.iloc[i]['Mitigation'])) # st.write("\t Text: \t{}".format(df.iloc[i]['text'].replace("\n", " "))) # st.markdown("###### Top few 'Adaptation' related paragraph/text ######") # df = df.sort_values(by = ['Adaptation'], ascending=False) # for i in range(3): # if df.iloc[i]['Adaptation'] > 0.5: # st.write('**Result {}** (Relevancy Score: {:.2f})'.format(i+1,df.iloc[i]['Adaptation'])) # st.write("\t Text: \t{}".format(df.iloc[i]['text'].replace("\n", " "))) # # st.write(df[['text','labels']]) # df['Validation'] = 'No' # df['Val-Mitigation'] = 'No' # df['Val-Adaptation'] = 'No' # df_xlsx = to_excel(df) # st.download_button(label='📥 Download Current Result', # data=df_xlsx , # file_name= 'file_adaptation-mitigation.xlsx') # # st.session_state.key4 = # # category =set(df.columns) # # removecols = {'Validation','Val-Adaptation','Val-Mitigation','text'} # # category = list(category - removecols) # else: # st.info("🤔 No document found, please try to upload it at the sidebar!") # logging.warning("Terminated as no document provided") # # Creating truth value dataframe # if 'key4' in st.session_state: # if st.session_state.key4 is not None: # df = st.session_state.key4 # st.markdown("###### Select the threshold for classifier ######") # c4, c5 = st.columns([1,1]) # with c4: # threshold = st.slider("Threshold", min_value=0.00, max_value=1.0, # step=0.01, value=0.5, # help = "Keep High Value if want refined result, low if dont want to miss anything" ) # category =set(df.columns) # removecols = {'Validation','Val-Adaptation','Val-Mitigation','text'} # category = list(category - removecols) # placeholder = {} # for val in category: # temp = df[val].astype(float) > threshold # temp = temp.astype(str) # placeholder[val] = dict(temp.value_counts()) # count_df = pd.DataFrame.from_dict(placeholder) # count_df = count_df.T # count_df = count_df.reset_index() # placeholder = [] # for i in range(len(count_df)): # placeholder.append([count_df.iloc[i]['index'],count_df['False'][i],'False']) # placeholder.append([count_df.iloc[i]['index'],count_df['True'][i],'True']) # count_df = pd.DataFrame(placeholder, columns = ['category','count','truth_value']) # fig = px.bar(count_df, x='category', y='count', # color='truth_value', # height=400) # st.write("") # st.plotly_chart(fig) # df['Validation'] = 'No' # df['Val-Mitigation'] = 'No' # df['Val-Adaptation'] = 'No' # df_xlsx = to_excel(df) # st.download_button(label='📥 Download Current Result', # data=df_xlsx , # file_name= 'file_adaptation-mitigation.xlsx')