ver02

app.py

@@ -3,6 +3,8 @@ import appStore.netzero as netzero
 import appStore.sector as sector
 import appStore.adapmit as adapmit
 import appStore.ghg as ghg
+import appStore.policyaction as policyaction
+import appStore.indicator as indicator
 import appStore.doc_processing as processing
 from utils.uploadAndExample import add_upload
 import streamlit as st
@@ -52,7 +54,7 @@ with st.expander("ℹ️ - About this app", expanded=False):
         """)
     st.write("")
 apps = [processing.app, target_extraction.app, netzero.app, ghg.app,
-        sector.app, adapmit.app]
+        sector.app, policyaction.app, indicator.app, adapmit.app]
 multiplier_val =1/len(apps)
 if st.button("Analyze Document"):
     prg = st.progress(0.0)
@@ -60,6 +62,17 @@ if st.button("Analyze Document"):
         func()
         prg.progress((i+1)*multiplier_val)
 
-    if 'key1' in st.session_state:
+    
+if 'key1' in st.session_state:
+    with st.sidebar:
+        topic = st.radio(
+                        "Which category you want to explore?",
+                        ('Target', 'Action', 'Policies/Plans'))
+    
+    if topic == 'Target':
         target_extraction.target_display()
-        # st.write(st.session_state.key1)
+    elif topic == 'Action':
+        policyaction.action_display()
+    else: 
+        policyaction.policy_display()
+    # st.write(st.session_state.key1)

appStore/indicator.py

@@ -0,0 +1,166 @@
+# set path
+import glob, os, sys; 
+sys.path.append('../utils')
+
+#import needed libraries
+import seaborn as sns
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import streamlit as st
+from utils.indicator_classifier import load_indicatorClassifier, indicator_classification
+import logging
+logger = logging.getLogger(__name__)
+from utils.config import get_classifier_params
+from utils.preprocessing import paraLengthCheck
+from io import BytesIO
+import xlsxwriter
+import plotly.express as px
+
+
+# Declare all the necessary variables
+classifier_identifier = 'indicator'
+params  = get_classifier_params(classifier_identifier)
+
+@st.cache_data
+def to_excel(df,sectorlist):
+    len_df = len(df)
+    output = BytesIO()
+    writer = pd.ExcelWriter(output, engine='xlsxwriter')
+    df.to_excel(writer, index=False, sheet_name='Sheet1')
+    workbook = writer.book
+    worksheet = writer.sheets['Sheet1']
+    worksheet.data_validation('S2:S{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': ['No', 'Yes', 'Discard']})
+    worksheet.data_validation('X2:X{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': sectorlist + ['Blank']})
+    worksheet.data_validation('T2:T{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': sectorlist + ['Blank']})
+    worksheet.data_validation('U2:U{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': sectorlist + ['Blank']})                               
+    worksheet.data_validation('V2:V{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': sectorlist + ['Blank']})
+    worksheet.data_validation('W2:U{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': sectorlist + ['Blank']})                            
+    writer.save()
+    processed_data = output.getvalue()
+    return processed_data
+
+def app():
+
+    ### Main app code ###
+    with st.container():
+                   
+            if 'key1' in st.session_state:
+                df = st.session_state.key1
+                classifier = load_indicatorClassifier(classifier_name=params['model_name'])
+                st.session_state['{}_classifier'.format(classifier_identifier)] = classifier
+
+                if sum(df['Target Label'] == 'TARGET') > 100:
+                    warning_msg = ": This might take sometime, please sit back and relax."
+                else:
+                    warning_msg = ""
+                    
+                df = indicator_classification(haystack_doc=df,
+                                            threshold= params['threshold'])
+
+                st.session_state.key1 = df
+
+
+                # # st.write(df)
+                # threshold= params['threshold']
+                # truth_df = df.drop(['text'],axis=1)
+                # truth_df = truth_df.astype(float) >= threshold
+                # truth_df = truth_df.astype(str)
+                # categories = list(truth_df.columns)
+
+                # placeholder = {}
+                # for val in categories:
+                #     placeholder[val] = dict(truth_df[val].value_counts())
+                # count_df = pd.DataFrame.from_dict(placeholder)
+                # count_df = count_df.T
+                # count_df = count_df.reset_index()
+                # # st.write(count_df)
+                # placeholder  = []
+                # for i in range(len(count_df)):
+                #     placeholder.append([count_df.iloc[i]['index'],count_df['True'][i],'Yes'])
+                #     placeholder.append([count_df.iloc[i]['index'],count_df['False'][i],'No'])
+                # count_df = pd.DataFrame(placeholder, columns = ['category','count','truth_value'])
+                # # st.write("Total Paragraphs: {}".format(len(df)))
+                # fig = px.bar(count_df, x='category', y='count',
+                #             color='truth_value')
+                # # c1, c2 = st.columns([1,1])
+                # # with c1:
+                # st.plotly_chart(fig,use_container_width= True)
+
+                # truth_df['labels'] = truth_df.apply(lambda x: {i if x[i]=='True' else None for i in categories}, axis=1)
+                # truth_df['labels'] = truth_df.apply(lambda x: list(x['labels'] -{None}),axis=1)
+                # # st.write(truth_df)
+                # df = pd.concat([df,truth_df['labels']],axis=1)
+                # df['Validation'] =  'No'
+                # df['Sector1'] = 'Blank'
+                # df['Sector2'] = 'Blank'
+                # df['Sector3'] = 'Blank'
+                # df['Sector4'] = 'Blank'
+                # df['Sector5'] = 'Blank'
+                # df_xlsx = to_excel(df,categories)
+                # st.download_button(label='📥 Download Current Result',
+                #                 data=df_xlsx ,
+            #     #               file_name= 'file_sector.xlsx')
+            # else:
+            #     st.info("🤔 No document found, please try to upload it at the sidebar!")
+            #     logging.warning("Terminated as no document provided")
+        
+        # # Creating truth value dataframe
+        # if 'key' in st.session_state:
+        #     if st.session_state.key is not None:
+        #         df = st.session_state.key
+        #         st.markdown("###### Select the threshold for classifier ######")
+        #         c4, c5 = st.columns([1,1])
+
+        #         with c4:                    
+        #             threshold = st.slider("Threshold", min_value=0.00, max_value=1.0,
+        #                                   step=0.01, value=0.5,
+        #                 help = "Keep High Value if want refined result, low if dont want to miss anything" )
+        #         sectors =set(df.columns)
+        #         removecols = {'Validation','Sector1','Sector2','Sector3','Sector4',
+        #                       'Sector5','text'}
+        #         sectors  = list(sectors - removecols)
+
+        #         placeholder = {}
+        #         for val in sectors:
+        #             temp = df[val].astype(float) > threshold
+        #             temp = temp.astype(str)
+        #             placeholder[val] = dict(temp.value_counts())
+                    
+        #         count_df = pd.DataFrame.from_dict(placeholder)
+        #         count_df = count_df.T
+        #         count_df = count_df.reset_index()
+        #         placeholder  = []
+        #         for i in range(len(count_df)):
+        #             placeholder.append([count_df.iloc[i]['index'],count_df['False'][i],'False'])
+        #             placeholder.append([count_df.iloc[i]['index'],count_df['True'][i],'True'])
+
+        #         count_df = pd.DataFrame(placeholder, columns = ['sector','count','truth_value'])
+        #         fig = px.bar(count_df, x='sector', y='count',
+        #                     color='truth_value',
+        #                     height=400)
+        #         st.write("")
+        #         st.plotly_chart(fig)
+
+                # df['Validation'] =  'No'
+                # df['Sector1'] = 'Blank'
+                # df['Sector2'] = 'Blank'
+                # df['Sector3'] = 'Blank'
+                # df['Sector4'] = 'Blank'
+                # df['Sector5'] = 'Blank'
+                # df_xlsx = to_excel(df,sectors)
+                # st.download_button(label='📥 Download Current Result',
+                #                 data=df_xlsx ,
+                #               file_name= 'file_sector.xlsx')

appStore/policyaction.py

@@ -0,0 +1,238 @@
+# set path
+import glob, os, sys; 
+sys.path.append('../utils')
+
+#import needed libraries
+import seaborn as sns
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import streamlit as st
+from utils.policyaction_classifier import load_policyactionClassifier, policyaction_classification
+import logging
+logger = logging.getLogger(__name__)
+from utils.config import get_classifier_params
+from utils.preprocessing import paraLengthCheck
+from io import BytesIO
+import xlsxwriter
+import plotly.express as px
+
+
+# Declare all the necessary variables
+classifier_identifier = 'policyaction'
+params  = get_classifier_params(classifier_identifier)
+
+@st.cache_data
+def to_excel(df):
+    df['Target Validation'] = 'No'
+    df['Netzero Validation'] = 'No'
+    df['GHG Validation'] = 'No'
+    df['Adapt-Mitig Validation'] = 'No'
+    df['Sector'] = 'No'
+    len_df = len(df)
+    output = BytesIO()
+    writer = pd.ExcelWriter(output, engine='xlsxwriter')
+    df.to_excel(writer, index=False, sheet_name='Sheet1')
+    workbook = writer.book
+    worksheet = writer.sheets['Sheet1']
+    worksheet.data_validation('L2:L{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': ['No', 'Yes', 'Discard']})
+    worksheet.data_validation('M2:L{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': ['No', 'Yes', 'Discard']})
+    worksheet.data_validation('N2:L{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': ['No', 'Yes', 'Discard']})
+    worksheet.data_validation('O2:L{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': ['No', 'Yes', 'Discard']})
+    worksheet.data_validation('P2:L{}'.format(len_df), 
+                              {'validate': 'list', 
+                               'source': ['No', 'Yes', 'Discard']})                                                                                                         
+    writer.save()
+    processed_data = output.getvalue()
+    return processed_data
+
+def app():
+
+    ### Main app code ###
+    with st.container():
+                   
+            if 'key1' in st.session_state:
+                df = st.session_state.key1
+                classifier = load_policyactionClassifier(classifier_name=params['model_name'])
+                st.session_state['{}_classifier'.format(classifier_identifier)] = classifier
+
+                if sum(df['Target Label'] == 'TARGET') > 100:
+                    warning_msg = ": This might take sometime, please sit back and relax."
+                else:
+                    warning_msg = ""
+                    
+                df = policyaction_classification(haystack_doc=df,
+                                            threshold= params['threshold'])
+
+                st.session_state.key1 = df
+
+
+
+def action_display():
+    if  'key1' in st.session_state:
+        df = st.session_state.key1
+                
+        
+        df['Action_check']  = df['Policy-Action Label'].apply(lambda x: True if 'Action' in x else False)
+        hits  = df[df['Action_check'] == True]
+        # hits['GHG Label'] = hits['GHG Label'].apply(lambda i: _lab_dict[i])
+        range_val = min(5,len(hits))
+        if range_val !=0:
+            count_action = len(hits)
+            #count_netzero = sum(hits['Netzero Label'] == 'NETZERO')
+            #count_ghg = sum(hits['GHG Label'] == 'GHG')
+            #count_economy = sum([True if 'Economy-wide' in x else False 
+            #                  for x in hits['Sector Label']])
+            
+            # count_df = df['Target Label'].value_counts()
+            # count_df = count_df.rename('count')
+            # count_df = count_df.rename_axis('Target Label').reset_index()
+            # count_df['Label_def'] = count_df['Target Label'].apply(lambda x: _lab_dict[x])
+
+            # fig = px.bar(count_df, y="Label_def", x="count", orientation='h', height=200)
+        #    c1, c2 = st.columns([1,1])
+        #    with c1:
+        #        st.write('**Target Paragraphs**: `{}`'.format(count_target))
+        #        st.write('**NetZero Related Paragraphs**: `{}`'.format(count_netzero))
+#
+ #               # st.plotly_chart(fig,use_container_width= True)
+  #          
+            # count_netzero = sum(hits['Netzero Label'] == 'NETZERO')
+            # count_ghg = sum(hits['GHG Label'] == 'LABEL_2')
+            # count_economy = sum([True if 'Economy-wide' in x else False 
+            #                   for x in hits['Sector Label']])
+     #       with c2:
+      #          st.write('**GHG Related Paragraphs**: `{}`'.format(count_ghg))
+       #         st.write('**Economy-wide Related Paragraphs**: `{}`'.format(count_economy))
+        #    st.write('-------------------')    
+    #        hits = hits.sort_values(by=['Relevancy'], ascending=False)
+     #       netzerohit = hits[hits['Netzero Label'] == 'NETZERO']
+      #      if not netzerohit.empty:
+       #         netzerohit = netzerohit.sort_values(by = ['Netzero Score'], ascending = False)
+        #        # st.write('-------------------')
+                # st.markdown("###### Netzero paragraph ######")
+      #          st.write('**Netzero paragraph** `page {}`: {}'.format(netzerohit.iloc[0]['page'],
+      #                          netzerohit.iloc[0]['text'].replace("\n", " ")))                        
+       #         st.write("")
+        #    else:
+         #       st.info("🤔 No Netzero paragraph found")
+
+            # st.write("**Result {}** `page {}` (Relevancy Score: {:.2f})'".format(i+1,hits.iloc[i]['page'],hits.iloc[i]['Relevancy'])")
+           # st.write('-------------------')
+            st.write("")
+            st.markdown("###### Top few Action Classified paragraph/text results from list of {} classified paragraphs ######".format(count_action))
+            st.markdown("""<hr style="height:10px;border:none;color:#097969;background-color:#097969;" /> """, unsafe_allow_html=True)
+            range_val = min(5,len(hits))
+            for i in range(range_val):
+                # the page number reflects the page that contains the main paragraph 
+                # according to split limit, the overlapping part can be on a separate page
+                st.write('**Result {}** : `page {}`, `Sector: {}`,\
+                            `Indicators: {}`, `Adapt-Mitig :{}`'\
+                    .format(i+1,
+                            hits.iloc[i]['page'], hits.iloc[i]['Sector Label'],
+                            hits.iloc[i]['Indicator Label'],hits.iloc[i]['Adapt-Mitig Label']))                        
+                st.write("\t Text: \t{}".format(hits.iloc[i]['text'].replace("\n", " ")))
+            hits = hits.reset_index(drop =True)
+            st.write('----------------')
+            st.write('Explore the data')
+            st.write(hits)
+            df.drop(columns = ['Action_check'],inplace=True)
+            df_xlsx = to_excel(df)
+            
+            with st.sidebar:
+                st.write('-------------')
+                st.download_button(label='📥 Download Result',
+                            data=df_xlsx ,
+                            file_name= 'cpu_analysis.xlsx')
+
+        else:
+            st.info("🤔 No Actions found")
+
+
+def policy_display():
+    if  'key1' in st.session_state:
+        df = st.session_state.key1
+                
+        
+        df['Policy_check']  = df['Policy-Action Label'].apply(lambda x: True if 'Policies & Plans' in x else False)
+        hits  = df[df['Policy_check'] == True]
+        # hits['GHG Label'] = hits['GHG Label'].apply(lambda i: _lab_dict[i])
+        range_val = min(5,len(hits))
+        if range_val !=0:
+            count_policy = len(hits)
+            #count_netzero = sum(hits['Netzero Label'] == 'NETZERO')
+            #count_ghg = sum(hits['GHG Label'] == 'GHG')
+            #count_economy = sum([True if 'Economy-wide' in x else False 
+            #                  for x in hits['Sector Label']])
+            
+            # count_df = df['Target Label'].value_counts()
+            # count_df = count_df.rename('count')
+            # count_df = count_df.rename_axis('Target Label').reset_index()
+            # count_df['Label_def'] = count_df['Target Label'].apply(lambda x: _lab_dict[x])
+
+            # fig = px.bar(count_df, y="Label_def", x="count", orientation='h', height=200)
+        #    c1, c2 = st.columns([1,1])
+        #    with c1:
+        #        st.write('**Target Paragraphs**: `{}`'.format(count_target))
+        #        st.write('**NetZero Related Paragraphs**: `{}`'.format(count_netzero))
+#
+ #               # st.plotly_chart(fig,use_container_width= True)
+  #          
+            # count_netzero = sum(hits['Netzero Label'] == 'NETZERO')
+            # count_ghg = sum(hits['GHG Label'] == 'LABEL_2')
+            # count_economy = sum([True if 'Economy-wide' in x else False 
+            #                   for x in hits['Sector Label']])
+     #       with c2:
+      #          st.write('**GHG Related Paragraphs**: `{}`'.format(count_ghg))
+       #         st.write('**Economy-wide Related Paragraphs**: `{}`'.format(count_economy))
+        #    st.write('-------------------')    
+    #        hits = hits.sort_values(by=['Relevancy'], ascending=False)
+     #       netzerohit = hits[hits['Netzero Label'] == 'NETZERO']
+      #      if not netzerohit.empty:
+       #         netzerohit = netzerohit.sort_values(by = ['Netzero Score'], ascending = False)
+        #        # st.write('-------------------')
+                # st.markdown("###### Netzero paragraph ######")
+      #          st.write('**Netzero paragraph** `page {}`: {}'.format(netzerohit.iloc[0]['page'],
+      #                          netzerohit.iloc[0]['text'].replace("\n", " ")))                        
+       #         st.write("")
+        #    else:
+         #       st.info("🤔 No Netzero paragraph found")
+
+            # st.write("**Result {}** `page {}` (Relevancy Score: {:.2f})'".format(i+1,hits.iloc[i]['page'],hits.iloc[i]['Relevancy'])")
+           # st.write('-------------------')
+            st.write("")
+            st.markdown("###### Top few Policy/Plans Classified paragraph/text results from list of {} classified paragraphs ######".format(count_policy))
+            st.markdown("""<hr style="height:10px;border:none;color:#097969;background-color:#097969;" /> """, unsafe_allow_html=True)
+            range_val = min(5,len(hits))
+            for i in range(range_val):
+                # the page number reflects the page that contains the main paragraph 
+                # according to split limit, the overlapping part can be on a separate page
+                st.write('**Result {}** : `page {}`, `Sector: {}`,\
+                            `Indicators: {}`, `Adapt-Mitig :{}`'\
+                    .format(i+1,
+                            hits.iloc[i]['page'], hits.iloc[i]['Sector Label'],
+                            hits.iloc[i]['Indicator Label'],hits.iloc[i]['Adapt-Mitig Label']))                        
+                st.write("\t Text: \t{}".format(hits.iloc[i]['text'].replace("\n", " ")))
+            hits = hits.reset_index(drop =True)
+            st.write('----------------')
+            st.write('Explore the data')
+            st.write(hits)
+            df.drop(columns = ['Policy_check'],inplace=True)
+            df_xlsx = to_excel(df)
+            
+            with st.sidebar:
+                st.write('-------------')
+                st.download_button(label='📥 Download Result',
+                            data=df_xlsx ,
+                            file_name= 'cpu_analysis.xlsx')
+
+        else:
+            st.info("🤔 No Policy/Plans found")

appStore/target.py

@@ -102,13 +102,7 @@ def target_display():
     if  'key1' in st.session_state:
         df = st.session_state.key1
                 
-        _lab_dict = {
-                      'LABEL_0':'NEGATIVE',
-                      'LABEL_1':'NOT GHG',
-                      'LABEL_2':'GHG',
-                      'NA':'NA',
-                      }
-        df['GHG Label'] = df['GHG Label'].apply(lambda i: _lab_dict[i])
+        
         hits  = df[df['Target Label'] == 'TARGET']
         # hits['GHG Label'] = hits['GHG Label'].apply(lambda i: _lab_dict[i])
         range_val = min(5,len(hits))

paramconfig.cfg

@@ -56,4 +56,24 @@ REMOVE_PUNC = 0
 SPLIT_LENGTH = 60
 SPLIT_OVERLAP = 10
 RESPECT_SENTENCE_BOUNDARY = 1
+TOP_KEY = 10
+
+[policyaction]
+THRESHOLD = 0.50
+MODEL = ppsingh/action-policy-plans-classifier
+SPLIT_BY = word
+REMOVE_PUNC = 0
+SPLIT_LENGTH = 60
+SPLIT_OVERLAP = 10
+RESPECT_SENTENCE_BOUNDARY = 1
+TOP_KEY = 10
+
+[indicator]
+THRESHOLD = 0.50
+MODEL = ilaria-oneofftech/ikitracs_mitigation
+SPLIT_BY = word
+REMOVE_PUNC = 0
+SPLIT_LENGTH = 60
+SPLIT_OVERLAP = 10
+RESPECT_SENTENCE_BOUNDARY = 1
 TOP_KEY = 10

requirements.txt

@@ -15,5 +15,6 @@ markdown==3.4.1
 summa==1.2.0
 plotly
 xlsxwriter
+altair==4.0
 streamlit-aggrid
 python-docx

utils/adapmit_classifier.py

@@ -67,13 +67,13 @@ def adapmit_classification(haystack_doc:pd.DataFrame,
     """
     logging.info("Working on Adaptation-Mitigation Identification")
     haystack_doc['Adapt-Mitig Label'] = 'NA'
-    df1 = haystack_doc[haystack_doc['Target Label'] == 'TARGET']
-    df = haystack_doc[haystack_doc['Target Label'] == 'NEGATIVE']
+   # df1 = haystack_doc[haystack_doc['Target Label'] == 'TARGET']
+   # df = haystack_doc[haystack_doc['Target Label'] == 'NEGATIVE']
 
     if not classifier_model:
         classifier_model = st.session_state['adapmit_classifier']
     
-    predictions = classifier_model(list(df1.text))
+    predictions = classifier_model(list(haystack_doc.text))
      # converting the predictions to desired format
     list_ = []
     for i in range(len(predictions)):
@@ -93,7 +93,7 @@ def adapmit_classification(haystack_doc:pd.DataFrame,
                                         else None for i in categories}, axis=1)
     truth_df['Adapt-Mitig Label'] = truth_df.apply(lambda x: 
                                   list(x['Adapt-Mitig Label'] -{None}),axis=1)
-    df1['Adapt-Mitig Label'] = list(truth_df['Adapt-Mitig Label'])
-    df = pd.concat([df,df1])
+    haystack_doc['Adapt-Mitig Label'] = list(truth_df['Adapt-Mitig Label'])
+    #df = pd.concat([df,df1])
 
-    return df
+    return haystack_doc

utils/ghg_classifier.py

@@ -10,9 +10,12 @@ from transformers import pipeline
 
 # Labels dictionary ###
 _lab_dict = {
-            'NEGATIVE':'NO GHG TARGET',
-            'TARGET':'GHG TARGET',
-            }
+                      'LABEL_0':'NEGATIVE',
+                      'LABEL_1':'NOT GHG',
+                      'LABEL_2':'GHG',
+                      'NA':'NA',
+                      }
+        
 
 @st.cache_resource
 def load_ghgClassifier(config_file:str = None, classifier_name:str = None):
@@ -82,6 +85,7 @@ def ghg_classification(haystack_doc:pd.DataFrame,
     labels_= [(l[0]['label'],l[0]['score']) for l in results]
     temp['GHG Label'],temp['GHG Score'] = zip(*labels_)
     df = pd.concat([df,temp])
+    df['GHG Label'] = df['GHG Label'].apply(lambda i: _lab_dict[i])
     df = df.reset_index(drop =True)
     df.index += 1
 

utils/indicator_classifier.py

@@ -0,0 +1,109 @@
+from typing import List, Tuple
+from typing_extensions import Literal
+import logging
+import pandas as pd
+from pandas import DataFrame, Series
+from utils.config import getconfig
+from utils.preprocessing import processingpipeline
+import streamlit as st
+from transformers import pipeline
+
+
+@st.cache_resource
+def load_indicatorClassifier(config_file:str = None, classifier_name:str = None):
+    """
+    loads the document classifier using haystack, where the name/path of model
+    in HF-hub as string is used to fetch the model object.Either configfile or 
+    model should be passed.
+    1. https://docs.haystack.deepset.ai/reference/document-classifier-api
+    2. https://docs.haystack.deepset.ai/docs/document_classifier
+    Params
+    --------
+    config_file: config file path from which to read the model name
+    classifier_name: if modelname is passed, it takes a priority if not \
+    found then will look for configfile, else raise error.
+    Return: document classifier model
+    """
+    if not classifier_name:
+        if not config_file:
+            logging.warning("Pass either model name or config file")
+            return
+        else:
+            config = getconfig(config_file)
+            classifier_name = config.get('indicator','MODEL')
+    
+    logging.info("Loading indicator classifier")
+    # we are using the pipeline as the model is multilabel and DocumentClassifier 
+    # from Haystack doesnt support multilabel
+    # in pipeline we use 'sigmoid' to explicitly tell pipeline to make it multilabel
+    # if not then it will automatically use softmax, which is not a desired thing.
+    # doc_classifier = TransformersDocumentClassifier(
+    #                     model_name_or_path=classifier_name,
+    #                     task="text-classification",
+    #                     top_k = None)
+
+    doc_classifier = pipeline("text-classification", 
+                            model=classifier_name, 
+                            return_all_scores=True, 
+                            function_to_apply= "sigmoid")
+
+    return doc_classifier
+
+
+@st.cache_data
+def indicator_classification(haystack_doc:pd.DataFrame,
+                        threshold:float = 0.5, 
+                        classifier_model:pipeline= None
+                        )->Tuple[DataFrame,Series]:
+    """
+    Text-Classification on the list of texts provided. Classifier provides the 
+    most appropriate label for each text. these labels are in terms of if text 
+    belongs to which particular Sustainable Devleopment Goal (SDG).
+    Params
+    ---------
+    haystack_doc: List of haystack Documents. The output of Preprocessing Pipeline 
+    contains the list of paragraphs in different format,here the list of 
+    Haystack Documents is used.
+    threshold: threshold value for the model to keep the results from classifier
+    classifiermodel: you can pass the classifier model directly,which takes priority
+    however if not then looks for model in streamlit session.
+    In case of streamlit avoid passing the model directly.
+    Returns
+    ----------
+    df: Dataframe with two columns['SDG:int', 'text']
+    x: Series object with the unique SDG covered in the document uploaded and 
+    the number of times it is covered/discussed/count_of_paragraphs. 
+    """
+    logging.info("Working on Indicator Identification")
+    haystack_doc['Indicator Label'] = 'NA'
+    haystack_doc['PA_check'] = haystack_doc['Policy-Action Label'].apply(lambda x: True if len(x) != 0 else False)
+                            
+    df1 = haystack_doc[haystack_doc['PA_check'] == True]
+    df = haystack_doc[haystack_doc['PA_check'] == False]
+    if not classifier_model:
+        classifier_model = st.session_state['indicator_classifier']
+    
+        predictions = classifier_model(list(df1.text))
+
+    list_ = []
+    for i in range(len(predictions)):
+
+      temp = predictions[i]
+      placeholder = {}
+      for j in range(len(temp)):
+        placeholder[temp[j]['label']] = temp[j]['score']
+      list_.append(placeholder)
+    labels_ = [{**list_[l]} for l in range(len(predictions))]
+    truth_df = DataFrame.from_dict(labels_)
+    truth_df = truth_df.round(2)
+    truth_df = truth_df.astype(float) >= threshold
+    truth_df = truth_df.astype(str)
+    categories = list(truth_df.columns)
+    truth_df['Indicator Label'] = truth_df.apply(lambda x: {i if x[i]=='True' else 
+                                              None for i in categories}, axis=1)
+    truth_df['Indicator Label'] = truth_df.apply(lambda x: list(x['Indicator Label'] 
+                                                            -{None}),axis=1)
+    df1['Indicator Label'] = list(truth_df['Indicator Label'])
+    df = pd.concat([df,df1])
+    df = df.drop(columns = ['PA_check'])
+    return df

utils/policyaction_classifier.py

@@ -0,0 +1,101 @@
+from typing import List, Tuple
+from typing_extensions import Literal
+import logging
+import pandas as pd
+from pandas import DataFrame, Series
+from utils.config import getconfig
+from utils.preprocessing import processingpipeline
+import streamlit as st
+from transformers import pipeline
+
+## Labels dictionary ###
+_lab_dict = {
+            'NEGATIVE':'NO TARGET INFO',
+            'TARGET':'TARGET',
+            }
+
+@st.cache_resource
+def load_policyactionClassifier(config_file:str = None, classifier_name:str = None):
+    """
+    loads the document classifier using haystack, where the name/path of model
+    in HF-hub as string is used to fetch the model object.Either configfile or 
+    model should be passed.
+    1. https://docs.haystack.deepset.ai/reference/document-classifier-api
+    2. https://docs.haystack.deepset.ai/docs/document_classifier
+    Params
+    --------
+    config_file: config file path from which to read the model name
+    classifier_name: if modelname is passed, it takes a priority if not \
+    found then will look for configfile, else raise error.
+    Return: document classifier model
+    """
+    if not classifier_name:
+        if not config_file:
+            logging.warning("Pass either model name or config file")
+            return
+        else:
+            config = getconfig(config_file)
+            classifier_name = config.get('policyaction','MODEL')
+    
+    logging.info("Loading classifier")  
+      
+    doc_classifier = pipeline("text-classification",
+                              model=classifier_name,
+                            return_all_scores=True, 
+                            function_to_apply= "sigmoid")
+
+    return doc_classifier
+
+
+@st.cache_data
+def policyaction_classification(haystack_doc:pd.DataFrame,
+                        threshold:float = 0.5, 
+                        classifier_model:pipeline= None
+                        )->Tuple[DataFrame,Series]:
+    """
+    Text-Classification on the list of texts provided. Classifier provides the 
+    most appropriate label for each text. these labels are in terms of if text 
+    belongs to which particular Sustainable Devleopment Goal (SDG).
+    Params
+    ---------
+    haystack_doc: List of haystack Documents. The output of Preprocessing Pipeline 
+    contains the list of paragraphs in different format,here the list of 
+    Haystack Documents is used.
+    threshold: threshold value for the model to keep the results from classifier
+    classifiermodel: you can pass the classifier model directly,which takes priority
+    however if not then looks for model in streamlit session.
+    In case of streamlit avoid passing the model directly.
+    Returns
+    ----------
+    df: Dataframe with two columns['SDG:int', 'text']
+    x: Series object with the unique SDG covered in the document uploaded and 
+    the number of times it is covered/discussed/count_of_paragraphs. 
+    """
+    logging.info("Working on Policy/Action. Extraction")
+    haystack_doc['Policy-Action Label'] = 'NA'                       
+    if not classifier_model:
+        classifier_model = st.session_state['policyaction_classifier']
+    
+    predictions = classifier_model(list(haystack_doc.text))
+    list_ = []
+    for i in range(len(predictions)):
+
+      temp = predictions[i]
+      placeholder = {}
+      for j in range(len(temp)):
+        placeholder[temp[j]['label']] = temp[j]['score']
+      list_.append(placeholder)
+    labels_ = [{**list_[l]} for l in range(len(predictions))]
+    truth_df = DataFrame.from_dict(labels_)
+    truth_df = truth_df.round(2)
+    truth_df = truth_df.astype(float) >= threshold
+    truth_df = truth_df.astype(str)
+    categories = list(truth_df.columns)
+    truth_df['Policy-Action Label'] = truth_df.apply(lambda x: {i if x[i]=='True' 
+                                        else None for i in categories}, axis=1)
+    truth_df['Policy-Action Label'] = truth_df.apply(lambda x: 
+                                  list(x['Policy-Action Label'] -{None}),axis=1)
+
+    haystack_doc['Policy-Action Label'] =  list(truth_df['Policy-Action Label'])   
+
+    return haystack_doc

utils/sector_classifier.py

@@ -76,12 +76,12 @@ def sector_classification(haystack_doc:pd.DataFrame,
     """
     logging.info("Working on Sector Identification")
     haystack_doc['Sector Label'] = 'NA'
-    df1 = haystack_doc[haystack_doc['Target Label'] == 'TARGET']
-    df = haystack_doc[haystack_doc['Target Label'] == 'NEGATIVE']
+ #   df1 = haystack_doc[haystack_doc['Target Label'] == 'TARGET']
+  #  df = haystack_doc[haystack_doc['Target Label'] == 'NEGATIVE']
     if not classifier_model:
         classifier_model = st.session_state['sector_classifier']
     
-        predictions = classifier_model(list(df1.text))
+        predictions = classifier_model(list(haystack_doc.text))
 
     list_ = []
     for i in range(len(predictions)):
@@ -101,6 +101,6 @@ def sector_classification(haystack_doc:pd.DataFrame,
                                               None for i in categories}, axis=1)
     truth_df['Sector Label'] = truth_df.apply(lambda x: list(x['Sector Label'] 
                                                             -{None}),axis=1)
-    df1['Sector Label'] = list(truth_df['Sector Label'])
-    df = pd.concat([df,df1])
-    return df
+    haystack_doc['Sector Label'] = list(truth_df['Sector Label'])
+  #  df = pd.concat([df,df1])
+    return haystack_doc