Upload 4 files

appStore/info.py

@@ -0,0 +1,72 @@
+import streamlit as st
+
+def app():
+    
+    
+    with open('style.css') as f:
+        st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
+    
+    st.markdown("<h2 style='text-align: center;  \
+                      color: black;'> Policy Action Tracker</h2>", 
+                      unsafe_allow_html=True)
+
+    
+    st.markdown("<div style='text-align: center; \
+                    color: grey;'>The Policy Action Tracker is an open-source\
+                         digital tool which aims to assist policy analysts and \
+                          other users in extracting and filtering relevant \
+                            information from policy documents.</div>",
+                        unsafe_allow_html=True)
+    footer = """
+           <div class="footer-custom">
+               Guidance & Feedback - <a href="https://www.linkedin.com/in/maren-bernlöhr-149891222" target="_blank">Maren Bernlöhr</a> |
+               <a href="https://www.linkedin.com/in/manuelkuhm" target="_blank">Manuel Kuhm</a> |
+               Developer - <a href="https://www.linkedin.com/in/erik-lehmann-giz/" target="_blank">Erik Lehmann</a>  |   
+               <a href="https://www.linkedin.com/in/jonas-nothnagel-bb42b114b/" target="_blank">Jonas Nothnagel</a>   |
+               <a href="https://www.linkedin.com/in/prashantpsingh/" target="_blank">Prashant Singh</a> |
+               
+           </div>
+       """
+    st.markdown(footer, unsafe_allow_html=True)
+
+    c1, c2, c3 =  st.columns([8,1,12])
+    with c1:
+        st.image("docStore/img/ndc.png")
+    with c3:
+        st.markdown('<div style="text-align: justify;">The manual extraction \
+        of relevant information from text documents is a \
+    time-consuming task for any policy analyst. As the amount and length of \
+    public policy documents in relation to sustainable development (such as \
+    National Development Plans and Nationally Determined Contributions) \
+    continuously increases, a major challenge for policy action tracking – the \
+    evaluation of stated goals and targets and their actual implementation on \
+    the ground – arises. Luckily, Artificial Intelligence (AI) and Natural \
+    Language Processing (NLP) methods can help in shortening and easing this \
+    task for policy analysts.</div><br>',
+    unsafe_allow_html=True)
+
+    intro = """
+    <div style="text-align: justify;">
+
+    For this purpose, the United Nations Sustainable Development Solutions \
+    Network (SDSN) and the Deutsche Gesellschaft für Internationale \
+    Zusammenarbeit (GIZ) GmbH are collaborated in the development \
+    of this AI-powered open-source web application that helps find and extract \
+    relevant information from public policy documents faster to facilitate \
+    evidence-based decision-making processes in sustainable development and beyond.  
+
+    This tool allows policy analysts and other users the possibility to rapidly \
+    search for relevant information/paragraphs in the document according to the \
+    user’s interest, classify the document’s content according to the Sustainable \
+    Development Goals (SDGs), and compare climate-related policy documents and NDCs \
+    across countries using open data from the German Institute of Development and \
+    Sustainability’s (IDOS) NDC Explorer. 
+    To understand the application's functionalities and learn more about ß
+    the project, see the attached concept note. We hope you like our application 😊
+
+
+    </div>
+    <br>
+    """
+    st.markdown(intro, unsafe_allow_html=True)
+    # st.image("docStore/img/paris.png")

appStore/keyword_search.py

@@ -0,0 +1,176 @@
+# set path
+import glob, os, sys; 
+sys.path.append('../utils')
+
+import streamlit as st
+import json
+import logging
+from utils.lexical_search import runLexicalPreprocessingPipeline, lexical_search
+from utils.semantic_search import runSemanticPreprocessingPipeline, semantic_keywordsearch
+from utils.checkconfig import getconfig
+from utils.streamlitcheck import checkbox_without_preselect
+
+# Declare all the necessary variables
+config = getconfig('paramconfig.cfg')
+split_by = config.get('semantic_search','SPLIT_BY')
+split_length = int(config.get('semantic_search','SPLIT_LENGTH'))
+split_overlap = int(config.get('semantic_search','SPLIT_OVERLAP'))
+split_respect_sentence_boundary = bool(int(config.get('semantic_search',
+                                    'RESPECT_SENTENCE_BOUNDARY')))
+remove_punc = bool(int(config.get('semantic_search','REMOVE_PUNC')))
+embedding_model = config.get('semantic_search','RETRIEVER')
+embedding_model_format = config.get('semantic_search','RETRIEVER_FORMAT')
+embedding_layer = int(config.get('semantic_search','RETRIEVER_EMB_LAYER'))
+embedding_dim  = int(config.get('semantic_search','EMBEDDING_DIM'))
+max_seq_len = int(config.get('semantic_search','MAX_SEQ_LENGTH')) 
+retriever_top_k = int(config.get('semantic_search','RETRIEVER_TOP_K'))
+reader_model = config.get('semantic_search','READER')
+reader_top_k = int(config.get('semantic_search','RETRIEVER_TOP_K'))
+top_k_per_candidate = int(config.get('semantic_search','READER_TOP_K_PER_CANDIDATE')) 
+lexical_split_by= config.get('lexical_search','SPLIT_BY')
+lexical_split_length=int(config.get('lexical_search','SPLIT_LENGTH'))
+lexical_split_overlap = int(config.get('lexical_search','SPLIT_OVERLAP'))
+lexical_remove_punc = bool(int(config.get('lexical_search','REMOVE_PUNC')))
+lexical_top_k=int(config.get('lexical_search','TOP_K'))
+
+def app():
+
+    with st.container():
+        st.markdown("<h1 style='text-align: center;  \
+                      color: black;'> Search</h1>", 
+                      unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=False):
+
+        st.write(
+            """     
+            The *Search* app is an interface \
+            for doing contextual and keyword searches in \
+            policy documents.  \
+            """)
+        st.write("")
+        st.write(""" The application allows its user to perform a search\
+             based on two options: a lexical search([TFIDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf))\
+              and semantic search. [bi-encoder](https://www.sbert.net/examples/applications/retrieve_rerank/README.html)\
+             The lexical search only \
+            displays paragraphs in the document with exact matching results, \
+            the semantic search shows paragraphs with meaningful connections \
+            (e.g., synonyms) based on the search context. Both \
+            methods employ a probabilistic retrieval framework in its identification\
+            of relevant paragraphs. By defualt the search is performed using \
+            'Semantic Search', and to find 'Exact/Lexical Matches' please tick the \
+            checkbox provided which will by-pass semantic search. Furthermore,\
+            the application allows the user to search for pre-defined keywords \
+            from different thematic buckets present in sidebar.""")
+        st.write("")
+        st.write(""" The Exact Matches gives back top {} findings, and Semantic
+        search provides with top {} answers.""".format(lexical_top_k, retriever_top_k))
+        st.write("")
+        st.write("")
+        st.markdown("Some runtime metrics tested with cpu: Intel(R) Xeon(R) CPU @ 2.20GHz, memory: 13GB")
+        col1,col2,col3= st.columns([2,4,4])
+        with col1:
+            st.caption("OCR File processing")
+            # st.markdown('<div style="text-align: center;">50 sec</div>', unsafe_allow_html=True)
+            st.write("50 sec")
+           
+        with col2:
+            st.caption("Lexical Search on 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">12 sec</div>', unsafe_allow_html=True)
+            st.write("15 sec")
+           
+        with col3:
+            st.caption("Semantic search on 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">120 sec</div>', unsafe_allow_html=True)
+            st.write("120 sec(including emebedding creation)")
+ 
+    with st.sidebar:
+        with open('docStore/sample/keywordexample.json','r') as json_file:
+            keywordexample = json.load(json_file)
+        
+        # genre = st.radio("Select Keyword Category", list(keywordexample.keys()))
+        st.caption("Select Keyword Category")
+        genre = checkbox_without_preselect(list(keywordexample.keys()))
+        if genre:
+            keywordList = keywordexample[genre]
+        else:
+            keywordList = None
+        
+        st.markdown("---")
+    
+    with st.container():
+        type_hinting = "Please enter here your question and we \
+                        will look for an answer in the document\
+                        OR enter the keyword you are looking \
+                        for and we will look for similar\
+                        context in the document.\
+                        You can also explore predefined sets of keywords from sidebar. "
+        if keywordList is not None:
+        #     queryList = st.text_input("You selected the {} category we \
+        #                 will look for these keywords in document".format(genre)
+        #                             value="{}".format(keywordList))
+            queryList = st.text_input(type_hinting,
+                                        value = "{}".format(keywordList))
+        else:
+             queryList = st.text_input(type_hinting,
+                                       placeholder="Enter keyword/query here")
+
+        searchtype = st.checkbox("Show only Exact Matches")
+        if st.button("Find them"):
+
+            if queryList == "":
+                st.info("🤔 No keyword provided, if you dont have any, \
+                                please try example sets from sidebar!")
+                logging.warning("Terminated as no keyword provided")
+            else:
+                if 'filepath' in st.session_state:
+                      
+                    if searchtype:
+                        all_documents = runLexicalPreprocessingPipeline(
+                                    file_name=st.session_state['filename'],
+                                    file_path=st.session_state['filepath'],
+                                    split_by=lexical_split_by,
+                                    split_length=lexical_split_length,
+                                    split_overlap=lexical_split_overlap,
+                                    remove_punc=lexical_remove_punc)
+                        logging.info("performing lexical search")
+                        with st.spinner("Performing Exact matching search \
+                                        (Lexical search) for you"):
+                            lexical_search(query=queryList,
+                        documents = all_documents['documents'],
+                                top_k = lexical_top_k )
+                    else:
+                        all_documents = runSemanticPreprocessingPipeline(
+                                            file_path= st.session_state['filepath'],
+                                            file_name  = st.session_state['filename'],
+                                            split_by=split_by,
+                                            split_length= split_length,
+                                            split_overlap=split_overlap,
+                                            remove_punc= remove_punc,
+                        split_respect_sentence_boundary=split_respect_sentence_boundary)
+                        if len(all_documents['documents']) > 100:
+                            warning_msg = ": This might take sometime, please sit back and relax."
+                        else:
+                            warning_msg = ""
+
+                        logging.info("starting semantic search")
+                        with st.spinner("Performing Similar/Contextual search{}".format(warning_msg)):
+                            semantic_keywordsearch(query = queryList, 
+                            documents = all_documents['documents'],
+                            embedding_model=embedding_model, 
+                            embedding_layer=embedding_layer,
+                            embedding_model_format=embedding_model_format,
+                            reader_model=reader_model,reader_top_k=reader_top_k,
+                            retriever_top_k=retriever_top_k, embedding_dim=embedding_dim,
+                            max_seq_len=max_seq_len,
+                            top_k_per_candidate = top_k_per_candidate)
+
+                else:
+                    st.info("🤔 No document found, please try to upload it at the sidebar!")
+                    logging.warning("Terminated as no document provided")
+        
+
+
+                    

appStore/multiapp.py

@@ -0,0 +1,70 @@
+"""Frameworks for running multiple Streamlit applications as a single app.
+"""
+import streamlit as st
+from PIL import Image
+from streamlit_option_menu import option_menu
+from utils.uploadAndExample import add_upload
+
+class MultiApp:
+    """Framework for combining multiple streamlit applications.
+    Usage:
+        def foo():
+            st.title("Hello Foo")
+        def bar():
+            st.title("Hello Bar")
+        app = MultiApp()
+        app.add_app("Foo", foo)
+        app.add_app("Bar", bar)
+        app.run()
+    It is also possible keep each application in a separate file.
+        import foo
+        import bar
+        app = MultiApp()
+        app.add_app("Foo", foo.app)
+        app.add_app("Bar", bar.app)
+        app.run()
+    """
+    def __init__(self):
+        self.apps = []
+
+    def add_app(self,title,icon, func):
+        """Adds a new application.
+        Parameters
+        ----------
+        func:
+            the python function to render this app.
+        title:
+            title of the app. Appears in the dropdown in the sidebar.
+        """
+        self.apps.append({
+            "title": title,
+            "icon": icon,
+            "function": func
+        })
+
+    def run(self):
+        
+        st.sidebar.write(format_func=lambda app: app['title'])
+        image = Image.open('docStore/img/sdsn.png')
+        st.sidebar.image(image, width =200)
+       
+        with st.sidebar:
+            selected = option_menu(None, [page["title"] for page in self.apps],
+                                   icons=[page["icon"] for page in self.apps],
+                                   menu_icon="cast", default_index=0)
+            st.markdown("---")          
+        
+        
+        for index, item in enumerate(self.apps):
+            if item["title"] == selected:
+                self.apps[index]["function"]()
+                break
+                
+   
+        choice = st.sidebar.radio(label = 'Select the Document',
+                            help = 'You can upload the document \
+                            or else you can try a example document', 
+                            options = ('Upload Document', 'Try Example'), 
+                            horizontal = True)
+        add_upload(choice)
+       

appStore/sdg_analysis.py

@@ -0,0 +1,179 @@
+# set path
+import glob, os, sys; 
+sys.path.append('../utils')
+
+#import needed libraries
+import seaborn as sns
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import streamlit as st
+from st_aggrid import AgGrid
+from st_aggrid.shared import ColumnsAutoSizeMode
+from utils.sdg_classifier import sdg_classification
+from utils.sdg_classifier import runSDGPreprocessingPipeline, load_sdgClassifier
+from utils.keyword_extraction import textrank
+import logging
+logger = logging.getLogger(__name__)
+from utils.checkconfig import getconfig
+
+
+# Declare all the necessary variables
+config = getconfig('paramconfig.cfg')
+model_name = config.get('sdg','MODEL')
+split_by = config.get('sdg','SPLIT_BY')
+split_length = int(config.get('sdg','SPLIT_LENGTH'))
+split_overlap = int(config.get('sdg','SPLIT_OVERLAP'))
+remove_punc = bool(int(config.get('sdg','REMOVE_PUNC')))
+split_respect_sentence_boundary = bool(int(config.get('sdg','RESPECT_SENTENCE_BOUNDARY')))
+threshold = float(config.get('sdg','THRESHOLD'))
+top_n = int(config.get('sdg','TOP_KEY'))
+
+
+def app():
+
+    #### APP INFO #####
+    with st.container():
+        st.markdown("<h1 style='text-align: center; color: black;'> SDG Classification and Keyphrase Extraction </h1>", unsafe_allow_html=True)
+        st.write(' ')
+        st.write(' ')
+
+    with st.expander("ℹ️ - About this app", expanded=False):
+
+        st.write(
+            """     
+            The *SDG Analysis* app is an easy-to-use interface built \
+                in Streamlit for analyzing policy documents with respect to SDG \
+                 Classification for the paragraphs/texts in the document and \
+                extracting the keyphrase per SDG label - developed by GIZ Data \
+                 and the Sustainable Development Solution Network. \n
+            """)
+        st.write("""**Document Processing:** The Uploaded/Selected document is \
+            automatically cleaned and split into paragraphs with a maximum \
+            length of 120 words using a Haystack preprocessing pipeline. The \
+            length of 120 is an empirical value which should reflect the length \
+            of a “context” and should limit the paragraph length deviation. \
+            However, since we want to respect the sentence boundary the limit \
+            can breach and hence this limit of 120 is tentative.  \n
+            """)
+        st.write("""**SDG cLassification:** The application assigns paragraphs \
+            to 16 of the 17 United Nations Sustainable Development Goals (SDGs).\
+            SDG 17 “Partnerships for the Goals” is excluded from the analysis due \
+            to its broad nature which could potentially inflate the results. \
+            Each paragraph is assigned to one SDG only. Again, the results are \
+            displayed in a summary table including the number of the SDG, a \
+            relevancy score highlighted through a green color shading, and the \
+            respective text of the analyzed paragraph. Additionally, a pie \
+            chart with a blue color shading is displayed which illustrates the \
+            three most prominent SDGs in the document. The SDG classification \
+            uses open-source training [data](https://zenodo.org/record/5550238#.Y25ICHbMJPY) \
+            from [OSDG.ai](https://osdg.ai/) which is a global \
+            partnerships and growing community of researchers and institutions \
+            interested in the classification of research according to the \
+            Sustainable Development Goals. The summary table only displays \
+            paragraphs with a calculated relevancy score above 85%.  \n""")
+
+        st.write("""**Keyphrase Extraction:** The application extracts 15 \
+            keyphrases from the document, for each SDG label and displays the \
+            results in a summary table. The keyphrases are extracted using \
+            using [Textrank](https://github.com/summanlp/textrank)\
+            which is an easy-to-use computational less expensive \
+            model leveraging combination of TFIDF and Graph networks.
+            """)
+        st.write("")
+        st.write("")
+        st.markdown("Some runtime metrics tested with cpu: Intel(R) Xeon(R) CPU @ 2.20GHz, memory: 13GB")
+        col1,col2,col3,col4 = st.columns([2,2,4,4])
+        with col1:
+            st.caption("Loading Time Classifier")
+            # st.markdown('<div style="text-align: center;">12 sec</div>', unsafe_allow_html=True)
+            st.write("12 sec")
+        with col2:
+            st.caption("OCR File processing")
+            # st.markdown('<div style="text-align: center;">50 sec</div>', unsafe_allow_html=True)
+            st.write("50 sec")
+        with col3:
+            st.caption("SDG Classification of 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">120 sec</div>', unsafe_allow_html=True)
+            st.write("120 sec")
+        with col4:
+            st.caption("Keyword extraction for 200 paragraphs(~ 35 pages)")
+            # st.markdown('<div style="text-align: center;">3 sec</div>', unsafe_allow_html=True)
+            st.write("3 sec")
+
+        
+
+    
+    ### Main app code ###
+    with st.container():
+        if st.button("RUN SDG Analysis"):
+                   
+            if 'filepath' in st.session_state:
+                file_name = st.session_state['filename']
+                file_path = st.session_state['filepath']
+                classifier = load_sdgClassifier(classifier_name=model_name)
+                st.session_state['sdg_classifier'] = classifier
+                all_documents = runSDGPreprocessingPipeline(file_name= file_name,
+                                        file_path= file_path, split_by= split_by,
+                                        split_length= split_length,
+                split_respect_sentence_boundary= split_respect_sentence_boundary,
+                split_overlap= split_overlap, remove_punc= remove_punc)
+
+                if len(all_documents['documents']) > 100:
+                    warning_msg = ": This might take sometime, please sit back and relax."
+                else:
+                    warning_msg = ""
+
+                with st.spinner("Running SDG Classification{}".format(warning_msg)):
+
+                    df, x = sdg_classification(haystack_doc=all_documents['documents'],
+                                                threshold= threshold)
+                    df = df.drop(['Relevancy'], axis = 1)
+                    sdg_labels = x.SDG.unique()
+                    textrank_keyword_list = []
+                    for label in sdg_labels:
+                        sdgdata = " ".join(df[df.SDG == label].text.to_list())
+                        textranklist_ = textrank(textdata=sdgdata, words= top_n)
+                        if len(textranklist_) > 0:
+                            textrank_keyword_list.append({'SDG':label, 'TextRank Keywords':",".join(textranklist_)})
+                    textrank_keywords_df = pd.DataFrame(textrank_keyword_list)
+
+
+                    plt.rcParams['font.size'] = 25
+                    colors = plt.get_cmap('Blues')(np.linspace(0.2, 0.7, len(x)))
+                    # plot
+                    fig, ax = plt.subplots()
+                    ax.pie(x['count'], colors=colors, radius=2, center=(4, 4),
+                        wedgeprops={"linewidth": 1, "edgecolor": "white"},
+                        textprops={'fontsize': 14}, 
+                        frame=False,labels =list(x.SDG_Num),
+                        labeldistance=1.2)
+                    # fig.savefig('temp.png', bbox_inches='tight',dpi= 100)
+                    
+
+                    st.markdown("#### Anything related to SDGs? ####")
+
+                    c4, c5, c6 = st.columns([1,2,2])
+
+                    with c5:
+                        st.pyplot(fig)
+                    with c6:
+                        labeldf = x['SDG_name'].values.tolist()
+                        labeldf = "<br>".join(labeldf)
+                        st.markdown(labeldf, unsafe_allow_html=True)
+                    st.write("")
+                    st.markdown("###### What keywords are present under SDG classified text? ######")
+
+                    AgGrid(textrank_keywords_df, reload_data = False, 
+                            update_mode="value_changed",
+                    columns_auto_size_mode = ColumnsAutoSizeMode.FIT_CONTENTS)
+                    st.write("")
+                    st.markdown("###### Top few SDG Classified paragraph/text results ######")
+
+                    AgGrid(df, reload_data = False, update_mode="value_changed",
+                    columns_auto_size_mode = ColumnsAutoSizeMode.FIT_CONTENTS)
+            else:
+                st.info("🤔 No document found, please try to upload it at the sidebar!")
+                logging.warning("Terminated as no document provided")
+
+