arxiv id list support

app.py

@@ -1,43 +1,71 @@
+
+from turtle import down
 import streamlit as st
 import pandas as pd
 import numpy as np
 
 from src.Surveyor import Surveyor
+from streamlit_tags import st_tags_sidebar
+
+
+@st.experimental_singleton
+def get_surveyor_instance(_print_fn, _survey_print_fn):
+     with st.spinner('Loading The-Surveyor ...'):
+        return Surveyor(_print_fn, _survey_print_fn, refresh_models=True)
 
-def run_survey(surveyor, research_keywords, max_search, num_papers):
+
+def run_survey(surveyor, download_placeholder, research_keywords=None, arxiv_ids=None, max_search=None, num_papers=None):
     zip_file_name, survey_file_name = surveyor.survey(research_keywords, 
+                                                      arxiv_ids,
                                                       max_search=max_search, 
                                                       num_papers=num_papers
                                                      )
+    show_survey_download(zip_file_name, survey_file_name, download_placeholder)
+
 
-    with open(str(zip_file_name), "rb") as file:
-        btn = st.download_button(
-             label="Download extracted topic-clustered-highlights, images and tables as zip",
-             data=file,
-             file_name=str(zip_file_name)
-           )
+def show_survey_download(zip_file_name, survey_file_name, download_placeholder):
+    download_placeholder.empty()
+    with download_placeholder.container():
+        with open(str(zip_file_name), "rb") as file:
+            btn = st.download_button(
+                label="Download extracted topic-clustered-highlights, images and tables as zip",
+                data=file,
+                file_name=str(zip_file_name)
+            )
 
-    with open(str(survey_file_name), "rb") as file:
-        btn = st.download_button(
-             label="Download detailed generated survey file",
-             data=file,
-             file_name=str(survey_file_name)
-           )
-        for line in file.readlines():
-            st.write(line)
+        with open(str(survey_file_name), "rb") as file:
+            btn = st.download_button(
+                label="Download detailed generated survey file",
+                data=file,
+                file_name=str(survey_file_name)
+            )
 
 
-def survey_space(surveyor):
+def survey_space(surveyor, download_placeholder):
+
     form = st.sidebar.form(key='survey_form')
-    research_keywords = form.text_input("What would you like to research in today?")
+    research_keywords = form.text_input("What would you like to research in today?", key='research_keywords')
     max_search = form.number_input("num_papers_to_search", help="maximium number of papers to glance through - defaults to 20", 
-                             min_value=1, max_value=60, value=10, step=1, key='max_search')
+                             min_value=1, max_value=50, value=10, step=1, key='max_search')
     num_papers = form.number_input("num_papers_to_select", help="maximium number of papers to select and analyse - defaults to 8",
-                             min_value=1, max_value=25, value=2, step=1, key='num_papers')
+                             min_value=1, max_value=8, value=2, step=1, key='num_papers')
     submit = form.form_submit_button('Submit')
 
+    st.sidebar.write('or')
+
+    arxiv_ids = st_tags_sidebar(
+                label='# Enter Keywords:',
+                value=[],
+                text='Press enter to add more',
+                maxtags = 6,
+                key='arxiv_ids')
+
     if submit:
-        run_survey(surveyor, research_keywords, max_search, num_papers)
+        run_survey(surveyor, download_placeholder, research_keywords, max_search, num_papers)
+    elif len(arxiv_ids):
+        run_survey(surveyor, download_placeholder, arxiv_ids)
+
+
 
 
 if __name__ == '__main__':
@@ -45,6 +73,7 @@ if __name__ == '__main__':
     std_col, survey_col = st.columns(2)
     std_col.header('execution log:')
     survey_col.header('Generated_survey:')
-    with st.spinner('Loading The-Surveyor ...'):
-        surveyor_obj = Surveyor(print_fn=std_col.write, survey_print_fn=survey_col.write, refresh_models=True)
-    survey_space(surveyor_obj)
+    download_placeholder = survey_col.container()
+    download_placeholder = st.empty()
+    surveyor_obj = get_surveyor_instance(_print_fn=std_col.write, _survey_print_fn=survey_col.write)
+    survey_space(surveyor_obj, survey_col)

arxiv_public_data/config.py

@@ -9,7 +9,7 @@ logging.basicConfig(
 baselog = logging.getLogger('arxivdata')
 logger = baselog.getChild('config')
 
-DEFAULT_PATH = os.path.join(os.path.abspath('../'), 'arxiv-data')
+DEFAULT_PATH = os.path.join(os.path.abspath('.'), 'arxiv-data')
 JSONFILE = './config.json'
 KEY = 'ARXIV_DATA'
 

requirements.txt

@@ -3,6 +3,7 @@ arxiv
 arxiv2bib
 boto3==1.9.118
 bert-extractive-summarizer
+fitz==0.0.1.dev2
 joblib
 keybert
 numpy
@@ -22,6 +23,7 @@ scispacy
 https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_core_sci_scibert-0.5.0.tar.gz
 https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_core_sci_lg-0.5.0.tar.gz
 streamlit
+streamlit-tags
 summarizer
 tabula
 tabula_py

src/Surveyor.py

@@ -1355,16 +1355,18 @@ class Surveyor:
         zipdir(dump_dir, zipf)
         return zip_name
 
-    def survey(self, query, max_search=None, num_papers=None, debug=False, weigh_authors=False):
+    def survey(self, query=None, id_list=None, max_search=None, num_papers=None, debug=False, weigh_authors=False):
         import joblib
         import os, shutil
         if not max_search:
             max_search = self.DEFAULTS['max_search']
         if not num_papers:
             num_papers = self.DEFAULTS['num_papers']
+        if (query is None) and (id_list is None):
+            raise ValueError('please provide a base to survey on: list of arxiv IDs or a few research keywords')
         # arxiv api relevance search and data preparation
         self.print_fn("\n-searching arXiv for top 100 papers.. ")
-        results, searched_papers = self.search(query, max_search=max_search)
+        results, searched_papers = self.search(query, id_list, max_search=max_search)
         joblib.dump(searched_papers, self.dump_dir + 'papers_metadata.dmp')
         self.print_fn("\n-found " + str(len(searched_papers)) + " papers")
 
@@ -1485,70 +1487,3 @@ class Surveyor:
             survey_file) + "\nAll outputs zip path :" + os.path.abspath(self.dump_dir + output_zip))
 
         return os.path.abspath(self.dump_dir + output_zip), os.path.abspath(survey_file)
-
-
-if __name__ == '__main__':
-    import argparse
-
-    parser = argparse.ArgumentParser(description='Generate a survey just from a query !!')
-    parser.add_argument('query', metavar='query_string', type=str,
-                        help='your research query/keywords')
-    parser.add_argument('--max_search', metavar='max_metadata_papers', type=int, default=None,
-                        help='maximium number of papers to gaze at - defaults to 100')
-    parser.add_argument('--num_papers', metavar='max_num_papers', type=int, default=None,
-                        help='maximium number of papers to download and analyse - defaults to 25')
-    parser.add_argument('--pdf_dir', metavar='pdf_dir', type=str, default=None,
-                        help='pdf paper storage directory - defaults to arxiv_data/tarpdfs/')
-    parser.add_argument('--txt_dir', metavar='txt_dir', type=str, default=None,
-                        help='text-converted paper storage directory - defaults to arxiv_data/fulltext/')
-    parser.add_argument('--img_dir', metavar='img_dir', type=str, default=None,
-                        help='image storage directory - defaults to arxiv_data/images/')
-    parser.add_argument('--tab_dir', metavar='tab_dir', type=str, default=None,
-                        help='tables storage directory - defaults to arxiv_data/tables/')
-    parser.add_argument('--dump_dir', metavar='dump_dir', type=str, default=None,
-                        help='all_output_dir - defaults to arxiv_dumps/')
-    parser.add_argument('--models_dir', metavar='save_models_dir', type=str, default=None,
-                        help='directory to save models (> 5GB) - defaults to saved_models/')
-    parser.add_argument('--title_model_name', metavar='title_model_name', type=str, default=None,
-                        help='title model name/tag in hugging-face, defaults to \'Callidior/bert2bert-base-arxiv-titlegen\'')
-    parser.add_argument('--ex_summ_model_name', metavar='extractive_summ_model_name', type=str, default=None,
-                        help='extractive summary model name/tag in hugging-face, defaults to \'allenai/scibert_scivocab_uncased\'')
-    parser.add_argument('--ledmodel_name', metavar='ledmodel_name', type=str, default=None,
-                        help='led model(for abstractive summary) name/tag in hugging-face, defaults to \'allenai/led-large-16384-arxiv\'')
-    parser.add_argument('--embedder_name', metavar='sentence_embedder_name', type=str, default=None,
-                        help='sentence embedder name/tag in hugging-face, defaults to \'paraphrase-MiniLM-L6-v2\'')
-    parser.add_argument('--nlp_name', metavar='spacy_model_name', type=str, default=None,
-                        help='spacy model name/tag in hugging-face (if changed - needs to be spacy-installed prior), defaults to \'en_core_sci_scibert\'')
-    parser.add_argument('--similarity_nlp_name', metavar='similarity_nlp_name', type=str, default=None,
-                        help='spacy downstream model(for similarity) name/tag in hugging-face (if changed - needs to be spacy-installed prior), defaults to \'en_core_sci_lg\'')
-    parser.add_argument('--kw_model_name', metavar='kw_model_name', type=str, default=None,
-                        help='keyword extraction model name/tag in hugging-face, defaults to \'distilbert-base-nli-mean-tokens\'')
-    parser.add_argument('--refresh_models', metavar='refresh_models', type=str, default=None,
-                        help='Refresh model downloads with given names (needs atleast one model name param above), defaults to False')
-    parser.add_argument('--high_gpu', metavar='high_gpu', type=str, default=None,
-                        help='High GPU usage permitted, defaults to False')
-
-    args = parser.parse_args()
-
-    surveyor = Surveyor(
-        pdf_dir=args.pdf_dir,
-        txt_dir=args.txt_dir,
-        img_dir=args.img_dir,
-        tab_dir=args.tab_dir,
-        dump_dir=args.dump_dir,
-        models_dir=args.models_dir,
-        title_model_name=args.title_model_name,
-        ex_summ_model_name=args.ex_summ_model_name,
-        ledmodel_name=args.ledmodel_name,
-        embedder_name=args.embedder_name,
-        nlp_name=args.nlp_name,
-        similarity_nlp_name=args.similarity_nlp_name,
-        kw_model_name=args.kw_model_name,
-        refresh_models=args.refresh_models,
-        high_gpu=args.high_gpu
-
-    )
-
-    surveyor.survey(args.query, max_search=args.max_search, num_papers=args.num_papers,
-                                              debug=False, weigh_authors=False)
-

survey.py

@@ -9,8 +9,11 @@ if __name__ == '__main__':
     import argparse
 
     parser = argparse.ArgumentParser(description='Generate a survey just from a query !!')
-    parser.add_argument('query', metavar='query_string', type=str,
-                        help='your research query/keywords')
+    
+    data = parser.add_mutually_exclusive_group(required=True)
+    data.add_argument('--query', type=str, help='your research query/keywords')
+    data.add_argument('--arxiv_ids', nargs='+', help='arxiv ids for your curated set of papers')
+
     parser.add_argument('--max_search', metavar='max_metadata_papers', type=int, default=None,
                         help='maximium number of papers to gaze at - defaults to 100')
     parser.add_argument('--num_papers', metavar='max_num_papers', type=int, default=None,
@@ -67,6 +70,6 @@ if __name__ == '__main__':
 
     )
 
-    surveyor.survey(args.query, max_search=args.max_search, num_papers=args.num_papers,
+    surveyor.survey(query=args.query, id_list=args.arxiv_ids, max_search=args.max_search, num_papers=args.num_papers,
                                               debug=False, weigh_authors=False)