import urllib
import streamlit as st
import requests
import re
from stqdm import stqdm
import os
import shutil
import time
from bs4 import BeautifulSoup as bs
from datetime import datetime
from urllib.parse import quote
class XRxivQuery:
def __init__(self, search_query, max_results, folder_name='docs', XRxiv_servers = [], search_by='all', sort_by='relevance'):
self.search_query = search_query
self.max_results = max_results
self.folder_name = folder_name
self.XRxiv_servers = XRxiv_servers
self.search_by = search_by
self.sort_by = sort_by
self.all_pdf_info = []
self.all_pdf_citation = []
def call_API(self):
search_query = self.search_query.strip().replace(" ", "+").split('+')#.replace(", ", "+").replace(",", "+")#.split('+')
if 'rxiv' in self.XRxiv_servers:
'''
Scraps the arXiv's html to get data from each entry in a search. Entries has the following formatting:
\n
http://arxiv.org/abs/2008.04584v2\n
2021-05-11T12:00:24Z\n
2020-08-11T08:47:06Z\n
Bayesian Selective Inference: Non-informative Priors\n
We discuss Bayesian inference for parameters selected using the data. First,\nwe provide a critical analysis of the existing positions in the literature\nregarding the correct Bayesian approach under selection. Second, we propose two\ntypes of non-informative priors for selection models. These priors may be\nemployed to produce a posterior distribution in the absence of prior\ninformation as well as to provide well-calibrated frequentist inference for the\nselected parameter. We test the proposed priors empirically in several\nscenarios.\n\n
\n Daniel G. Rasines\n \n \n G. Alastair Young\n \n
24 pages, 7 figures\n
\n
\n
\n
\n
\n
\n
'''
print('Searching Arxiv\n')
# Call arXiv API
journal = 'arXiv'
# print(" ".join(search_query))
# print(self.search_query.strip().replace(", ", "+").replace(" ", "+").replace(",", "+"))
arXiv_url=f'http://export.arxiv.org/api/query?search_query={self.search_by}:{"+".join(search_query)}&sortBy={self.sort_by}&start=0&max_results={self.max_results}'
# print(arXiv_url)
with urllib.request.urlopen(arXiv_url) as url:
s = url.read()
# Parse the xml data
from lxml import html
root = html.fromstring(s)
# Fetch relevant pdf information
pdf_entries = root.xpath("entry")
pdf_titles = []
pdf_authors = []
pdf_urls = []
pdf_categories = []
folder_names = []
pdf_citation = []
pdf_years = []
for i, pdf in enumerate(pdf_entries):
pdf_titles.append(re.sub('[^a-zA-Z0-9]', ' ', pdf.xpath("title/text()")[0]))
pdf_authors.append(pdf.xpath("author/name/text()"))
pdf_urls.append(pdf.xpath("link[@title='pdf']/@href")[0])
pdf_categories.append(pdf.xpath("category/@term"))
folder_names.append(self.folder_name)
pdf_years.append(pdf.xpath('updated/text()')[0][:4])
pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. {journal} [{pdf_categories[i][0]}] ({pdf_years[i]}), (available at {pdf_urls[i]}).")
# self.all_pdf_citation.append(pdf_citation)
pdf_info = list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
self.all_pdf_info.append(pdf_info)
if 'biorxiv' in self.XRxiv_servers or 'medrxiv' in self.XRxiv_servers:
'''
Scraps the arXiv's html to get data from each entry in a search. Entries has the following formatting:
\n
'''
if 'biorxiv' in self.XRxiv_servers and 'medrxiv' not in self.XRxiv_servers:
print('Searching biorxiv\n')
journals_str = f'%20jcode%3Abiorxiv'
if 'biorxiv' not in self.XRxiv_servers and 'medrxiv' in self.XRxiv_servers:
print('Searching medrxiv\n')
journals_str = f'%20jcode%3Amedrxiv'
if 'biorxiv' in self.XRxiv_servers and 'medrxiv' in self.XRxiv_servers:
print('Searching both biorxiv and medrxiv\n')
journals_str = f'%20jcode%3Abiorxiv%7C%7Cmedrxiv'
subject_str = ('%20').join(self.search_query[0].split())
for subject in search_query[1:]:
subject_str = subject_str + '%252B' + ('%20').join(subject.split())
current_dateTime = datetime.now()
today = str(current_dateTime)[:10]
start_day = '2013-01-01'
arXiv_url = f'https://www.biorxiv.org/search/'
arXiv_url += subject_str + journals_str + f'%20limit_from%3A2{start_day}%20limit_to%3A{today}%20numresults%3A{self.max_results}%20sort%3Arelevance-rank%20format_result%3Astandard'
url_response = requests.post(arXiv_url)
html = bs(url_response.text, features='html.parser')
pdf_entries = html.find_all(attrs={'class': 'search-result'})
pdf_titles = []
pdf_authors = []
pdf_urls = []
pdf_categories = []
folder_names = []
pdf_citation = []
pdf_years = []
for i, pdf in enumerate(pdf_entries):
pdf_titles.append(pdf.find('span', attrs={'class': 'highwire-cite-title'}).text.strip())
pdf_authors.append(pdf.find('span', attrs={'class': 'highwire-citation-authors'}).text.strip().split(', '))
pdf_url = pdf.find('a', href=True)['href']
if pdf_url[:4] != 'http':
pdf_url = f'http://www.biorxiv.org'+ pdf_url
pdf_urls.append(pdf_url)
pdf_categories.append(pdf.find('span', attrs={'class': 'highwire-cite-metadata-journal highwire-cite-metadata'}).text.strip())
folder_names.append(self.folder_name)
pdf_years.append(pdf.find('span', attrs={'class': 'highwire-cite-metadata-pages highwire-cite-metadata'}).text.strip()[:4])
pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. {pdf_categories[i]} ({pdf_years[i]}), (available at {pdf_urls[i]}).")
pdf_info = list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
self.all_pdf_info.append(pdf_info)
self.all_pdf_info = [item for sublist in self.all_pdf_info for item in sublist]
print(self.all_pdf_info)
return self.all_pdf_info
def download_pdf(self):
# if len(os.listdir(f'./{folder_name}') ) != 0:
# check folder is empty to avoid using papers from old runs:
# os.remove(f'./{folder_name}/*')
# print(pdf_info)
all_reference_text = []
for i,p in enumerate(stqdm(self.all_pdf_info, desc='Searching and downloading papers')):
pdf_title=p[0]
pdf_category=p[3]
pdf_url=p[1]
if pdf_category in ['medRxiv', 'bioRxiv']:
pdf_url += '.full.pdf'
pdf_file_name=p[0].replace(':','').replace('/','').replace('.','')
folder_name=p[4]
pdf_citation=p[5]
r = requests.get(pdf_url, allow_redirects=True)
if i == 0:
if not os.path.exists(f'{folder_name}'):
os.makedirs(f"{folder_name}")
else:
shutil.rmtree(f'{folder_name}')
os.makedirs(f"{folder_name}")
with open(f'{folder_name}/{pdf_file_name}.pdf', 'wb') as f:
f.write(r.content)
if i == 0:
st.markdown("###### Papers found:")
st.markdown(f"{i+1}. {pdf_citation}")
time.sleep(0.15)
all_reference_text.append(f"{i+1}. {pdf_citation}\n")
if 'all_reference_text' not in st.session_state:
st.session_state.key = 'all_reference_text'
st.session_state['all_reference_text'] = ' '.join(all_reference_text)
def call_arXiv_API(search_query, search_by='all', sort_by='relevance', max_results='10', folder_name='docs'):
'''
Scraps the arXiv's html to get data from each entry in a search. Entries has the following formatting:
\n
http://arxiv.org/abs/2008.04584v2\n
2021-05-11T12:00:24Z\n
2020-08-11T08:47:06Z\n
Bayesian Selective Inference: Non-informative Priors\n
We discuss Bayesian inference for parameters selected using the data. First,\nwe provide a critical analysis of the existing positions in the literature\nregarding the correct Bayesian approach under selection. Second, we propose two\ntypes of non-informative priors for selection models. These priors may be\nemployed to produce a posterior distribution in the absence of prior\ninformation as well as to provide well-calibrated frequentist inference for the\nselected parameter. We test the proposed priors empirically in several\nscenarios.\n\n
\n Daniel G. Rasines\n \n \n G. Alastair Young\n \n
24 pages, 7 figures\n
\n
\n
\n
\n
\n
\n
'''
# Remove space in seach query
search_query=search_query.strip().replace(" ", "+").replace(", ","+").replace(",","+")
# Call arXiv API
arXiv_url=f'http://export.arxiv.org/api/query?search_query={search_by}:{search_query}&sortBy={sort_by}&start=0&max_results={max_results}'
with urllib.request.urlopen(arXiv_url) as url:
s = url.read()
from lxml import html
# Parse the xml data
root = html.fromstring(s)
# Fetch relevant pdf information
pdf_entries = root.xpath("entry")
pdf_titles = []
pdf_authors = []
pdf_urls = []
pdf_categories = []
folder_names = []
pdf_citation = []
pdf_years = []
for i, pdf in enumerate(pdf_entries):
pdf_titles.append(re.sub('[^a-zA-Z0-9]', ' ', pdf.xpath("title/text()")[0]))
pdf_authors.append(pdf.xpath("author/name/text()"))
pdf_urls.append(pdf.xpath("link[@title='pdf']/@href")[0])
pdf_categories.append(pdf.xpath("category/@term"))
folder_names.append(folder_name)
pdf_years.append(pdf.xpath('updated/text()')[0][:4])
pdf_citation.append(f"{', '.join(pdf_authors[i])}, {pdf_titles[i]}. arXiv [{pdf_categories[i][0]}] ({pdf_years[i]}), (available at {pdf_urls[i]}).")
pdf_info=list(zip(pdf_titles, pdf_urls, pdf_authors, pdf_categories, folder_names, pdf_citation))
# Check number of available files
if len(pdf_urls)