Upload app.py

app.py

@@ -194,6 +194,23 @@ class Paper:
         text_list = []
         section_dict = {}
 
+        # # 先处理Abstract章节
+        # for page_index, page in enumerate(self.pdf):
+        #     cur_text = page.get_text()
+        #     # 如果该页面是Abstract章节所在页面
+        #     if page_index == list(self.section_page_dict.values())[0]:
+        #         abs_str = "Abstract"
+        #         # 获取Abstract章节的起始位置
+        #         first_index = cur_text.find(abs_str)
+        #         # 查找下一个章节的关键词，这里是Introduction
+        #         intro_str = "Introduction"
+        #         if intro_str in cur_text:
+        #             second_index = cur_text.find(intro_str)
+        #         elif intro_str.upper() in cur_text:
+        #             second_index = cur_text.find(intro_str.upper())
+        #         # 将Abstract章节内容加入字典中
+        #         section_dict[abs_str] = cur_text[first_index+len(abs_str)+1:second_index].replace('-\n',
+        #                                                                                         '').replace('\n', ' ').split('I.')[0].split("II.")[0]
 
         # 再处理其他章节：
         text_list = [page.get_text() for page in self.pdf]
@@ -260,27 +277,198 @@ class Reader:
         self.filter_keys = filter_keys # 用于在摘要中筛选的关键词
         self.root_path = root_path
         self.file_format = 'md' # or 'txt'，如果为图片，则必须为'md'
-
-
+        self.save_image = False
+        if self.save_image:
+            self.gitee_key = self.config.get('Gitee', 'api')
+        else:
+            self.gitee_key = ''
+                
+    def get_arxiv(self, max_results=30):
+        search = arxiv.Search(query=self.query,
+                              max_results=max_results,                              
+                              sort_by=self.sort,
+                              sort_order=arxiv.SortOrder.Descending,
+                              )       
+        return search
+     
+    def filter_arxiv(self, max_results=30):
+        search = self.get_arxiv(max_results=max_results)
+        print("all search:")
+        for index, result in enumerate(search.results()):
+            print(index, result.title, result.updated)
+            
+        filter_results = []   
+        filter_keys = self.filter_keys
+        
+        print("filter_keys:", self.filter_keys)
+        # 确保每个关键词都能在摘要中找到，才算是目标论文
+        for index, result in enumerate(search.results()):
+            abs_text = result.summary.replace('-\n', '-').replace('\n', ' ')
+            meet_num = 0
+            for f_key in filter_keys.split(" "):
+                if f_key.lower() in abs_text.lower():
+                    meet_num += 1
+            if meet_num == len(filter_keys.split(" ")):
+                filter_results.append(result)
+                # break
+        print("filter_results:", len(filter_results))
+        print("filter_papers:")
+        for index, result in enumerate(filter_results):
+            print(index, result.title, result.updated)
+        return filter_results
+    
     def validateTitle(self, title):
         # 将论文的乱七八糟的路径格式修正
         rstr = r"[\/\\\:\*\?\"\<\>\|]" # '/ \ : * ? " < > |'
         new_title = re.sub(rstr, "_", title) # 替换为下划线
         return new_title
 
+    def download_pdf(self, filter_results):
+        # 先创建文件夹
+        date_str = str(datetime.datetime.now())[:13].replace(' ', '-')        
+        key_word = str(self.key_word.replace(':', ' '))        
+        path = self.root_path  + 'pdf_files/' + self.query.replace('au: ', '').replace('title: ', '').replace('ti: ', '').replace(':', ' ')[:25] + '-' + date_str
+        try:
+            os.makedirs(path)
+        except:
+            pass
+        print("All_paper:", len(filter_results))
+        # 开始下载：
+        paper_list = []
+        for r_index, result in enumerate(filter_results):
+            try:
+                title_str = self.validateTitle(result.title)
+                pdf_name = title_str+'.pdf'
+                # result.download_pdf(path, filename=pdf_name)
+                self.try_download_pdf(result, path, pdf_name)
+                paper_path = os.path.join(path, pdf_name)
+                print("paper_path:", paper_path)
+                paper = Paper(path=paper_path,
+                              url=result.entry_id,
+                              title=result.title,
+                              abs=result.summary.replace('-\n', '-').replace('\n', ' '),
+                              authers=[str(aut) for aut in result.authors],
+                              )
+                # 下载完毕，开始解析：
+                paper.parse_pdf()
+                paper_list.append(paper)
+            except Exception as e:
+                print("download_error:", e)
+                pass
+        return paper_list
+    
+    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4, max=10),
+                    stop=tenacity.stop_after_attempt(5),
+                    reraise=True)
+    def try_download_pdf(self, result, path, pdf_name):
+        result.download_pdf(path, filename=pdf_name)
+    
+    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4, max=10),
+                    stop=tenacity.stop_after_attempt(5),
+                    reraise=True)
+    def upload_gitee(self, image_path, image_name='', ext='png'):
+        """
+        上传到码云
+        :return:
+        """ 
+        with open(image_path, 'rb') as f:
+            base64_data = base64.b64encode(f.read())
+            base64_content = base64_data.decode()
+        
+        date_str = str(datetime.datetime.now())[:19].replace(':', '-').replace(' ', '-') + '.' + ext
+        path = image_name+ '-' +date_str
+        
+        payload = {
+            "access_token": self.gitee_key,
+            "owner": self.config.get('Gitee', 'owner'),
+            "repo": self.config.get('Gitee', 'repo'),
+            "path": self.config.get('Gitee', 'path'),
+            "content": base64_content,
+            "message": "upload image"
+        }
+        # 这里需要修改成你的gitee的账户和仓库名，以及文件夹的名字：
+        url = f'https://gitee.com/api/v5/repos/'+self.config.get('Gitee', 'owner')+'/'+self.config.get('Gitee', 'repo')+'/contents/'+self.config.get('Gitee', 'path')+'/'+path
+        rep = requests.post(url, json=payload).json()
+        print("rep:", rep)
+        if 'content' in rep.keys():
+            image_url = rep['content']['download_url']
+        else:
+            image_url = r"https://gitee.com/api/v5/repos/"+self.config.get('Gitee', 'owner')+'/'+self.config.get('Gitee', 'repo')+'/contents/'+self.config.get('Gitee', 'path')+'/' + path
+            
+        return image_url
     
     def summary_with_chat(self, paper_list, key):
         htmls = []
         for paper_index, paper in enumerate(paper_list):
             # 第一步先用title，abs，和introduction进行总结。
-
+            text = ''
+            text += 'Title:' + paper.title
+            text += 'Url:' + paper.url
+            text += 'Abstrat:' + paper.abs
+            # intro
+            text += list(paper.section_text_dict.values())[0]
+            max_token = 2500 * 4
+            text = text[:max_token]
             chat_summary_text = self.chat_summary(text=text, key=str(key))           
             htmls.append(chat_summary_text)
-
-            chat_review_text = self.chat_review(text=text, key=str(key))
-            htmls.append(chat_review_text)
+            
+            # TODO 往md文档中插入论文里的像素最大的一张图片，这个方案可以弄的更加智能一些：
+            first_image, ext = paper.get_image_path()
+            if first_image is None or self.gitee_key == '':
+                pass
+            else:                
+                image_title = self.validateTitle(paper.title)
+                image_url = self.upload_gitee(image_path=first_image, image_name=image_title, ext=ext)
+                htmls.append("\n")
+                htmls.append("![Fig]("+image_url+")")
+                htmls.append("\n")
+            # 第二步总结方法：
+            # TODO，由于有些文章的方法章节名是算法名，所以简单的通过关键词来筛选，很难获取，后面需要用其他的方案去优化。
+            method_key = ''
+            for parse_key in paper.section_text_dict.keys():
+                if 'method' in parse_key.lower() or 'approach' in parse_key.lower():
+                    method_key = parse_key
+                    break
+                
+            if method_key != '':
+                text = ''
+                method_text = ''
+                summary_text = ''
+                summary_text += "<summary>" + chat_summary_text
+                # methods                
+                method_text += paper.section_text_dict[method_key]   
+                # TODO 把这个变成tenacity的自动判别！             
+                max_token = 2500 * 4
+                text = summary_text + "\n <Methods>:\n" + method_text 
+                text = text[:max_token]
+                chat_method_text = self.chat_method(text=text, key=str(key))
+                htmls.append(chat_method_text)
+            else:
+                chat_method_text = ''
+            htmls.append("\n")
+            
+            # 第三步总结全文，并打分：
+            conclusion_key = ''
+            for parse_key in paper.section_text_dict.keys():
+                if 'conclu' in parse_key.lower():
+                    conclusion_key = parse_key
+                    break
+            
+            text = ''
+            conclusion_text = ''
+            summary_text = ''
+            summary_text += "<summary>" + chat_summary_text + "\n <Method summary>:\n" + chat_method_text            
+            if conclusion_key != '':
+                # conclusion                
+                conclusion_text += paper.section_text_dict[conclusion_key]                
+                max_token = 2500 * 4
+                text = summary_text + "\n <Conclusion>:\n" + conclusion_text 
+            else:
+                text = summary_text
+            text = text[:max_token]
+            chat_conclusion_text = self.chat_conclusion(text=text, key=str(key))
+            htmls.append(chat_conclusion_text)
             htmls.append("\n")
-
             md_text = "\n".join(htmls)
             
             return markdown.markdown(md_text)
@@ -289,31 +477,25 @@ class Reader:
     @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4, max=10),
                     stop=tenacity.stop_after_attempt(5),
                     reraise=True)
-    def chat_conclusion(self, text):
-        openai.api_key = self.chat_api_list[self.cur_api]
-        self.cur_api += 1
-        self.cur_api = 0 if self.cur_api >= len(self.chat_api_list) - 1 else self.cur_api
+    def chat_conclusion(self, text, key):
+        openai.api_key = key
         response = openai.ChatCompletion.create(
             model="gpt-3.5-turbo",
             # prompt需要用英语替换，少占用token。
             messages=[
-                {"role": "system",
-                 "content": "You are a reviewer in the field of [" + self.key_word + "] and you need to critically review this article"},
-                # chatgpt 角色
-                {"role": "assistant",
-                 "content": "This is the <summary> and <conclusion> part of an English literature, where <summary> you have already summarized, but <conclusion> part, I need your help to summarize the following questions:" + text},
-                # 背景知识，可以参考OpenReview的审稿流程
+                {"role": "system", "content": "你是一个["+self.key_word+"]领域的审稿人，你需要严格评审这篇文章"},  # chatgpt 角色
+                {"role": "assistant", "content": "这是一篇英文文献的<summary>和<conclusion>部分内容，其中<summary>你已经总结好了，但是<conclusion>部分，我需要你帮忙归纳下面问题："+text},  # 背景知识，可以参考OpenReview的审稿流程
                 {"role": "user", "content": """                 
-                 8. Make the following summary.Be sure to use Chinese answers (proper nouns need to be marked in English).
-                    - (1):What is the significance of this piece of work?
-                    - (2):Summarize the strengths and weaknesses of this article in three dimensions: innovation point, performance, and workload.                   
+                 8. 做出如下总结：
+                    - (1):这篇工作的意义如何？
+                    - (2):从创新点、性能、工作量这三个维度，总结这篇文章的优点和缺点。                   
                     .......
-                 Follow the format of the output later: 
-                 8. Conclusion: \n\n
-                    - (1):xxx;\n                     
-                    - (2):Innovation point: xxx; Performance: xxx; Workload: xxx;\n                      
-
-                 Be sure to use Chinese answers (proper nouns need to be marked in English), statements as concise and academic as possible, do not repeat the content of the previous <summary>, the value of the use of the original numbers, be sure to strictly follow the format, the corresponding content output to xxx, in accordance with \n line feed, ....... means fill in according to the actual requirements, if not, you can not write.                 
+                 按照后面的格式输出: 
+                 8. Conclusion:
+                    - (1):xxx;                     
+                    - (2):创新点: xxx; 性能: xxx; 工作量: xxx;                      
+                 
+                 务必使用中文回答（专有名词需要用英文标注)，语句尽量简洁且学术，不要和之前的<summary>内容重复，数值使用原文数字, 务必严格按照格式，将对应内容输出到xxx中，.......代表按照实际需求填写，如果没有可以不用写.                 
                  """},
             ]
         )
@@ -321,38 +503,32 @@ class Reader:
         for choice in response.choices:
             result += choice.message.content
         print("conclusion_result:\n", result)
-        return result
-
+        return result            
+    
     @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4, max=10),
                     stop=tenacity.stop_after_attempt(5),
                     reraise=True)
-    def chat_method(self, text):
-        openai.api_key = self.chat_api_list[self.cur_api]
-        self.cur_api += 1
-        self.cur_api = 0 if self.cur_api >= len(self.chat_api_list) - 1 else self.cur_api
+    def chat_method(self, text, key):
+        openai.api_key = key
         response = openai.ChatCompletion.create(
             model="gpt-3.5-turbo",
             messages=[
-                {"role": "system",
-                 "content": "You are a researcher in the field of [" + self.key_word + "] who is good at summarizing papers using concise statements"},
-                # chatgpt 角色
-                {"role": "assistant",
-                 "content": "This is the <summary> and <Method> part of an English document, where <summary> you have summarized, but the <Methods> part, I need your help to read and summarize the following questions." + text},
-                # 背景知识
+                {"role": "system", "content": "你是一个["+self.key_word+"]领域的科研人员，善于使用精炼的语句总结论文"},  # chatgpt 角色
+                {"role": "assistant", "content": "这是一篇英文文献的<summary>和<Method>部分内容，其中<summary>你已经总结好了，但是<Methods>部分，我需要你帮忙阅读并归纳下面问题："+text},  # 背景知识
                 {"role": "user", "content": """                 
-                 7. Describe in detail the methodological idea of this article. Be sure to use Chinese answers (proper nouns need to be marked in English). For example, its steps are.
+                 7. 详细描述这篇文章的方法思路。比如说它的步骤是：
                     - (1):...
                     - (2):...
                     - (3):...
                     - .......
-                 Follow the format of the output that follows: 
-                 7. Methods: \n\n
-                    - (1):xxx;\n 
-                    - (2):xxx;\n 
-                    - (3):xxx;\n  
-                    ....... \n\n     
-
-                 Be sure to use Chinese answers (proper nouns need to be marked in English), statements as concise and academic as possible, do not repeat the content of the previous <summary>, the value of the use of the original numbers, be sure to strictly follow the format, the corresponding content output to xxx, in accordance with \n line feed, ....... means fill in according to the actual requirements, if not, you can not write.                 
+                 按照后面的格式输出: 
+                 7. Methods:
+                    - (1):xxx; 
+                    - (2):xxx; 
+                    - (3):xxx;  
+                    .......     
+                 
+                 务必使用中文回答（专有名词需要用英文标注)，语句尽量简洁且学术，不要和之前的<summary>内容重复，数值使用原文数字, 务必严格按照格式，将对应内容输出到xxx中，按照\n换行，.......代表按照实际需求填写，如果没有可以不用写.                 
                  """},
             ]
         )
@@ -365,44 +541,37 @@ class Reader:
     @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4, max=10),
                     stop=tenacity.stop_after_attempt(5),
                     reraise=True)
-    def chat_summary(self, text):
-        openai.api_key = self.chat_api_list[self.cur_api]
-        self.cur_api += 1
-        self.cur_api = 0 if self.cur_api >= len(self.chat_api_list) - 1 else self.cur_api
-
+    def chat_summary(self, text, key):
+        openai.api_key = key
         response = openai.ChatCompletion.create(
             model="gpt-3.5-turbo",
             messages=[
-                {"role": "system",
-                 "content": "You are a researcher in the field of [" + self.key_word + "] who is good at summarizing papers using concise statements"},
-                # chatgpt 角色
-                {"role": "assistant",
-                 "content": "This is the title, author, link, abstract and introduction of an English document. I need your help to read and summarize the following questions: " + text},
-                # 背景知识
+                {"role": "system", "content": "你是一个["+self.key_word+"]领域的科研人员，善于使用精炼的语句总结论文"},  # chatgpt 角色
+                {"role": "assistant", "content": "这是一篇英文文献的标题，作者，链接，Abstract和Introduction部分内容，我需要你帮忙阅读并归纳下面问题："+text},  # 背景知识
                 {"role": "user", "content": """                 
-                 1. Mark the title of the paper (with Chinese translation)
-                 2. list all the authors' names (use English)
-                 3. mark the first author's affiliation (output Chinese translation only)                 
-                 4. mark the keywords of this article (use English)
-                 5. link to the paper, Github code link (if available, fill in Github:None if not)
-                 6. summarize according to the following four points.Be sure to use Chinese answers (proper nouns need to be marked in English)
-                    - (1):What is the research background of this article?
-                    - (2):What are the past methods? What are the problems with them? Is the approach well motivated?
-                    - (3):What is the research methodology proposed in this paper?
-                    - (4):On what task and what performance is achieved by the methods in this paper? Can the performance support their goals?
-                 Follow the format of the output that follows:                  
-                 1. Title: xxx\n\n
-                 2. Authors: xxx\n\n
-                 3. Affiliation: xxx\n\n                 
-                 4. Keywords: xxx\n\n   
-                 5. Urls: xxx or xxx , xxx \n\n      
-                 6. Summary: \n\n
-                    - (1):xxx;\n 
-                    - (2):xxx;\n 
-                    - (3):xxx;\n  
-                    - (4):xxx.\n\n     
-
-                 Be sure to use Chinese answers (proper nouns need to be marked in English), statements as concise and academic as possible, do not have too much repetitive information, numerical values using the original numbers, be sure to strictly follow the format, the corresponding content output to xxx, in accordance with \n line feed.                 
+                 1. 标记出这篇文献的标题(加上中文翻译)
+                 2. 列举所有的作者姓名 (使用英文)
+                 3. 标记第一作者的单位（只输出中文翻译）                 
+                 4. 标记出这篇文章的关键词(使用英文)
+                 5. 论文链接，Github代码链接（如果有的话，没有的话请填写Github:None）
+                 6. 按照下面四个点进行总结：
+                    - (1):这篇文章的研究背景是什么？
+                    - (2):过去的方法有哪些？它们存在什么问题？本文和过去的研究有哪些本质的区别？Is the approach well motivated?
+                    - (3):本文提出的研究方法是什么？
+                    - (4):本文方法在什么任务上，取得了什么性能？性能能否支持他们的目标？
+                 按照后面的格式输出:                  
+                 1. Title: xxx
+                 2. Authors: xxx
+                 3. Affiliation: xxx                
+                 4. Keywords: xxx   
+                 5. Urls: xxx or xxx , xxx      
+                 6. Summary:
+                    - (1):xxx;
+                    - (2):xxx;
+                    - (3):xxx; 
+                    - (4):xxx.    
+                 
+                 务必使用中文回答（专有名词需要用英文标注)，语句尽量简洁且学术，不要有太多重复的信息，数值使用原文数字, 务必严格按照格式，将对应内容输出到xxx中，按照\n换行.                 
                  """},
             ]
         )
@@ -410,69 +579,8 @@ class Reader:
         for choice in response.choices:
             result += choice.message.content
         print("summary_result:\n", result)
-        return result
-    @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4, max=10),
-                    stop=tenacity.stop_after_attempt(5),
-                    reraise=True)
-    def chat_review(self, text):
-        openai.api_key = self.chat_api_list[self.cur_api]
-        self.cur_api += 1
-        self.cur_api = 0 if self.cur_api >= len(self.chat_api_list) - 1 else self.cur_api
-
-        response = openai.ChatCompletion.create(
-            model="gpt-3.5-turbo",
-            messages=[
-                {"role": "system",
-                 "content": "You are a researcher in the field of [" + self.key_word + "] who is good at reviewing papers using concise statements"},
-                # chatgpt 角色
-                {"role": "assistant",
-                 "content": "This is the title, author, link, abstract, introduction, method, experiments, and conclusions of an English document. I need your help to read and summarize the following questions: " + text},
-                # 背景知识
-                {"role": "user", "content": """                 
-                 1. Mark the title of the paper (use English)
-                 2. list all the authors' names (use English)
-                 3. mark the first author's affiliation (use English)                 
-                 4. mark the keywords of this article (use English)
-                 5. link to the paper, Github code link (if available, fill in Github:None if not)
-                 6. summarize according to the following four points.Be sure to use Chinese answers (proper nouns need to be marked in English)
-                    - (1):What is the research background of this article?
-                    - (2):What are the past methods? What are the problems with them? Is the approach well motivated?
-                    - (3):What is the research methodology proposed in this paper?
-                    - (4):On what task and what performance is achieved by the methods in this paper? Can the performance support their goals?
-                 Follow the format of the output that follows:                  
-                 1. Title: xxx\n\n    
-                 2. Summary: \n\n
-                    - (1):xxx;\n 
-                    - (2):xxx;\n 
-                    - (3):xxx;\n  
-                    - (4):xxx.\n\n     
-                 3. Strength: \n\n
-                    - (1):xxx;\n 
-                    - (2):xxx;\n 
-                    - (3):xxx;\n  
-                    - (4):xxx.\n\n
-                 4. Weakness: \n\n
-                    - (1):xxx;\n 
-                    - (2):xxx;\n 
-                    - (3):xxx;\n  
-                    - (4):xxx.\n\n
-                 5. Other questions: \n\n
-                    - (1):grammars;\n 
-                    - (2):figures;\n 
-                    - (3):tables;\n  
-                    - (4):other detailed questions.\n\n      
-                 Be sure to use English answers, statements as concise and academic as possible, do not have too much repetitive information, numerical values using the original numbers, be sure to strictly follow the format, the corresponding content output to xxx, in accordance with \n line feed.                 
-                 """},
-            ]
-        )
-        result = ''
-        for choice in response.choices:
-            result += choice.message.content
-        print("Review_result:\n", result)
-        return result
-
-
-
+        return result        
+            
     def export_to_markdown(self, text, file_name, mode='w'):
         # 使用markdown模块的convert方法，将文本转换为html格式
         # html = markdown.markdown(text)
@@ -487,8 +595,10 @@ class Reader:
         print(f"Query: {self.query}")
         print(f"Sort: {self.sort}")                
 
-def upload_pdf(key,file):
-
+def upload_pdf(key, text, file):
+    # 检查两个输入都不为空
+    if not key or not text or not file:
+        return "两个输入都不能为空，请输入字符并上传 PDF 文件！"
     # 判断PDF文件
     if file and file.name.split(".")[-1].lower() != "pdf":
         return '请勿上传非 PDF 文件！'
@@ -501,18 +611,20 @@ def upload_pdf(key,file):
         return sum_info
 
 # 标题
-title = "ChatPaperPlus"
+title = "ChatPaper"
 # 描述
 description = '''<div align='center'>
 
-Use ChatGPT to summary and Review the papers.
+Use ChatGPT to summary the papers.
 
+Star our Github [ChatPaper](https://github.com/kaixindelele/ChatPaper)
 
 </div>
 '''
 # 创建Gradio界面
 ip = [
     gradio.inputs.Textbox(label="请输入你的API-key(必填)", default=""),
+    gradio.inputs.Textbox(label="请输入论文大标题索引(用英文逗号隔开,必填)", default="'Abstract,Introduction,Related Work,Background,Preliminary,Problem Formulation,Methods,Methodology,Method,Approach,Approaches,Materials and Methods,Experiment Settings,Experiment,Experimental Results,Evaluation,Experiments,Results,Findings,Data Analysis,Discussion,Results and Discussion,Conclusion,References'"),
     gradio.inputs.File(label="请上传论文PDF(必填)")
 ]