Spaces:
Runtime error
Runtime error
Merge branch 'v3.1' of github.com:binary-husky/chatgpt_academic into v3.1
Browse files- README.md +7 -3
- crazy_functions/crazy_utils.py +4 -3
- crazy_functions/批量翻译PDF文档_多线程.py +4 -4
- docs/Dockerfile+ChatGLM +3 -0
- docs/logo.png +0 -0
- main.py +1 -1
- request_llm/bridge_all.py +8 -5
- toolbox.py +1 -1
README.md
CHANGED
@@ -1,4 +1,6 @@
|
|
1 |
-
|
|
|
|
|
2 |
|
3 |
**如果喜欢这个项目,请给它一个Star;如果你发明了更好用的快捷键或函数插件,欢迎发issue或者pull requests**
|
4 |
|
@@ -68,11 +70,13 @@ huggingface免科学上网[在线体验](https://huggingface.co/spaces/qingxu98/
|
|
68 |
<img src="https://user-images.githubusercontent.com/96192199/226935232-6b6a73ce-8900-4aee-93f9-733c7e6fef53.png" width="700" >
|
69 |
</div>
|
70 |
|
71 |
-
- 多种大语言模型混合调用([v3.1分支](https://github.com/binary-husky/chatgpt_academic/tree/v3.1)测试中)
|
72 |
<div align="center">
|
73 |
-
<img src="https://user-images.githubusercontent.com/96192199/
|
74 |
</div>
|
75 |
|
|
|
|
|
76 |
|
77 |
|
78 |
## 直接运行 (Windows, Linux or MacOS)
|
|
|
1 |
+
|
2 |
+
|
3 |
+
# <img src="docs/logo.png" width="40" > ChatGPT 学术优化
|
4 |
|
5 |
**如果喜欢这个项目,请给它一个Star;如果你发明了更好用的快捷键或函数插件,欢迎发issue或者pull requests**
|
6 |
|
|
|
70 |
<img src="https://user-images.githubusercontent.com/96192199/226935232-6b6a73ce-8900-4aee-93f9-733c7e6fef53.png" width="700" >
|
71 |
</div>
|
72 |
|
73 |
+
- 多种大语言模型混合调用(ChatGLM + OpenAI-GPT3.5 + [API2D](https://api2d.com/)-GPT4, [v3.1分支](https://github.com/binary-husky/chatgpt_academic/tree/v3.1)测试中)
|
74 |
<div align="center">
|
75 |
+
<img src="https://user-images.githubusercontent.com/96192199/232537274-deca0563-7aa6-4b5d-94a2-b7c453c47794.png" width="700" >
|
76 |
</div>
|
77 |
|
78 |
+
v3.1的[huggingface测试版](https://huggingface.co/spaces/qingxu98/academic-chatgpt-beta)(huggingface版不支持chatglm)
|
79 |
+
|
80 |
|
81 |
|
82 |
## 直接运行 (Windows, Linux or MacOS)
|
crazy_functions/crazy_utils.py
CHANGED
@@ -448,6 +448,7 @@ def read_and_clean_pdf_text(fp):
|
|
448 |
pf = 998
|
449 |
for l in t['lines']:
|
450 |
txt_line = "".join([wtf['text'] for wtf in l['spans']])
|
|
|
451 |
pf = primary_ffsize(l)
|
452 |
meta_line.append([txt_line, pf, l['bbox'], l])
|
453 |
for wtf in l['spans']: # for l in t['lines']:
|
@@ -558,8 +559,8 @@ def read_and_clean_pdf_text(fp):
|
|
558 |
meta_txt = meta_txt.replace('\n', '\n\n')
|
559 |
|
560 |
############################## <第 5 步,展示分割效果> ##################################
|
561 |
-
for f in finals:
|
562 |
-
|
563 |
-
|
564 |
|
565 |
return meta_txt, page_one_meta
|
|
|
448 |
pf = 998
|
449 |
for l in t['lines']:
|
450 |
txt_line = "".join([wtf['text'] for wtf in l['spans']])
|
451 |
+
if len(txt_line) == 0: continue
|
452 |
pf = primary_ffsize(l)
|
453 |
meta_line.append([txt_line, pf, l['bbox'], l])
|
454 |
for wtf in l['spans']: # for l in t['lines']:
|
|
|
559 |
meta_txt = meta_txt.replace('\n', '\n\n')
|
560 |
|
561 |
############################## <第 5 步,展示分割效果> ##################################
|
562 |
+
# for f in finals:
|
563 |
+
# print亮黄(f)
|
564 |
+
# print亮绿('***************************')
|
565 |
|
566 |
return meta_txt, page_one_meta
|
crazy_functions/批量翻译PDF文档_多线程.py
CHANGED
@@ -13,7 +13,7 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_
|
|
13 |
# 基本信息:功能、贡献者
|
14 |
chatbot.append([
|
15 |
"函数插件功能?",
|
16 |
-
"
|
17 |
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
18 |
|
19 |
# 尝试导入依赖,如果缺少依赖,则给出安装建议
|
@@ -59,7 +59,7 @@ def 批量翻译PDF文档(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_
|
|
59 |
def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, sys_prompt):
|
60 |
import os
|
61 |
import tiktoken
|
62 |
-
TOKEN_LIMIT_PER_FRAGMENT =
|
63 |
generated_conclusion_files = []
|
64 |
for index, fp in enumerate(file_manifest):
|
65 |
|
@@ -91,13 +91,13 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot,
|
|
91 |
# 多线,翻译
|
92 |
gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
93 |
inputs_array=[
|
94 |
-
f"
|
95 |
inputs_show_user_array=[f"\n---\n 原文: \n\n {frag.replace('#', '')} \n---\n 翻译:\n " for frag in paper_fragments],
|
96 |
llm_kwargs=llm_kwargs,
|
97 |
chatbot=chatbot,
|
98 |
history_array=[[paper_meta] for _ in paper_fragments],
|
99 |
sys_prompt_array=[
|
100 |
-
"
|
101 |
# max_workers=5 # OpenAI所允许的最大并行过载
|
102 |
)
|
103 |
|
|
|
13 |
# 基本信息:功能、贡献者
|
14 |
chatbot.append([
|
15 |
"函数插件功能?",
|
16 |
+
"批量翻译PDF文档。函数插件贡献者: Binary-Husky"])
|
17 |
yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
|
18 |
|
19 |
# 尝试导入依赖,如果缺少依赖,则给出安装建议
|
|
|
59 |
def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, sys_prompt):
|
60 |
import os
|
61 |
import tiktoken
|
62 |
+
TOKEN_LIMIT_PER_FRAGMENT = 1280
|
63 |
generated_conclusion_files = []
|
64 |
for index, fp in enumerate(file_manifest):
|
65 |
|
|
|
91 |
# 多线,翻译
|
92 |
gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency(
|
93 |
inputs_array=[
|
94 |
+
f"你需要翻译以下内容:\n{frag}" for frag in paper_fragments],
|
95 |
inputs_show_user_array=[f"\n---\n 原文: \n\n {frag.replace('#', '')} \n---\n 翻译:\n " for frag in paper_fragments],
|
96 |
llm_kwargs=llm_kwargs,
|
97 |
chatbot=chatbot,
|
98 |
history_array=[[paper_meta] for _ in paper_fragments],
|
99 |
sys_prompt_array=[
|
100 |
+
"请你作为一个学术翻译,负责把学术论文准确翻译成中文。注意文章中的每一句话都要翻译。" for _ in paper_fragments],
|
101 |
# max_workers=5 # OpenAI所允许的最大并行过载
|
102 |
)
|
103 |
|
docs/Dockerfile+ChatGLM
CHANGED
@@ -36,6 +36,9 @@ from transformers import AutoModel, AutoTokenizer \n\
|
|
36 |
chatglm_tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) \n\
|
37 |
chatglm_model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).float() ' >> warm_up_chatglm.py
|
38 |
RUN python3 -u warm_up_chatglm.py
|
|
|
|
|
|
|
39 |
RUN $useProxyNetwork git pull
|
40 |
|
41 |
# 为chatgpt-academic配置代理和API-KEY (非必要 可选步骤)
|
|
|
36 |
chatglm_tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) \n\
|
37 |
chatglm_model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).float() ' >> warm_up_chatglm.py
|
38 |
RUN python3 -u warm_up_chatglm.py
|
39 |
+
|
40 |
+
# 禁用缓存,确保更新代码
|
41 |
+
ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache
|
42 |
RUN $useProxyNetwork git pull
|
43 |
|
44 |
# 为chatgpt-academic配置代理和API-KEY (非必要 可选步骤)
|
docs/logo.png
ADDED
main.py
CHANGED
@@ -182,7 +182,7 @@ def main():
|
|
182 |
threading.Thread(target=auto_update, name="self-upgrade", daemon=True).start()
|
183 |
|
184 |
auto_opentab_delay()
|
185 |
-
demo.queue(concurrency_count=CONCURRENT_COUNT).launch(server_name="0.0.0.0", server_port=PORT, auth=AUTHENTICATION)
|
186 |
|
187 |
if __name__ == "__main__":
|
188 |
main()
|
|
|
182 |
threading.Thread(target=auto_update, name="self-upgrade", daemon=True).start()
|
183 |
|
184 |
auto_opentab_delay()
|
185 |
+
demo.queue(concurrency_count=CONCURRENT_COUNT).launch(server_name="0.0.0.0", server_port=PORT, auth=AUTHENTICATION, favicon_path="docs/logo.png")
|
186 |
|
187 |
if __name__ == "__main__":
|
188 |
main()
|
request_llm/bridge_all.py
CHANGED
@@ -23,6 +23,9 @@ from .bridge_tgui import predict as tgui_ui
|
|
23 |
|
24 |
colors = ['#FF00FF', '#00FFFF', '#FF0000', '#990099', '#009999', '#990044']
|
25 |
|
|
|
|
|
|
|
26 |
model_info = {
|
27 |
# openai
|
28 |
"gpt-3.5-turbo": {
|
@@ -31,7 +34,7 @@ model_info = {
|
|
31 |
"endpoint": "https://api.openai.com/v1/chat/completions",
|
32 |
"max_token": 4096,
|
33 |
"tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
|
34 |
-
"token_cnt":
|
35 |
},
|
36 |
|
37 |
"gpt-4": {
|
@@ -40,7 +43,7 @@ model_info = {
|
|
40 |
"endpoint": "https://api.openai.com/v1/chat/completions",
|
41 |
"max_token": 8192,
|
42 |
"tokenizer": tiktoken.encoding_for_model("gpt-4"),
|
43 |
-
"token_cnt":
|
44 |
},
|
45 |
|
46 |
# api_2d
|
@@ -50,7 +53,7 @@ model_info = {
|
|
50 |
"endpoint": "https://openai.api2d.net/v1/chat/completions",
|
51 |
"max_token": 4096,
|
52 |
"tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
|
53 |
-
"token_cnt":
|
54 |
},
|
55 |
|
56 |
"api2d-gpt-4": {
|
@@ -59,7 +62,7 @@ model_info = {
|
|
59 |
"endpoint": "https://openai.api2d.net/v1/chat/completions",
|
60 |
"max_token": 8192,
|
61 |
"tokenizer": tiktoken.encoding_for_model("gpt-4"),
|
62 |
-
"token_cnt":
|
63 |
},
|
64 |
|
65 |
# chatglm
|
@@ -69,7 +72,7 @@ model_info = {
|
|
69 |
"endpoint": None,
|
70 |
"max_token": 1024,
|
71 |
"tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
|
72 |
-
"token_cnt":
|
73 |
},
|
74 |
|
75 |
}
|
|
|
23 |
|
24 |
colors = ['#FF00FF', '#00FFFF', '#FF0000', '#990099', '#009999', '#990044']
|
25 |
|
26 |
+
get_token_num_gpt35 = lambda txt: len(tiktoken.encoding_for_model("gpt-3.5-turbo").encode(txt, disallowed_special=()))
|
27 |
+
get_token_num_gpt4 = lambda txt: len(tiktoken.encoding_for_model("gpt-4").encode(txt, disallowed_special=()))
|
28 |
+
|
29 |
model_info = {
|
30 |
# openai
|
31 |
"gpt-3.5-turbo": {
|
|
|
34 |
"endpoint": "https://api.openai.com/v1/chat/completions",
|
35 |
"max_token": 4096,
|
36 |
"tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
|
37 |
+
"token_cnt": get_token_num_gpt35,
|
38 |
},
|
39 |
|
40 |
"gpt-4": {
|
|
|
43 |
"endpoint": "https://api.openai.com/v1/chat/completions",
|
44 |
"max_token": 8192,
|
45 |
"tokenizer": tiktoken.encoding_for_model("gpt-4"),
|
46 |
+
"token_cnt": get_token_num_gpt4,
|
47 |
},
|
48 |
|
49 |
# api_2d
|
|
|
53 |
"endpoint": "https://openai.api2d.net/v1/chat/completions",
|
54 |
"max_token": 4096,
|
55 |
"tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
|
56 |
+
"token_cnt": get_token_num_gpt35,
|
57 |
},
|
58 |
|
59 |
"api2d-gpt-4": {
|
|
|
62 |
"endpoint": "https://openai.api2d.net/v1/chat/completions",
|
63 |
"max_token": 8192,
|
64 |
"tokenizer": tiktoken.encoding_for_model("gpt-4"),
|
65 |
+
"token_cnt": get_token_num_gpt4,
|
66 |
},
|
67 |
|
68 |
# chatglm
|
|
|
72 |
"endpoint": None,
|
73 |
"max_token": 1024,
|
74 |
"tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
|
75 |
+
"token_cnt": get_token_num_gpt35,
|
76 |
},
|
77 |
|
78 |
}
|
toolbox.py
CHANGED
@@ -401,7 +401,7 @@ def on_file_uploaded(files, chatbot, txt, txt2, checkboxes):
|
|
401 |
chatbot.append(['我上传了文件,请查收',
|
402 |
f'[Local Message] 收到以下文件: \n\n{moved_files_str}' +
|
403 |
f'\n\n调用路径参数已自动修正到: \n\n{txt}' +
|
404 |
-
f'\n\n
|
405 |
return chatbot, txt, txt2
|
406 |
|
407 |
|
|
|
401 |
chatbot.append(['我上传了文件,请查收',
|
402 |
f'[Local Message] 收到以下文件: \n\n{moved_files_str}' +
|
403 |
f'\n\n调用路径参数已自动修正到: \n\n{txt}' +
|
404 |
+
f'\n\n现在您点击任意“红颜色”标识的函数插件时,以上文件将被作为输入参数'+err_msg])
|
405 |
return chatbot, txt, txt2
|
406 |
|
407 |
|