3v324v23 commited on
Commit
b0409b9
1 Parent(s): 28aa6d1

tiktoken做lazyload处理

Browse files
check_proxy.py CHANGED
@@ -103,7 +103,7 @@ def auto_update():
103
  import json
104
  proxies, = get_conf('proxies')
105
  response = requests.get(
106
- "https://raw.githubusercontent.com/binary-husky/chatgpt_academic/master/version", proxies=proxies, timeout=1)
107
  remote_json_data = json.loads(response.text)
108
  remote_version = remote_json_data['version']
109
  if remote_json_data["show_feature"]:
@@ -133,6 +133,13 @@ def auto_update():
133
  except:
134
  print('自动更新程序:已禁用')
135
 
 
 
 
 
 
 
 
136
 
137
  if __name__ == '__main__':
138
  import os
 
103
  import json
104
  proxies, = get_conf('proxies')
105
  response = requests.get(
106
+ "https://raw.githubusercontent.com/binary-husky/chatgpt_academic/master/version", proxies=proxies, timeout=5)
107
  remote_json_data = json.loads(response.text)
108
  remote_version = remote_json_data['version']
109
  if remote_json_data["show_feature"]:
 
133
  except:
134
  print('自动更新程序:已禁用')
135
 
136
+ def warm_up_modules():
137
+ print('正在执行一些模块的预热...')
138
+ from request_llm.bridge_all import model_info
139
+ enc = model_info["gpt-3.5-turbo"]['tokenizer']
140
+ enc.encode("模块预热", disallowed_special=())
141
+ enc = model_info["gpt-4"]['tokenizer']
142
+ enc.encode("模块预热", disallowed_special=())
143
 
144
  if __name__ == '__main__':
145
  import os
crazy_functions/Latex全文润色.py CHANGED
@@ -11,9 +11,8 @@ class PaperFileGroup():
11
  self.sp_file_tag = []
12
 
13
  # count_token
14
- import tiktoken
15
- from toolbox import get_conf
16
- enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
17
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
18
  self.get_token_num = get_token_num
19
 
 
11
  self.sp_file_tag = []
12
 
13
  # count_token
14
+ from request_llm.bridge_all import model_info
15
+ enc = model_info["gpt-3.5-turbo"]['tokenizer']
 
16
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
17
  self.get_token_num = get_token_num
18
 
crazy_functions/Latex全文翻译.py CHANGED
@@ -11,9 +11,8 @@ class PaperFileGroup():
11
  self.sp_file_tag = []
12
 
13
  # count_token
14
- import tiktoken
15
- from toolbox import get_conf
16
- enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
17
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
18
  self.get_token_num = get_token_num
19
 
 
11
  self.sp_file_tag = []
12
 
13
  # count_token
14
+ from request_llm.bridge_all import model_info
15
+ enc = model_info["gpt-3.5-turbo"]['tokenizer']
 
16
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
17
  self.get_token_num = get_token_num
18
 
crazy_functions/crazy_utils.py CHANGED
@@ -2,9 +2,9 @@ import traceback
2
  from toolbox import update_ui, get_conf
3
 
4
  def input_clipping(inputs, history, max_token_limit):
5
- import tiktoken
6
  import numpy as np
7
- enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
 
8
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
9
 
10
  mode = 'input-and-history'
 
2
  from toolbox import update_ui, get_conf
3
 
4
  def input_clipping(inputs, history, max_token_limit):
 
5
  import numpy as np
6
+ from request_llm.bridge_all import model_info
7
+ enc = model_info["gpt-3.5-turbo"]['tokenizer']
8
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
9
 
10
  mode = 'input-and-history'
crazy_functions/代码重写为全英文_多线程.py CHANGED
@@ -59,9 +59,8 @@ def 全项目切换英文(txt, llm_kwargs, plugin_kwargs, chatbot, history, sys_
59
 
60
  # 第5步:Token限制下的截断与处理
61
  MAX_TOKEN = 3000
62
- import tiktoken
63
- from toolbox import get_conf
64
- enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
65
  def get_token_fn(txt): return len(enc.encode(txt, disallowed_special=()))
66
 
67
 
 
59
 
60
  # 第5步:Token限制下的截断与处理
61
  MAX_TOKEN = 3000
62
+ from request_llm.bridge_all import model_info
63
+ enc = model_info["gpt-3.5-turbo"]['tokenizer']
 
64
  def get_token_fn(txt): return len(enc.encode(txt, disallowed_special=()))
65
 
66
 
crazy_functions/批量Markdown翻译.py CHANGED
@@ -11,9 +11,8 @@ class PaperFileGroup():
11
  self.sp_file_tag = []
12
 
13
  # count_token
14
- import tiktoken
15
- from toolbox import get_conf
16
- enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
17
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
18
  self.get_token_num = get_token_num
19
 
 
11
  self.sp_file_tag = []
12
 
13
  # count_token
14
+ from request_llm.bridge_all import model_info
15
+ enc = model_info["gpt-3.5-turbo"]['tokenizer']
 
16
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
17
  self.get_token_num = get_token_num
18
 
crazy_functions/批量翻译PDF文档_多线程.py CHANGED
@@ -68,8 +68,8 @@ def 解析PDF(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot,
68
 
69
  # 递归地切割PDF文件
70
  from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
71
- from toolbox import get_conf
72
- enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
73
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
74
  paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf(
75
  txt=file_content, get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT)
 
68
 
69
  # 递归地切割PDF文件
70
  from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
71
+ from request_llm.bridge_all import model_info
72
+ enc = model_info["gpt-3.5-turbo"]['tokenizer']
73
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
74
  paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf(
75
  txt=file_content, get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT)
crazy_functions/理解PDF文档内容.py CHANGED
@@ -17,8 +17,8 @@ def 解析PDF(file_name, llm_kwargs, plugin_kwargs, chatbot, history, system_pro
17
  TOKEN_LIMIT_PER_FRAGMENT = 2500
18
 
19
  from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
20
- from toolbox import get_conf
21
- enc = tiktoken.encoding_for_model("gpt-3.5-turbo")
22
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
23
  paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf(
24
  txt=file_content, get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT)
 
17
  TOKEN_LIMIT_PER_FRAGMENT = 2500
18
 
19
  from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf
20
+ from request_llm.bridge_all import model_info
21
+ enc = model_info["gpt-3.5-turbo"]['tokenizer']
22
  def get_token_num(txt): return len(enc.encode(txt, disallowed_special=()))
23
  paper_fragments = breakdown_txt_to_satisfy_token_limit_for_pdf(
24
  txt=file_content, get_token_fn=get_token_num, limit=TOKEN_LIMIT_PER_FRAGMENT)
main.py CHANGED
@@ -40,7 +40,7 @@ def main():
40
  set_theme = adjust_theme()
41
 
42
  # 代理与自动更新
43
- from check_proxy import check_proxy, auto_update
44
  proxy_info = check_proxy(proxies)
45
 
46
  gr_L1 = lambda: gr.Row().style()
@@ -180,6 +180,7 @@ def main():
180
  webbrowser.open_new_tab(f"http://localhost:{PORT}/?__dark-theme=true")
181
  threading.Thread(target=open, name="open-browser", daemon=True).start()
182
  threading.Thread(target=auto_update, name="self-upgrade", daemon=True).start()
 
183
 
184
  auto_opentab_delay()
185
  demo.queue(concurrency_count=CONCURRENT_COUNT).launch(server_name="0.0.0.0", server_port=PORT, auth=AUTHENTICATION, favicon_path="docs/logo.png")
 
40
  set_theme = adjust_theme()
41
 
42
  # 代理与自动更新
43
+ from check_proxy import check_proxy, auto_update, warm_up_modules
44
  proxy_info = check_proxy(proxies)
45
 
46
  gr_L1 = lambda: gr.Row().style()
 
180
  webbrowser.open_new_tab(f"http://localhost:{PORT}/?__dark-theme=true")
181
  threading.Thread(target=open, name="open-browser", daemon=True).start()
182
  threading.Thread(target=auto_update, name="self-upgrade", daemon=True).start()
183
+ threading.Thread(target=warm_up_modules, name="warm-up", daemon=True).start()
184
 
185
  auto_opentab_delay()
186
  demo.queue(concurrency_count=CONCURRENT_COUNT).launch(server_name="0.0.0.0", server_port=PORT, auth=AUTHENTICATION, favicon_path="docs/logo.png")
request_llm/bridge_all.py CHANGED
@@ -9,7 +9,7 @@
9
  2. predict_no_ui_long_connection:在实验过程中发现调用predict_no_ui处理长文档时,和openai的连接容易断掉,这个函数用stream的方式解决这个问题,同样支持多线程
10
  """
11
  import tiktoken
12
-
13
  from concurrent.futures import ThreadPoolExecutor
14
 
15
  from .bridge_chatgpt import predict_no_ui_long_connection as chatgpt_noui
@@ -18,13 +18,31 @@ from .bridge_chatgpt import predict as chatgpt_ui
18
  from .bridge_chatglm import predict_no_ui_long_connection as chatglm_noui
19
  from .bridge_chatglm import predict as chatglm_ui
20
 
21
- from .bridge_tgui import predict_no_ui_long_connection as tgui_noui
22
- from .bridge_tgui import predict as tgui_ui
23
 
24
  colors = ['#FF00FF', '#00FFFF', '#FF0000', '#990099', '#009999', '#990044']
25
 
26
- get_token_num_gpt35 = lambda txt: len(tiktoken.encoding_for_model("gpt-3.5-turbo").encode(txt, disallowed_special=()))
27
- get_token_num_gpt4 = lambda txt: len(tiktoken.encoding_for_model("gpt-4").encode(txt, disallowed_special=()))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
 
29
  model_info = {
30
  # openai
@@ -33,7 +51,7 @@ model_info = {
33
  "fn_without_ui": chatgpt_noui,
34
  "endpoint": "https://api.openai.com/v1/chat/completions",
35
  "max_token": 4096,
36
- "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
37
  "token_cnt": get_token_num_gpt35,
38
  },
39
 
@@ -42,7 +60,7 @@ model_info = {
42
  "fn_without_ui": chatgpt_noui,
43
  "endpoint": "https://api.openai.com/v1/chat/completions",
44
  "max_token": 8192,
45
- "tokenizer": tiktoken.encoding_for_model("gpt-4"),
46
  "token_cnt": get_token_num_gpt4,
47
  },
48
 
@@ -52,7 +70,7 @@ model_info = {
52
  "fn_without_ui": chatgpt_noui,
53
  "endpoint": "https://openai.api2d.net/v1/chat/completions",
54
  "max_token": 4096,
55
- "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
56
  "token_cnt": get_token_num_gpt35,
57
  },
58
 
@@ -61,7 +79,7 @@ model_info = {
61
  "fn_without_ui": chatgpt_noui,
62
  "endpoint": "https://openai.api2d.net/v1/chat/completions",
63
  "max_token": 8192,
64
- "tokenizer": tiktoken.encoding_for_model("gpt-4"),
65
  "token_cnt": get_token_num_gpt4,
66
  },
67
 
@@ -71,7 +89,7 @@ model_info = {
71
  "fn_without_ui": chatglm_noui,
72
  "endpoint": None,
73
  "max_token": 1024,
74
- "tokenizer": tiktoken.encoding_for_model("gpt-3.5-turbo"),
75
  "token_cnt": get_token_num_gpt35,
76
  },
77
 
 
9
  2. predict_no_ui_long_connection:在实验过程中发现调用predict_no_ui处理长文档时,和openai的连接容易断掉,这个函数用stream的方式解决这个问题,同样支持多线程
10
  """
11
  import tiktoken
12
+ from functools import wraps, lru_cache
13
  from concurrent.futures import ThreadPoolExecutor
14
 
15
  from .bridge_chatgpt import predict_no_ui_long_connection as chatgpt_noui
 
18
  from .bridge_chatglm import predict_no_ui_long_connection as chatglm_noui
19
  from .bridge_chatglm import predict as chatglm_ui
20
 
21
+ # from .bridge_tgui import predict_no_ui_long_connection as tgui_noui
22
+ # from .bridge_tgui import predict as tgui_ui
23
 
24
  colors = ['#FF00FF', '#00FFFF', '#FF0000', '#990099', '#009999', '#990044']
25
 
26
+ class LazyloadTiktoken(object):
27
+ def __init__(self, model):
28
+ self.model = model
29
+
30
+ @staticmethod
31
+ @lru_cache(maxsize=128)
32
+ def get_encoder(model):
33
+ print('正在加载tokenizer,如果是第一次运行,可能需要一点时间下载参数')
34
+ tmp = tiktoken.encoding_for_model(model)
35
+ print('加载tokenizer完毕')
36
+ return tmp
37
+
38
+ def encode(self, *args, **kwargs):
39
+ encoder = self.get_encoder(self.model)
40
+ return encoder.encode(*args, **kwargs)
41
+
42
+ tokenizer_gpt35 = LazyloadTiktoken("gpt-3.5-turbo")
43
+ tokenizer_gpt4 = LazyloadTiktoken("gpt-4")
44
+ get_token_num_gpt35 = lambda txt: len(tokenizer_gpt35.encode(txt, disallowed_special=()))
45
+ get_token_num_gpt4 = lambda txt: len(tokenizer_gpt4.encode(txt, disallowed_special=()))
46
 
47
  model_info = {
48
  # openai
 
51
  "fn_without_ui": chatgpt_noui,
52
  "endpoint": "https://api.openai.com/v1/chat/completions",
53
  "max_token": 4096,
54
+ "tokenizer": tokenizer_gpt35,
55
  "token_cnt": get_token_num_gpt35,
56
  },
57
 
 
60
  "fn_without_ui": chatgpt_noui,
61
  "endpoint": "https://api.openai.com/v1/chat/completions",
62
  "max_token": 8192,
63
+ "tokenizer": tokenizer_gpt4,
64
  "token_cnt": get_token_num_gpt4,
65
  },
66
 
 
70
  "fn_without_ui": chatgpt_noui,
71
  "endpoint": "https://openai.api2d.net/v1/chat/completions",
72
  "max_token": 4096,
73
+ "tokenizer": tokenizer_gpt35,
74
  "token_cnt": get_token_num_gpt35,
75
  },
76
 
 
79
  "fn_without_ui": chatgpt_noui,
80
  "endpoint": "https://openai.api2d.net/v1/chat/completions",
81
  "max_token": 8192,
82
+ "tokenizer": tokenizer_gpt4,
83
  "token_cnt": get_token_num_gpt4,
84
  },
85
 
 
89
  "fn_without_ui": chatglm_noui,
90
  "endpoint": None,
91
  "max_token": 1024,
92
+ "tokenizer": tokenizer_gpt35,
93
  "token_cnt": get_token_num_gpt35,
94
  },
95
 
request_llm/bridge_chatglm.py CHANGED
@@ -5,6 +5,8 @@ import importlib
5
  from toolbox import update_ui, get_conf
6
  from multiprocessing import Process, Pipe
7
 
 
 
8
  #################################################################################
9
  class GetGLMHandle(Process):
10
  def __init__(self):
@@ -12,13 +14,26 @@ class GetGLMHandle(Process):
12
  self.parent, self.child = Pipe()
13
  self.chatglm_model = None
14
  self.chatglm_tokenizer = None
 
 
 
15
  self.start()
16
- print('初始化')
17
 
 
 
 
 
 
 
 
 
 
18
  def ready(self):
19
  return self.chatglm_model is not None
20
 
21
  def run(self):
 
 
22
  while True:
23
  try:
24
  if self.chatglm_model is None:
@@ -33,7 +48,12 @@ class GetGLMHandle(Process):
33
  else:
34
  break
35
  except:
36
- pass
 
 
 
 
 
37
  while True:
38
  kwargs = self.child.recv()
39
  try:
@@ -64,7 +84,11 @@ def predict_no_ui_long_connection(inputs, llm_kwargs, history=[], sys_prompt="",
64
  global glm_handle
65
  if glm_handle is None:
66
  glm_handle = GetGLMHandle()
67
- observe_window[0] = "ChatGLM尚未加载,加载需要一段时间。注意,取决于`config.py`的配置,ChatGLM消耗大量的内存(CPU)或显存(GPU),也许会导致低配计算机卡死 ……"
 
 
 
 
68
 
69
  # chatglm 没有 sys_prompt 接口,因此把prompt加入 history
70
  history_feedin = []
@@ -93,8 +117,11 @@ def predict(inputs, llm_kwargs, plugin_kwargs, chatbot, history=[], system_promp
93
  global glm_handle
94
  if glm_handle is None:
95
  glm_handle = GetGLMHandle()
96
- chatbot[-1] = (inputs, "ChatGLM尚未加载,加载需要一段时间。注意,取决于`config.py`的配置,ChatGLM消耗大量的内存(CPU)或显存(GPU),也许会导致低配计算机卡死 ……")
97
  yield from update_ui(chatbot=chatbot, history=[])
 
 
 
98
 
99
  if additional_fn is not None:
100
  import core_functional
 
5
  from toolbox import update_ui, get_conf
6
  from multiprocessing import Process, Pipe
7
 
8
+ load_message = "ChatGLM尚未加载,加载需要一段时间。注意,取决于`config.py`的配置,ChatGLM消耗大量的内存(CPU)或显存(GPU),也许会导致低配计算机卡死 ……"
9
+
10
  #################################################################################
11
  class GetGLMHandle(Process):
12
  def __init__(self):
 
14
  self.parent, self.child = Pipe()
15
  self.chatglm_model = None
16
  self.chatglm_tokenizer = None
17
+ self.info = ""
18
+ self.success = True
19
+ self.check_dependency()
20
  self.start()
 
21
 
22
+ def check_dependency(self):
23
+ try:
24
+ import sentencepiece
25
+ self.info = "依赖检测通过"
26
+ self.success = True
27
+ except:
28
+ self.info = "缺少ChatGLM的依赖,如果要使用ChatGLM,除了基础的pip依赖以外,您还需要运行`pip install -r request_llm/requirements_chatglm.txt`安装ChatGLM的依赖。"
29
+ self.success = False
30
+
31
  def ready(self):
32
  return self.chatglm_model is not None
33
 
34
  def run(self):
35
+ # 第一次运行,加载参数
36
+ retry = 0
37
  while True:
38
  try:
39
  if self.chatglm_model is None:
 
48
  else:
49
  break
50
  except:
51
+ retry += 1
52
+ if retry > 3:
53
+ self.child.send('[Local Message] Call ChatGLM fail 不能正常加载ChatGLM的参数。')
54
+ raise RuntimeError("不能正常加载ChatGLM的参数!")
55
+
56
+ # 进入任务等待状态
57
  while True:
58
  kwargs = self.child.recv()
59
  try:
 
84
  global glm_handle
85
  if glm_handle is None:
86
  glm_handle = GetGLMHandle()
87
+ observe_window[0] = load_message + "\n\n" + glm_handle.info
88
+ if not glm_handle.success:
89
+ error = glm_handle.info
90
+ glm_handle = None
91
+ raise RuntimeError(error)
92
 
93
  # chatglm 没有 sys_prompt 接口,因此把prompt加入 history
94
  history_feedin = []
 
117
  global glm_handle
118
  if glm_handle is None:
119
  glm_handle = GetGLMHandle()
120
+ chatbot[-1] = (inputs, load_message + "\n\n" + glm_handle.info)
121
  yield from update_ui(chatbot=chatbot, history=[])
122
+ if not glm_handle.success:
123
+ glm_handle = None
124
+ return
125
 
126
  if additional_fn is not None:
127
  import core_functional
toolbox.py CHANGED
@@ -25,7 +25,6 @@ def ArgsGeneralWrapper(f):
25
  装饰器函数,用于重组输入参数,改变输入参数的顺序与结构。
26
  """
27
  def decorated(cookies, max_length, llm_model, txt, txt2, top_p, temperature, chatbot, history, system_prompt, *args):
28
- from request_llm.bridge_all import model_info
29
  txt_passon = txt
30
  if txt == "" and txt2 != "": txt_passon = txt2
31
  # 引入一个有cookie的chatbot
 
25
  装饰器函数,用于重组输入参数,改变输入参数的顺序与结构。
26
  """
27
  def decorated(cookies, max_length, llm_model, txt, txt2, top_p, temperature, chatbot, history, system_prompt, *args):
 
28
  txt_passon = txt
29
  if txt == "" and txt2 != "": txt_passon = txt2
30
  # 引入一个有cookie的chatbot