3v324v23 commited on
Commit
3af0bbd
1 Parent(s): bfa6661

成功借助tgui调用更多LLM

Browse files
Files changed (4) hide show
  1. config.py +1 -1
  2. predict.py +4 -1
  3. request_llm/README.md +39 -0
  4. request_llm/bridge_tgui.py +26 -8
config.py CHANGED
@@ -1,5 +1,5 @@
1
  # [step 1]>> 例如: API_KEY = "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" (此key无效)
2
- API_KEY = "sk-此处填API密钥"
3
 
4
  # [step 2]>> 改为True应用代理,如果直接在海外服务器部署,此处不修改
5
  USE_PROXY = False
 
1
  # [step 1]>> 例如: API_KEY = "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" (此key无效)
2
+ API_KEY = "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"
3
 
4
  # [step 2]>> 改为True应用代理,如果直接在海外服务器部署,此处不修改
5
  USE_PROXY = False
predict.py CHANGED
@@ -244,7 +244,10 @@ def generate_payload(inputs, top_p, temperature, history, system_prompt, stream)
244
 
245
 
246
  if not LLM_MODEL.startswith('gpt'):
247
- from request_llm.bridge_tgui import predict_tgui
 
248
  predict = predict_tgui
 
 
249
 
250
 
 
244
 
245
 
246
  if not LLM_MODEL.startswith('gpt'):
247
+ # 函数重载到另一个文件
248
+ from request_llm.bridge_tgui import predict_tgui, predict_tgui_no_ui
249
  predict = predict_tgui
250
+ predict_no_ui = predict_tgui_no_ui
251
+ predict_no_ui_long_connection = predict_tgui_no_ui
252
 
253
 
request_llm/README.md ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 如何使用其他大语言模型
2
+
3
+ ## 1. 先运行text-generation
4
+ ``` sh
5
+ # 下载模型
6
+ git clone https://github.com/oobabooga/text-generation-webui.git
7
+
8
+ # 安装text-generation的额外依赖
9
+ pip install accelerate bitsandbytes flexgen gradio llamacpp markdown numpy peft requests rwkv safetensors sentencepiece tqdm datasets git+https://github.com/huggingface/transformers
10
+
11
+ # 切换路径
12
+ cd text-generation-webui
13
+
14
+ # 下载模型
15
+ python download-model.py facebook/opt-1.3b
16
+
17
+ # 其他可选如 facebook/galactica-1.3b
18
+ # facebook/galactica-6.7b
19
+ # facebook/galactica-120b
20
+
21
+ # Pymalion 6B is a proof-of-concept dialogue model based on EleutherAI's GPT-J-6B.
22
+ # facebook/pygmalion-1.3b
23
+
24
+ # 启动text-generation,注意把模型的斜杠改成下划线
25
+ python server.py --cpu --listen --listen-port 7860 --model facebook_galactica-1.3b
26
+ ```
27
+
28
+ ## 2. 修改config.py
29
+ ```
30
+ # LLM_MODEL格式为 [模型]@[ws地址] @[ws端口]
31
+ LLM_MODEL = "pygmalion-1.3b@localhost@7860"
32
+ ```
33
+
34
+
35
+ ## 3. 运行!
36
+ ```
37
+ cd chatgpt-academic
38
+ python main.py
39
+ ```
request_llm/bridge_tgui.py CHANGED
@@ -11,6 +11,7 @@ import websockets
11
  import logging
12
  import time
13
  import threading
 
14
  from toolbox import get_conf
15
  LLM_MODEL, = get_conf('LLM_MODEL')
16
 
@@ -22,7 +23,7 @@ def random_hash():
22
 
23
  async def run(context):
24
  params = {
25
- 'max_new_tokens': 200,
26
  'do_sample': True,
27
  'temperature': 0.5,
28
  'top_p': 0.9,
@@ -103,9 +104,10 @@ def predict_tgui(inputs, top_p, temperature, chatbot=[], history=[], system_prom
103
  if "PreProcess" in functional[additional_fn]: inputs = functional[additional_fn]["PreProcess"](inputs) # 获取预处理函数(如果有的话)
104
  inputs = functional[additional_fn]["Prefix"] + inputs + functional[additional_fn]["Suffix"]
105
 
106
- raw_input = inputs
107
  logging.info(f'[raw_input] {raw_input}')
108
- chatbot.append((inputs, ""))
 
109
  yield chatbot, history, "等待响应"
110
 
111
  prompt = inputs
@@ -113,11 +115,11 @@ def predict_tgui(inputs, top_p, temperature, chatbot=[], history=[], system_prom
113
 
114
  mutable = [""]
115
  def run_coorotine(mutable):
116
- async def get_result():
117
  async for response in run(prompt):
118
  # Print intermediate steps
119
- mutable += response
120
- asyncio.run(get_result())
121
 
122
  thread_listen = threading.Thread(target=run_coorotine, args=(mutable,))
123
  thread_listen.start()
@@ -129,9 +131,25 @@ def predict_tgui(inputs, top_p, temperature, chatbot=[], history=[], system_prom
129
  tgui_say = mutable[0]
130
  history[-1] = tgui_say
131
  chatbot[-1] = (history[-2], history[-1])
132
- yield chatbot, history, status_text
133
 
134
  logging.info(f'[response] {tgui_say}')
135
 
136
 
137
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  import logging
12
  import time
13
  import threading
14
+ import importlib
15
  from toolbox import get_conf
16
  LLM_MODEL, = get_conf('LLM_MODEL')
17
 
 
23
 
24
  async def run(context):
25
  params = {
26
+ 'max_new_tokens': 1024,
27
  'do_sample': True,
28
  'temperature': 0.5,
29
  'top_p': 0.9,
 
104
  if "PreProcess" in functional[additional_fn]: inputs = functional[additional_fn]["PreProcess"](inputs) # 获取预处理函数(如果有的话)
105
  inputs = functional[additional_fn]["Prefix"] + inputs + functional[additional_fn]["Suffix"]
106
 
107
+ raw_input = "What I would like to say is the following: " + inputs
108
  logging.info(f'[raw_input] {raw_input}')
109
+ history.extend([inputs, ""])
110
+ chatbot.append([inputs, ""])
111
  yield chatbot, history, "等待响应"
112
 
113
  prompt = inputs
 
115
 
116
  mutable = [""]
117
  def run_coorotine(mutable):
118
+ async def get_result(mutable):
119
  async for response in run(prompt):
120
  # Print intermediate steps
121
+ mutable[0] = response
122
+ asyncio.run(get_result(mutable))
123
 
124
  thread_listen = threading.Thread(target=run_coorotine, args=(mutable,))
125
  thread_listen.start()
 
131
  tgui_say = mutable[0]
132
  history[-1] = tgui_say
133
  chatbot[-1] = (history[-2], history[-1])
134
+ yield chatbot, history, "status_text"
135
 
136
  logging.info(f'[response] {tgui_say}')
137
 
138
 
139
+
140
+ def predict_tgui_no_ui(inputs, top_p, temperature, history=[], sys_prompt=""):
141
+ raw_input = "What I would like to say is the following: " + inputs
142
+ prompt = inputs
143
+ tgui_say = ""
144
+ mutable = [""]
145
+ def run_coorotine(mutable):
146
+ async def get_result(mutable):
147
+ async for response in run(prompt):
148
+ # Print intermediate steps
149
+ mutable[0] = response
150
+ asyncio.run(get_result(mutable))
151
+ thread_listen = threading.Thread(target=run_coorotine, args=(mutable,))
152
+ thread_listen.start()
153
+ thread_listen.join()
154
+ tgui_say = mutable[0]
155
+ return tgui_say