3v324v23 commited on
Commit
2420d62
1 Parent(s): 3af0bbd

接入TGUI

Browse files
Files changed (3) hide show
  1. config.py +1 -1
  2. request_llm/README.md +9 -12
  3. request_llm/bridge_tgui.py +7 -4
config.py CHANGED
@@ -34,7 +34,7 @@ WEB_PORT = -1
34
  MAX_RETRY = 2
35
 
36
  # OpenAI模型选择是(gpt4现在只对申请成功的人开放)
37
- LLM_MODEL = "pygmalion-1.3b@localhost@7860" # "gpt-3.5-turbo"
38
 
39
  # OpenAI的API_URL
40
  API_URL = "https://api.openai.com/v1/chat/completions"
 
34
  MAX_RETRY = 2
35
 
36
  # OpenAI模型选择是(gpt4现在只对申请成功的人开放)
37
+ LLM_MODEL = "TGUI:galactica-1.3b@localhost:7860" # "gpt-3.5-turbo"
38
 
39
  # OpenAI的API_URL
40
  API_URL = "https://api.openai.com/v1/chat/completions"
request_llm/README.md CHANGED
@@ -2,7 +2,7 @@
2
 
3
  ## 1. 先运行text-generation
4
  ``` sh
5
- # 下载模型
6
  git clone https://github.com/oobabooga/text-generation-webui.git
7
 
8
  # 安装text-generation的额外依赖
@@ -12,28 +12,25 @@ pip install accelerate bitsandbytes flexgen gradio llamacpp markdown numpy peft
12
  cd text-generation-webui
13
 
14
  # 下载模型
15
- python download-model.py facebook/opt-1.3b
16
-
17
- # 其他可选如 facebook/galactica-1.3b
18
  # facebook/galactica-6.7b
19
  # facebook/galactica-120b
20
-
21
- # Pymalion 6B is a proof-of-concept dialogue model based on EleutherAI's GPT-J-6B.
22
- # facebook/pygmalion-1.3b
23
 
24
  # 启动text-generation,注意把模型的斜杠改成下划线
25
  python server.py --cpu --listen --listen-port 7860 --model facebook_galactica-1.3b
26
  ```
27
 
28
  ## 2. 修改config.py
 
 
 
29
  ```
30
- # LLM_MODEL格式为 [模型]@[ws地址] @[ws端口]
31
- LLM_MODEL = "pygmalion-1.3b@localhost@7860"
32
- ```
33
-
34
 
35
  ## 3. 运行!
36
- ```
37
  cd chatgpt-academic
38
  python main.py
39
  ```
 
2
 
3
  ## 1. 先运行text-generation
4
  ``` sh
5
+ # 下载模型( text-generation 这么牛的项目,别忘了给人家star )
6
  git clone https://github.com/oobabooga/text-generation-webui.git
7
 
8
  # 安装text-generation的额外依赖
 
12
  cd text-generation-webui
13
 
14
  # 下载模型
15
+ python download-model.py facebook/galactica-1.3b
16
+ # 其他可选如 facebook/opt-1.3b
 
17
  # facebook/galactica-6.7b
18
  # facebook/galactica-120b
19
+ # facebook/pygmalion-1.3b 等
20
+ # 详情见 https://github.com/oobabooga/text-generation-webui
 
21
 
22
  # 启动text-generation,注意把模型的斜杠改成下划线
23
  python server.py --cpu --listen --listen-port 7860 --model facebook_galactica-1.3b
24
  ```
25
 
26
  ## 2. 修改config.py
27
+ ``` sh
28
+ # LLM_MODEL格式较复杂 TGUI:[模型]@[ws地址]:[ws端口] , 端口要和上面给定的端口一致
29
+ LLM_MODEL = "TGUI:galactica-1.3b@localhost:7860"
30
  ```
 
 
 
 
31
 
32
  ## 3. 运行!
33
+ ``` sh
34
  cd chatgpt-academic
35
  python main.py
36
  ```
request_llm/bridge_tgui.py CHANGED
@@ -15,7 +15,10 @@ import importlib
15
  from toolbox import get_conf
16
  LLM_MODEL, = get_conf('LLM_MODEL')
17
 
18
- model_name, addr, port = LLM_MODEL.split('@')
 
 
 
19
 
20
  def random_hash():
21
  letters = string.ascii_lowercase + string.digits
@@ -117,11 +120,11 @@ def predict_tgui(inputs, top_p, temperature, chatbot=[], history=[], system_prom
117
  def run_coorotine(mutable):
118
  async def get_result(mutable):
119
  async for response in run(prompt):
120
- # Print intermediate steps
121
  mutable[0] = response
122
  asyncio.run(get_result(mutable))
123
 
124
- thread_listen = threading.Thread(target=run_coorotine, args=(mutable,))
125
  thread_listen.start()
126
 
127
  while thread_listen.is_alive():
@@ -145,7 +148,7 @@ def predict_tgui_no_ui(inputs, top_p, temperature, history=[], sys_prompt=""):
145
  def run_coorotine(mutable):
146
  async def get_result(mutable):
147
  async for response in run(prompt):
148
- # Print intermediate steps
149
  mutable[0] = response
150
  asyncio.run(get_result(mutable))
151
  thread_listen = threading.Thread(target=run_coorotine, args=(mutable,))
 
15
  from toolbox import get_conf
16
  LLM_MODEL, = get_conf('LLM_MODEL')
17
 
18
+ # "TGUI:galactica-1.3b@localhost:7860"
19
+ model_name, addr_port = LLM_MODEL.split('@')
20
+ assert ':' in addr_port, "LLM_MODEL 格式不正确!" + LLM_MODEL
21
+ addr, port = addr_port.split(':')
22
 
23
  def random_hash():
24
  letters = string.ascii_lowercase + string.digits
 
120
  def run_coorotine(mutable):
121
  async def get_result(mutable):
122
  async for response in run(prompt):
123
+ print(response[len(mutable[0]):])
124
  mutable[0] = response
125
  asyncio.run(get_result(mutable))
126
 
127
+ thread_listen = threading.Thread(target=run_coorotine, args=(mutable,), daemon=True)
128
  thread_listen.start()
129
 
130
  while thread_listen.is_alive():
 
148
  def run_coorotine(mutable):
149
  async def get_result(mutable):
150
  async for response in run(prompt):
151
+ print(response[len(mutable[0]):])
152
  mutable[0] = response
153
  asyncio.run(get_result(mutable))
154
  thread_listen = threading.Thread(target=run_coorotine, args=(mutable,))