Msp olivierdehaene HF staff commited on
Commit
41b6b43
0 Parent(s):

Duplicate from olivierdehaene/chat-llm-streaming

Browse files

Co-authored-by: Olivier Dehaene <olivierdehaene@users.noreply.huggingface.co>

Files changed (4) hide show
  1. .gitattributes +34 -0
  2. README.md +13 -0
  3. app.py +318 -0
  4. requirements.txt +2 -0
.gitattributes ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Chat Llm Streaming
3
+ emoji: 📊
4
+ colorFrom: blue
5
+ colorTo: gray
6
+ sdk: gradio
7
+ sdk_version: 3.20.1
8
+ app_file: app.py
9
+ pinned: false
10
+ duplicated_from: olivierdehaene/chat-llm-streaming
11
+ ---
12
+
13
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,318 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import gradio as gr
4
+
5
+ from text_generation import Client, InferenceAPIClient
6
+
7
+ openchat_preprompt = (
8
+ "\n<human>: Hi!\n<bot>: My name is Bot, model version is 0.15, part of an open-source kit for "
9
+ "fine-tuning new bots! I was created by Together, LAION, and Ontocord.ai and the open-source "
10
+ "community. I am not human, not evil and not alive, and thus have no thoughts and feelings, "
11
+ "but I am programmed to be helpful, polite, honest, and friendly.\n"
12
+ )
13
+
14
+
15
+ def get_client(model: str):
16
+ if model == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
17
+ return Client(os.getenv("OPENCHAT_API_URL"))
18
+ return InferenceAPIClient(model, token=os.getenv("HF_TOKEN", None))
19
+
20
+
21
+ def get_usernames(model: str):
22
+ """
23
+ Returns:
24
+ (str, str, str, str): pre-prompt, username, bot name, separator
25
+ """
26
+ if model == "OpenAssistant/oasst-sft-1-pythia-12b":
27
+ return "", "<|prompter|>", "<|assistant|>", "<|endoftext|>"
28
+ if model == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
29
+ return openchat_preprompt, "<human>: ", "<bot>: ", "\n"
30
+ return "", "User: ", "Assistant: ", "\n"
31
+
32
+
33
+ def predict(
34
+ model: str,
35
+ inputs: str,
36
+ typical_p: float,
37
+ top_p: float,
38
+ temperature: float,
39
+ top_k: int,
40
+ repetition_penalty: float,
41
+ watermark: bool,
42
+ chatbot,
43
+ history,
44
+ ):
45
+ client = get_client(model)
46
+ preprompt, user_name, assistant_name, sep = get_usernames(model)
47
+
48
+ history.append(inputs)
49
+
50
+ past = []
51
+ for data in chatbot:
52
+ user_data, model_data = data
53
+
54
+ if not user_data.startswith(user_name):
55
+ user_data = user_name + user_data
56
+ if not model_data.startswith(sep + assistant_name):
57
+ model_data = sep + assistant_name + model_data
58
+
59
+ past.append(user_data + model_data.rstrip() + sep)
60
+
61
+ if not inputs.startswith(user_name):
62
+ inputs = user_name + inputs
63
+
64
+ total_inputs = preprompt + "".join(past) + inputs + sep + assistant_name.rstrip()
65
+
66
+ partial_words = ""
67
+
68
+ if model == "OpenAssistant/oasst-sft-1-pythia-12b":
69
+ iterator = client.generate_stream(
70
+ total_inputs,
71
+ typical_p=typical_p,
72
+ truncate=1000,
73
+ watermark=watermark,
74
+ max_new_tokens=500,
75
+ )
76
+ else:
77
+ iterator = client.generate_stream(
78
+ total_inputs,
79
+ top_p=top_p if top_p < 1.0 else None,
80
+ top_k=top_k,
81
+ truncate=1000,
82
+ repetition_penalty=repetition_penalty,
83
+ watermark=watermark,
84
+ temperature=temperature,
85
+ max_new_tokens=500,
86
+ stop_sequences=[user_name.rstrip(), assistant_name.rstrip()],
87
+ )
88
+
89
+ for i, response in enumerate(iterator):
90
+ if response.token.special:
91
+ continue
92
+
93
+ partial_words = partial_words + response.token.text
94
+ if partial_words.endswith(user_name.rstrip()):
95
+ partial_words = partial_words.rstrip(user_name.rstrip())
96
+ if partial_words.endswith(assistant_name.rstrip()):
97
+ partial_words = partial_words.rstrip(assistant_name.rstrip())
98
+
99
+ if i == 0:
100
+ history.append(" " + partial_words)
101
+ elif response.token.text not in user_name:
102
+ history[-1] = partial_words
103
+
104
+ chat = [
105
+ (history[i].strip(), history[i + 1].strip())
106
+ for i in range(0, len(history) - 1, 2)
107
+ ]
108
+ yield chat, history
109
+
110
+
111
+ def reset_textbox():
112
+ return gr.update(value="")
113
+
114
+
115
+ def radio_on_change(
116
+ value: str,
117
+ disclaimer,
118
+ typical_p,
119
+ top_p,
120
+ top_k,
121
+ temperature,
122
+ repetition_penalty,
123
+ watermark,
124
+ ):
125
+ if value == "OpenAssistant/oasst-sft-1-pythia-12b":
126
+ typical_p = typical_p.update(value=0.2, visible=True)
127
+ top_p = top_p.update(visible=False)
128
+ top_k = top_k.update(visible=False)
129
+ temperature = temperature.update(visible=False)
130
+ disclaimer = disclaimer.update(visible=False)
131
+ repetition_penalty = repetition_penalty.update(visible=False)
132
+ watermark = watermark.update(False)
133
+ elif value == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
134
+ typical_p = typical_p.update(visible=False)
135
+ top_p = top_p.update(value=0.25, visible=True)
136
+ top_k = top_k.update(value=50, visible=True)
137
+ temperature = temperature.update(value=0.6, visible=True)
138
+ repetition_penalty = repetition_penalty.update(value=1.01, visible=True)
139
+ watermark = watermark.update(False)
140
+ disclaimer = disclaimer.update(visible=True)
141
+ else:
142
+ typical_p = typical_p.update(visible=False)
143
+ top_p = top_p.update(value=0.95, visible=True)
144
+ top_k = top_k.update(value=4, visible=True)
145
+ temperature = temperature.update(value=0.5, visible=True)
146
+ repetition_penalty = repetition_penalty.update(value=1.03, visible=True)
147
+ watermark = watermark.update(True)
148
+ disclaimer = disclaimer.update(visible=False)
149
+ return (
150
+ disclaimer,
151
+ typical_p,
152
+ top_p,
153
+ top_k,
154
+ temperature,
155
+ repetition_penalty,
156
+ watermark,
157
+ )
158
+
159
+
160
+ title = """<h1 align="center">Large Language Model Chat API</h1>"""
161
+ description = """Language models can be conditioned to act like dialogue agents through a conversational prompt that typically takes the form:
162
+
163
+ ```
164
+ User: <utterance>
165
+ Assistant: <utterance>
166
+ User: <utterance>
167
+ Assistant: <utterance>
168
+ ...
169
+ ```
170
+
171
+ In this app, you can explore the outputs of multiple LLMs when prompted in this way.
172
+ """
173
+
174
+ text_generation_inference = """
175
+ <div align="center">Powered by: <a href=https://github.com/huggingface/text-generation-inference>Text Generation Inference</a></div>
176
+ """
177
+
178
+ openchat_disclaimer = """
179
+ <div align="center">Checkout the official <a href=https://huggingface.co/spaces/togethercomputer/OpenChatKit>OpenChatKit feedback app</a> for the full experience.</div>
180
+ """
181
+
182
+ with gr.Blocks(
183
+ css="""#col_container {margin-left: auto; margin-right: auto;}
184
+ #chatbot {height: 520px; overflow: auto;}"""
185
+ ) as demo:
186
+ gr.HTML(title)
187
+ gr.Markdown(text_generation_inference, visible=True)
188
+ with gr.Column(elem_id="col_container"):
189
+ model = gr.Radio(
190
+ value="OpenAssistant/oasst-sft-1-pythia-12b",
191
+ choices=[
192
+ "OpenAssistant/oasst-sft-1-pythia-12b",
193
+ # "togethercomputer/GPT-NeoXT-Chat-Base-20B",
194
+ "google/flan-t5-xxl",
195
+ "google/flan-ul2",
196
+ "bigscience/bloom",
197
+ "bigscience/bloomz",
198
+ "EleutherAI/gpt-neox-20b",
199
+ ],
200
+ label="Model",
201
+ interactive=True,
202
+ )
203
+
204
+ chatbot = gr.Chatbot(elem_id="chatbot")
205
+ inputs = gr.Textbox(
206
+ placeholder="Hi there!", label="Type an input and press Enter"
207
+ )
208
+ disclaimer = gr.Markdown(openchat_disclaimer, visible=False)
209
+ state = gr.State([])
210
+ b1 = gr.Button()
211
+
212
+ with gr.Accordion("Parameters", open=False):
213
+ typical_p = gr.Slider(
214
+ minimum=-0,
215
+ maximum=1.0,
216
+ value=0.2,
217
+ step=0.05,
218
+ interactive=True,
219
+ label="Typical P mass",
220
+ )
221
+ top_p = gr.Slider(
222
+ minimum=-0,
223
+ maximum=1.0,
224
+ value=0.25,
225
+ step=0.05,
226
+ interactive=True,
227
+ label="Top-p (nucleus sampling)",
228
+ visible=False,
229
+ )
230
+ temperature = gr.Slider(
231
+ minimum=-0,
232
+ maximum=5.0,
233
+ value=0.6,
234
+ step=0.1,
235
+ interactive=True,
236
+ label="Temperature",
237
+ visible=False,
238
+ )
239
+ top_k = gr.Slider(
240
+ minimum=1,
241
+ maximum=50,
242
+ value=50,
243
+ step=1,
244
+ interactive=True,
245
+ label="Top-k",
246
+ visible=False,
247
+ )
248
+ repetition_penalty = gr.Slider(
249
+ minimum=0.1,
250
+ maximum=3.0,
251
+ value=1.03,
252
+ step=0.01,
253
+ interactive=True,
254
+ label="Repetition Penalty",
255
+ visible=False,
256
+ )
257
+ watermark = gr.Checkbox(value=False, label="Text watermarking")
258
+
259
+ model.change(
260
+ lambda value: radio_on_change(
261
+ value,
262
+ disclaimer,
263
+ typical_p,
264
+ top_p,
265
+ top_k,
266
+ temperature,
267
+ repetition_penalty,
268
+ watermark,
269
+ ),
270
+ inputs=model,
271
+ outputs=[
272
+ disclaimer,
273
+ typical_p,
274
+ top_p,
275
+ top_k,
276
+ temperature,
277
+ repetition_penalty,
278
+ watermark,
279
+ ],
280
+ )
281
+
282
+ inputs.submit(
283
+ predict,
284
+ [
285
+ model,
286
+ inputs,
287
+ typical_p,
288
+ top_p,
289
+ temperature,
290
+ top_k,
291
+ repetition_penalty,
292
+ watermark,
293
+ chatbot,
294
+ state,
295
+ ],
296
+ [chatbot, state],
297
+ )
298
+ b1.click(
299
+ predict,
300
+ [
301
+ model,
302
+ inputs,
303
+ typical_p,
304
+ top_p,
305
+ temperature,
306
+ top_k,
307
+ repetition_penalty,
308
+ watermark,
309
+ chatbot,
310
+ state,
311
+ ],
312
+ [chatbot, state],
313
+ )
314
+ b1.click(reset_textbox, [], [inputs])
315
+ inputs.submit(reset_textbox, [], [inputs])
316
+
317
+ gr.Markdown(description)
318
+ demo.queue(concurrency_count=16).launch(debug=True)
requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ text-generation==0.3.0
2
+ gradio==3.20.1