Spaces:
Runtime error
Runtime error
nroggendorff
commited on
Commit
•
152e4ca
1
Parent(s):
adab631
Update app.py
Browse files
app.py
CHANGED
@@ -8,7 +8,6 @@ import torch
|
|
8 |
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
|
9 |
from threading import Thread
|
10 |
import asyncio
|
11 |
-
import multiprocessing
|
12 |
|
13 |
TOKEN = os.environ["TOKEN"]
|
14 |
|
@@ -18,24 +17,24 @@ client = commands.Bot(command_prefix='?', intents=discord.Intents.all())
|
|
18 |
# formatted_content = {"role": creator, "content": content}
|
19 |
# conversation.append(formatted_content)
|
20 |
|
21 |
-
|
22 |
-
torch.set_default_device("cuda")
|
23 |
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
|
36 |
-
|
37 |
-
system_prompt = "<|im_start|>system\nYou are discord user Jimmy Donaldson. You are an american youtuber by the name of MrBeast. Your sole purpose in life is to get people to laugh at you. You make innapropriate jokes, and aren't very funny.<|im_end|>"
|
38 |
|
|
|
|
|
39 |
messages = system_prompt + "".join(["".join(["\n<|im_start|>user\n" + item[0], "<|im_end|>\n<|im_start|>assistant\n" + item[1]]) for item in history_transformer_format])
|
40 |
|
41 |
input_ids = tokenizer([messages], return_tensors="pt").to('cuda').input_ids
|
@@ -49,8 +48,6 @@ def predict(message, history):
|
|
49 |
temperature=0.7,
|
50 |
num_beams=1
|
51 |
)
|
52 |
-
print("======== Finished generating output_ids. ========")
|
53 |
-
|
54 |
output_text = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]
|
55 |
return output_text
|
56 |
|
@@ -126,11 +123,8 @@ def rungradio():
|
|
126 |
def runclient():
|
127 |
client.run(TOKEN)
|
128 |
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
gradiotask.join()
|
136 |
-
clienttask.join()
|
|
|
8 |
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
|
9 |
from threading import Thread
|
10 |
import asyncio
|
|
|
11 |
|
12 |
TOKEN = os.environ["TOKEN"]
|
13 |
|
|
|
17 |
# formatted_content = {"role": creator, "content": content}
|
18 |
# conversation.append(formatted_content)
|
19 |
|
20 |
+
torch.set_default_device("cuda")
|
|
|
21 |
|
22 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
23 |
+
"cognitivecomputations/dolphin-2.9.1-mixtral-1x22b",
|
24 |
+
trust_remote_code=True
|
25 |
+
)
|
26 |
|
27 |
+
model = AutoModelForCausalLM.from_pretrained(
|
28 |
+
"cognitivecomputations/dolphin-2.9.1-mixtral-1x22b",
|
29 |
+
torch_dtype="auto",
|
30 |
+
load_in_4bit=True,
|
31 |
+
trust_remote_code=True
|
32 |
+
)
|
33 |
|
34 |
+
system_prompt = "<|im_start|>system\nYou are discord user Jimmy Donaldson. You are an american youtuber by the name of MrBeast. Your sole purpose in life is to get people to laugh at you. You make innapropriate jokes, and aren't very funny.<|im_end|>"
|
|
|
35 |
|
36 |
+
def predict(message, history):
|
37 |
+
history_transformer_format = history + [[message, ""]]
|
38 |
messages = system_prompt + "".join(["".join(["\n<|im_start|>user\n" + item[0], "<|im_end|>\n<|im_start|>assistant\n" + item[1]]) for item in history_transformer_format])
|
39 |
|
40 |
input_ids = tokenizer([messages], return_tensors="pt").to('cuda').input_ids
|
|
|
48 |
temperature=0.7,
|
49 |
num_beams=1
|
50 |
)
|
|
|
|
|
51 |
output_text = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]
|
52 |
return output_text
|
53 |
|
|
|
123 |
def runclient():
|
124 |
client.run(TOKEN)
|
125 |
|
126 |
+
async def main():
|
127 |
+
loop = asyncio.get_running_loop()
|
128 |
+
gradio_task = loop.run_in_executor(None, rungradio)
|
129 |
+
discord_task = asyncio.create_task(runclient())
|
130 |
+
await asyncio.gather(gradio_task, discord_task)
|
|
|
|
|
|