Spaces:
Runtime error
Runtime error
nroggendorff
commited on
Commit
•
cb04241
1
Parent(s):
f363de8
Update app.py
Browse files
app.py
CHANGED
@@ -2,23 +2,61 @@ import discord
|
|
2 |
from discord.ext import commands
|
3 |
import requests
|
4 |
import os
|
|
|
5 |
|
6 |
-
TOKEN = os.environ
|
7 |
|
8 |
client = commands.Bot(command_prefix='?', intents=discord.Intents.all())
|
9 |
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
|
17 |
def generate(content):
|
18 |
-
append(content, "user")
|
19 |
-
|
20 |
-
response
|
21 |
-
append(response, "assistant")
|
22 |
return response
|
23 |
|
24 |
async def respond(channel, message):
|
|
|
2 |
from discord.ext import commands
|
3 |
import requests
|
4 |
import os
|
5 |
+
import spaces
|
6 |
|
7 |
+
TOKEN = os.environ["TOKEN"]
|
8 |
|
9 |
client = commands.Bot(command_prefix='?', intents=discord.Intents.all())
|
10 |
|
11 |
+
#def append(content, creator):
|
12 |
+
# formatted_content = {"role": creator, "content": content}
|
13 |
+
# conversation.append(formatted_content)
|
14 |
+
|
15 |
+
@spaces.GPU(duration=120)
|
16 |
+
def predict(message, history):
|
17 |
+
torch.set_default_device("cuda")
|
18 |
+
|
19 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
20 |
+
"cognitivecomputations/dolphin-2.8-mistral-7b-v02",
|
21 |
+
trust_remote_code=True
|
22 |
+
)
|
23 |
+
model = AutoModelForCausalLM.from_pretrained(
|
24 |
+
"cognitivecomputations/dolphin-2.8-mistral-7b-v02",
|
25 |
+
torch_dtype="auto",
|
26 |
+
load_in_4bit=True,
|
27 |
+
trust_remote_code=True
|
28 |
+
)
|
29 |
+
history_transformer_format = history + [[message, ""]]
|
30 |
+
|
31 |
+
system_prompt = "<|im_start|>system\nYou are discord user Jimmy Donaldson. You are an american youtuber by the name of MrBeast. Your sole purpose in life is to get people to laugh at you. You make innapropriate jokes, and aren't very funny.<|im_end|>"
|
32 |
+
messages = system_prompt + "".join(["".join(["\n<|im_start|>user\n" + item[0], "<|im_end|>\n<|im_start|>assistant\n" + item[1]]) for item in history_transformer_format])
|
33 |
+
input_ids = tokenizer([messages], return_tensors="pt").to('cuda')
|
34 |
+
streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
|
35 |
+
generate_kwargs = dict(
|
36 |
+
input_ids,
|
37 |
+
streamer=streamer,
|
38 |
+
max_new_tokens=10000,
|
39 |
+
do_sample=True,
|
40 |
+
top_p=0.95,
|
41 |
+
top_k=50,
|
42 |
+
temperature=0.7,
|
43 |
+
num_beams=1
|
44 |
+
)
|
45 |
+
t = Thread(target=model.generate, kwargs=generate_kwargs)
|
46 |
+
t.start()
|
47 |
+
|
48 |
+
final_message = []
|
49 |
+
for new_token in streamer:
|
50 |
+
final_message.append(new_token)
|
51 |
+
if '<|im_end|>' in new_token:
|
52 |
+
break
|
53 |
+
|
54 |
+
return ''.join(final_message)
|
55 |
|
56 |
def generate(content):
|
57 |
+
#append(content, "user")
|
58 |
+
predict(content, [])
|
59 |
+
#append(response, "assistant")
|
|
|
60 |
return response
|
61 |
|
62 |
async def respond(channel, message):
|