Update app.py
Browse files
app.py
CHANGED
@@ -63,7 +63,7 @@ model = AutoModelForCausalLM.from_pretrained(model_id, token= token,
|
|
63 |
torch_dtype=torch.float16,
|
64 |
# load_in_8bit=True,
|
65 |
# # # torch_dtype=torch.fl,
|
66 |
-
|
67 |
low_cpu_mem_usage=True,
|
68 |
# device_map='cuda',
|
69 |
# device_map=accelerator.device_map,
|
@@ -122,7 +122,7 @@ def respond(
|
|
122 |
gen_text = tokenizer.decode(gen_tokens[0])
|
123 |
print(gen_text)
|
124 |
gen_text= gen_text.replace(input_str,'')
|
125 |
-
gen_text= gen_text.replace('<|
|
126 |
|
127 |
yield gen_text
|
128 |
|
|
|
63 |
torch_dtype=torch.float16,
|
64 |
# load_in_8bit=True,
|
65 |
# # # torch_dtype=torch.fl,
|
66 |
+
attn_implementation="flash_attention_2",
|
67 |
low_cpu_mem_usage=True,
|
68 |
# device_map='cuda',
|
69 |
# device_map=accelerator.device_map,
|
|
|
122 |
gen_text = tokenizer.decode(gen_tokens[0])
|
123 |
print(gen_text)
|
124 |
gen_text= gen_text.replace(input_str,'')
|
125 |
+
gen_text= gen_text.replace('<|im_end|>','')
|
126 |
|
127 |
yield gen_text
|
128 |
|