Spaces:
Sleeping
Sleeping
Daniel Fried
commited on
Commit
·
003db9a
1
Parent(s):
50e8b8f
add use_normal_tokenizers option
Browse files- modules/app.py +6 -2
modules/app.py
CHANGED
@@ -1,8 +1,12 @@
|
|
1 |
import sys
|
2 |
from typing import List
|
3 |
import traceback
|
|
|
4 |
# needs to be imported *before* transformers
|
5 |
-
|
|
|
|
|
|
|
6 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
7 |
import json
|
8 |
|
@@ -20,7 +24,7 @@ import json
|
|
20 |
PORT = 7860
|
21 |
VERBOSE = False
|
22 |
|
23 |
-
BIG_MODEL =
|
24 |
|
25 |
if BIG_MODEL:
|
26 |
CUDA = True
|
|
|
1 |
import sys
|
2 |
from typing import List
|
3 |
import traceback
|
4 |
+
import os
|
5 |
# needs to be imported *before* transformers
|
6 |
+
if os.path.exists('use_normal_tokenizers'):
|
7 |
+
import tokenizers
|
8 |
+
else:
|
9 |
+
import tokenizers_patch
|
10 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
11 |
import json
|
12 |
|
|
|
24 |
PORT = 7860
|
25 |
VERBOSE = False
|
26 |
|
27 |
+
BIG_MODEL = False
|
28 |
|
29 |
if BIG_MODEL:
|
30 |
CUDA = True
|