update
Browse files- utils/symbol.py +1 -0
- vocab/__init__.py +1 -1
utils/symbol.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
special_symbols: https://github.com/google/sentencepiece/blob/master/doc/special_symbols.md
|
|
|
|
| 3 |
"""
|
| 4 |
|
| 5 |
import sys
|
|
|
|
| 1 |
"""
|
| 2 |
special_symbols: https://github.com/google/sentencepiece/blob/master/doc/special_symbols.md
|
| 3 |
+
emoji:
|
| 4 |
"""
|
| 5 |
|
| 6 |
import sys
|
vocab/__init__.py
CHANGED
|
@@ -44,7 +44,7 @@ uniq_tokenizers = [
|
|
| 44 |
|
| 45 |
all_tokenizers = [
|
| 46 |
"gpt_35_turbo",
|
| 47 |
-
"
|
| 48 |
"gpt2",
|
| 49 |
"gpt2_chinese",
|
| 50 |
"bert_base_cased",
|
|
|
|
| 44 |
|
| 45 |
all_tokenizers = [
|
| 46 |
"gpt_35_turbo",
|
| 47 |
+
"gpt_4",
|
| 48 |
"gpt2",
|
| 49 |
"gpt2_chinese",
|
| 50 |
"bert_base_cased",
|