add aya
Browse files- vocab/__init__.py +1 -1
- vocab/aya_101/__init__.py +5 -0
vocab/__init__.py
CHANGED
|
@@ -158,7 +158,7 @@ all_tokenizers = [
|
|
| 158 |
"deepseek_llm_7b_base",
|
| 159 |
"gemma_7b",
|
| 160 |
"olmo_7b",
|
| 161 |
-
|
| 162 |
]
|
| 163 |
|
| 164 |
all_tokenizers = sorted(all_tokenizers)
|
|
|
|
| 158 |
"deepseek_llm_7b_base",
|
| 159 |
"gemma_7b",
|
| 160 |
"olmo_7b",
|
| 161 |
+
"aya_101",
|
| 162 |
]
|
| 163 |
|
| 164 |
all_tokenizers = sorted(all_tokenizers)
|
vocab/aya_101/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
from transformers import AutoTokenizer
|
| 4 |
+
|
| 5 |
+
tokenizer = AutoTokenizer.from_pretrained("CohereForAI/aya-101")
|