add more tokenizer

vocab/__init__.py

@@ -46,7 +46,7 @@ tokenizer.special_tokens_map
 tokenizer.dependency [sentencepiece, tiktoken, icetk] 
 """
 
-Animal = Enum('Animal', 'ANT BEE CAT DOG')
+# Animal = Enum('Animal', 'ANT BEE CAT DOG')
 
 uniq_tokenizers = [
     ""
@@ -95,6 +95,9 @@ all_tokenizers = [
     "baichuan",
     "baichuan2",
     "internlm_chat_7b",
+    "internlm2_chat_7b",
+    "internlm2_math_7b",
+    "internlm_xcomposer_7b",
     "falcon_7b",
     "falcon_180b",
     # "goat",
@@ -111,7 +114,8 @@ all_tokenizers = [
     # 未分类
     "skywork_13b_base",
     "skywork_13b_math",
-    "mistral",
+    "mistral_7b",
+    "mixtral_8_7b",
     "t5_small",
     "t5_base",
     "t5_large",
@@ -119,6 +123,13 @@ all_tokenizers = [
     "fastchat_t5_3b",
     "pko_t5_large",
     "wizardcoder_15b_v1",
+    "yi_6b",
+    "yi_34b",
+    "yi_vl34b",
+    "orion_14b_chat",
+    "phi_1",
+    "phi_2",
+    "solar_10_7b",
 "wizardcoder_python_7b_v1",
 "wizardlm_7b_v1",
 "wizardmath_70b_v1",
@@ -128,7 +139,6 @@ all_tokenizers = [
 "deepseek_llm_7b_base",
 
 
-
 ]
 
 all_tokenizers = sorted(all_tokenizers)

vocab/internlm2_chat_7b/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("internlm/internlm2-chat-7b", trust_remote_code=True)

vocab/internlm2_math_7b/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("internlm/internlm2-math-7b", trust_remote_code=True)

vocab/internlm_xcomposer_7b/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("internlm/internlm-xcomposer-7b", trust_remote_code=True)

vocab/mixtral_8_7b/__init__.py

@@ -0,0 +1,2 @@
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1", trust_remote_code=True)

vocab/orion_14b_chat/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("OrionStarAI/Orion-14B-Chat", trust_remote_code=True)

vocab/phi_1/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-1", trust_remote_code=True)

vocab/phi_2/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-2", trust_remote_code=True)

vocab/solar_10_7b/__init__.py

@@ -0,0 +1,4 @@
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("Upstage/SOLAR-10.7B-v1.0")

vocab/yi_34b/__init__.py

@@ -0,0 +1,4 @@
+
+
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("01-ai/Yi-34B", trust_remote_code=True)

vocab/yi_6b/__init__.py

@@ -0,0 +1,3 @@
+
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("01-ai/Yi-6B", trust_remote_code=True)

vocab/yi_vl34b/__init__.py

@@ -0,0 +1,9 @@
+"""
+
+Yi-VL adopts the LLaVA architecture,
+"""
+
+
+
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("01-ai/Yi-VL-34B", trust_remote_code=True)