voidful
/

SmolLM2-360M-Instruct-ASR

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

voidful commited on 13 days ago

Commit

838059d

•

1 Parent(s): f6321de

Upload tokenizer

Files changed (2) hide show

added_tokens.json +5 -1
tokenizer_config.json +32 -0

added_tokens.json CHANGED Viewed

@@ -1538,6 +1538,8 @@
   "<|hu|>": 50286,
   "<|hy|>": 50312,
   "<|id|>": 50275,
   "<|is|>": 50311,
   "<|it|>": 50274,
   "<|ja|>": 50266,
@@ -1607,5 +1609,7 @@
   "<|yi|>": 50335,
   "<|yo|>": 50325,
   "<|yue|>": 50358,
-  "<|zh|>": 50260
 }

   "<|hu|>": 50286,
   "<|hy|>": 50312,
   "<|id|>": 50275,
+  "<|im_end|>": 51869,
+  "<|im_start|>": 51868,
   "<|is|>": 50311,
   "<|it|>": 50274,
   "<|ja|>": 50266,
   "<|yi|>": 50335,
   "<|yo|>": 50325,
   "<|yue|>": 50358,
+  "<|zh|>": 50260,
+  "[END_PAD]": 51867,
+  "[PAD]": 51866
 }

tokenizer_config.json CHANGED Viewed

@@ -12872,6 +12872,38 @@
       "rstrip": false,
       "single_word": false,
       "special": false
     }
   },
   "additional_special_tokens": [

       "rstrip": false,
       "single_word": false,
       "special": false
+    },
+    "51866": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51867": {
+      "content": "[END_PAD]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51868": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51869": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "additional_special_tokens": [